LLVMFuzzerTestOneInput: 29| 6.85k|extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { 30| 6.85k| FuzzedDataProvider fdp(data, size); 31| 6.85k| constexpr int kMaxStringSize = 1024; 32| 6.85k| std::string source = fdp.ConsumeRandomLengthString(kMaxStringSize); 33| 27.4k| for (auto& e : simdutf::get_available_implementations()) { ------------------ | Branch (33:16): [True: 27.4k, False: 6.85k] ------------------ 34| 27.4k| if (!e->supported_by_runtime_system()) { ------------------ | Branch (34:9): [True: 6.85k, False: 20.5k] ------------------ 35| 6.85k| continue; 36| 6.85k| } 37| | /** 38| | * Transcoding from UTF-8 to UTF-16LE. 39| | */ 40| 20.5k| bool validutf8 = e->validate_utf8(source.c_str(), source.size()); 41| 20.5k| auto rutf8 = e->validate_utf8_with_errors(source.c_str(), source.size()); 42| 20.5k| if (validutf8 != (rutf8.error == simdutf::SUCCESS)) { // they should agree ------------------ | Branch (42:9): [True: 0, False: 20.5k] ------------------ 43| 0| print_input(source, e); 44| 0| abort(); 45| 0| } 46| 20.5k| if (validutf8) { ------------------ | Branch (46:9): [True: 18.9k, False: 1.62k] ------------------ 47| | // We need a buffer of size where to write the UTF-16LE words. 48| 18.9k| size_t expected_utf16words = 49| 18.9k| e->utf16_length_from_utf8(source.c_str(), source.size()); 50| 18.9k| std::unique_ptr utf16_output{ 51| 18.9k| new char16_t[expected_utf16words]}; 52| | // convert to UTF-16LE 53| 18.9k| size_t utf16words = e->convert_utf8_to_utf16le( 54| 18.9k| source.c_str(), source.size(), utf16_output.get()); 55| | // It wrote utf16words * sizeof(char16_t) bytes. 56| 18.9k| bool validutf16 = e->validate_utf16le(utf16_output.get(), utf16words); 57| 18.9k| if (!validutf16) { ------------------ | Branch (57:11): [True: 0, False: 18.9k] ------------------ 58| 0| print_input(source, e); 59| 0| abort(); 60| 0| } 61| | // convert it back: 62| | // We need a buffer of size where to write the UTF-8 words. 63| 18.9k| size_t expected_utf8words = 64| 18.9k| e->utf8_length_from_utf16le(utf16_output.get(), utf16words); 65| 18.9k| std::unique_ptr utf8_output{new char[expected_utf8words]}; 66| | // convert to UTF-8 67| 18.9k| size_t utf8words = e->convert_utf16le_to_utf8( 68| 18.9k| utf16_output.get(), utf16words, utf8_output.get()); 69| 18.9k| std::string final_string(utf8_output.get(), utf8words); 70| 18.9k| if (final_string != source) { ------------------ | Branch (70:11): [True: 0, False: 18.9k] ------------------ 71| 0| print_input(source, e); 72| 0| abort(); 73| 0| } 74| 18.9k| } else { 75| | // invalid input!!! 76| | // We need a buffer of size where to write the UTF-16LE words. 77| 1.62k| size_t expected_utf16words = 78| 1.62k| e->utf16_length_from_utf8(source.c_str(), source.size()); 79| 1.62k| std::unique_ptr utf16_output{ 80| 1.62k| new char16_t[expected_utf16words]}; 81| | // convert to UTF-16LE 82| 1.62k| size_t utf16words = e->convert_utf8_to_utf16le( 83| 1.62k| source.c_str(), source.size(), utf16_output.get()); 84| 1.62k| if (utf16words != 0) { ------------------ | Branch (84:11): [True: 0, False: 1.62k] ------------------ 85| 0| print_input(source, e); 86| 0| abort(); 87| 0| } 88| 1.62k| } 89| | 90| | /** 91| | * Transcoding from UTF-8 to UTF-16BE. 92| | */ 93| 20.5k| if (validutf8) { ------------------ | Branch (93:9): [True: 18.9k, False: 1.62k] ------------------ 94| | // We need a buffer of size where to write the UTF-16BE words. 95| 18.9k| size_t expected_utf16words = 96| 18.9k| e->utf16_length_from_utf8(source.c_str(), source.size()); 97| 18.9k| std::unique_ptr utf16_output{ 98| 18.9k| new char16_t[expected_utf16words]}; 99| | // convert to UTF-16BE 100| 18.9k| size_t utf16words = e->convert_utf8_to_utf16be( 101| 18.9k| source.c_str(), source.size(), utf16_output.get()); 102| | // It wrote utf16words * sizeof(char16_t) bytes. 103| 18.9k| bool validutf16 = e->validate_utf16be(utf16_output.get(), utf16words); 104| 18.9k| if (!validutf16) { ------------------ | Branch (104:11): [True: 0, False: 18.9k] ------------------ 105| 0| print_input(source, e); 106| 0| abort(); 107| 0| } 108| | // convert it back: 109| | // We need a buffer of size where to write the UTF-8 words. 110| 18.9k| size_t expected_utf8words = 111| 18.9k| e->utf8_length_from_utf16be(utf16_output.get(), utf16words); 112| 18.9k| std::unique_ptr utf8_output{new char[expected_utf8words]}; 113| | // convert to UTF-8 114| 18.9k| size_t utf8words = e->convert_utf16be_to_utf8( 115| 18.9k| utf16_output.get(), utf16words, utf8_output.get()); 116| 18.9k| std::string final_string(utf8_output.get(), utf8words); 117| 18.9k| if (final_string != source) { ------------------ | Branch (117:11): [True: 0, False: 18.9k] ------------------ 118| 0| print_input(source, e); 119| 0| abort(); 120| 0| } 121| 18.9k| } else { 122| | // invalid input!!! 123| | // We need a buffer of size where to write the UTF-16BE words. 124| 1.62k| size_t expected_utf16words = 125| 1.62k| e->utf16_length_from_utf8(source.c_str(), source.size()); 126| 1.62k| std::unique_ptr utf16_output{ 127| 1.62k| new char16_t[expected_utf16words]}; 128| | // convert to UTF-16BE 129| 1.62k| size_t utf16words = e->convert_utf8_to_utf16be( 130| 1.62k| source.c_str(), source.size(), utf16_output.get()); 131| 1.62k| if (utf16words != 0) { ------------------ | Branch (131:11): [True: 0, False: 1.62k] ------------------ 132| 0| print_input(source, e); 133| 0| abort(); 134| 0| } 135| 1.62k| } 136| | /** 137| | * Transcoding from UTF-8 to UTF-32. 138| | */ 139| 20.5k| if (validutf8) { ------------------ | Branch (139:9): [True: 18.9k, False: 1.62k] ------------------ 140| | // We need a buffer of size where to write the UTF-32 words. 141| 18.9k| size_t expected_utf32words = 142| 18.9k| e->utf32_length_from_utf8(source.c_str(), source.size()); 143| 18.9k| std::unique_ptr utf32_output{ 144| 18.9k| new char32_t[expected_utf32words]}; 145| | // convert to UTF-32 146| 18.9k| size_t utf32words = e->convert_utf8_to_utf32( 147| 18.9k| source.c_str(), source.size(), utf32_output.get()); 148| | // It wrote utf32words * sizeof(char32_t) bytes. 149| 18.9k| bool validutf32 = e->validate_utf32(utf32_output.get(), utf32words); 150| 18.9k| if (!validutf32) { ------------------ | Branch (150:11): [True: 0, False: 18.9k] ------------------ 151| 0| return -1; 152| 0| } 153| | // convert it back: 154| | // We need a buffer of size where to write the UTF-8 words. 155| 18.9k| size_t expected_utf8words = 156| 18.9k| e->utf8_length_from_utf32(utf32_output.get(), utf32words); 157| 18.9k| std::unique_ptr utf8_output{new char[expected_utf8words]}; 158| | // convert to UTF-8 159| 18.9k| size_t utf8words = e->convert_utf32_to_utf8( 160| 18.9k| utf32_output.get(), utf32words, utf8_output.get()); 161| 18.9k| std::string final_string(utf8_output.get(), utf8words); 162| 18.9k| if (source != final_string) { ------------------ | Branch (162:11): [True: 0, False: 18.9k] ------------------ 163| 0| print_input(source, e); 164| 0| abort(); 165| 0| } 166| 18.9k| } else { 167| | // invalid input!!! 168| 1.62k| size_t expected_utf32words = 169| 1.62k| e->utf32_length_from_utf8(source.c_str(), source.size()); 170| 1.62k| std::unique_ptr utf32_output{ 171| 1.62k| new char32_t[expected_utf32words]}; 172| | // convert to UTF-32 173| 1.62k| size_t utf32words = e->convert_utf8_to_utf32( 174| 1.62k| source.c_str(), source.size(), utf32_output.get()); 175| 1.62k| if (utf32words != 0) { ------------------ | Branch (175:11): [True: 0, False: 1.62k] ------------------ 176| 0| print_input(source, e); 177| 0| abort(); 178| 0| } 179| 1.62k| } 180| | 181| | /** 182| | * Transcoding from UTF-8 to Latin 1 183| | */ 184| 20.5k| if (validutf8) { ------------------ | Branch (184:9): [True: 18.9k, False: 1.62k] ------------------ 185| | // We need a buffer of size where to write the UTF-16LE words. 186| 18.9k| size_t expected_latin1words = 187| 18.9k| e->latin1_length_from_utf8(source.c_str(), source.size()); 188| 18.9k| std::unique_ptr latin1_output{new char[expected_latin1words]}; 189| | // convert to latin1 190| 18.9k| size_t latin1words = e->convert_utf8_to_latin1( 191| 18.9k| source.c_str(), source.size(), latin1_output.get()); 192| 18.9k| if (latin1words != 0) { ------------------ | Branch (192:11): [True: 771, False: 18.1k] ------------------ 193| | // convert it back: 194| | // We need a buffer of size where to write the UTF-8 words. 195| 771| size_t expected_utf8words = 196| 771| e->utf8_length_from_latin1(latin1_output.get(), latin1words); 197| 771| std::unique_ptr utf8_output{new char[expected_utf8words]}; 198| | // convert to UTF-8 199| 771| size_t utf8words = e->convert_latin1_to_utf8( 200| 771| latin1_output.get(), latin1words, utf8_output.get()); 201| 771| std::string final_string(utf8_output.get(), utf8words); 202| 771| if (final_string != source) { ------------------ | Branch (202:13): [True: 0, False: 771] ------------------ 203| 0| print_input(source, e); 204| 0| abort(); 205| 0| } 206| 771| } 207| 18.9k| } else { 208| | // invalid input!!! 209| | // We need a buffer of size where to write the Latin 1 words. 210| 1.62k| size_t expected_latin1words = 211| 1.62k| e->latin1_length_from_utf8(source.c_str(), source.size()); 212| 1.62k| std::unique_ptr latin1_output{new char[expected_latin1words]}; 213| | // convert to Latin 1 214| 1.62k| size_t latin1words = e->convert_utf8_to_latin1( 215| 1.62k| source.c_str(), source.size(), latin1_output.get()); 216| 1.62k| if (latin1words != 0) { ------------------ | Branch (216:11): [True: 0, False: 1.62k] ------------------ 217| 0| print_input(source, e); 218| 0| abort(); 219| 0| } 220| 1.62k| } 221| | /** 222| | * Transcoding from UTF-16LE to UTF-8. 223| | */ 224| 20.5k| { 225| | // Get new source data here as this will allow the fuzzer to optimize it's 226| | // input for UTF16-LE. 227| 20.5k| source = fdp.ConsumeRandomLengthString(kMaxStringSize); 228| | // We copy to avoid alignment issues. 229| 20.5k| std::unique_ptr utf16_source{new char16_t[source.size() / 2]}; 230| 20.5k| if (source.data() != nullptr) { ------------------ | Branch (230:11): [True: 20.5k, False: 0] ------------------ 231| 20.5k| std::memcpy(utf16_source.get(), source.data(), source.size() / 2 * 2); 232| 20.5k| } 233| 20.5k| bool validutf16le = 234| 20.5k| e->validate_utf16le(utf16_source.get(), source.size() / 2); 235| 20.5k| auto rutf16le = e->validate_utf16le_with_errors(utf16_source.get(), 236| 20.5k| source.size() / 2); 237| 20.5k| if (validutf16le != ------------------ | Branch (237:11): [True: 0, False: 20.5k] ------------------ 238| 20.5k| (rutf16le.error == simdutf::SUCCESS)) { // they should agree 239| 0| print_input(source, e); 240| 0| abort(); 241| 0| } 242| 20.5k| if (validutf16le) { ------------------ | Branch (242:11): [True: 20.3k, False: 253] ------------------ 243| | // We need a buffer of size where to write the UTF-16 words. 244| 20.3k| size_t expected_utf8words = 245| 20.3k| e->utf8_length_from_utf16le(utf16_source.get(), source.size() / 2); 246| 20.3k| std::unique_ptr utf8_output{new char[expected_utf8words]}; 247| 20.3k| size_t utf8words = e->convert_utf16le_to_utf8( 248| 20.3k| utf16_source.get(), source.size() / 2, utf8_output.get()); 249| | // It wrote utf16words * sizeof(char16_t) bytes. 250| 20.3k| bool validutf8 = e->validate_utf8(utf8_output.get(), utf8words); 251| 20.3k| if (!validutf8) { ------------------ | Branch (251:13): [True: 0, False: 20.3k] ------------------ 252| 0| print_input(source, e); 253| 0| abort(); 254| 0| } 255| | // convert it back: 256| | // We need a buffer of size where to write the UTF-16 words. 257| 20.3k| size_t expected_utf16words = 258| 20.3k| e->utf16_length_from_utf8(utf8_output.get(), utf8words); 259| 20.3k| std::unique_ptr utf16_output{ 260| 20.3k| new char16_t[expected_utf16words]}; 261| | // convert to UTF-8 262| 20.3k| size_t utf16words = e->convert_utf8_to_utf16le( 263| 20.3k| utf8_output.get(), utf8words, utf16_output.get()); 264| 104k| for (size_t i = 0; i < source.size() / 2; i++) { ------------------ | Branch (264:28): [True: 83.8k, False: 20.3k] ------------------ 265| 83.8k| if (utf16_output.get()[i] != (utf16_source.get())[i]) { ------------------ | Branch (265:15): [True: 0, False: 83.8k] ------------------ 266| 0| print_input(source, e); 267| 0| abort(); 268| 0| } 269| 83.8k| } 270| 20.3k| } else { 271| | // invalid input!!! 272| | // We need a buffer of size where to write the UTF-16 words. 273| 253| size_t expected_utf8words = 274| 253| e->utf8_length_from_utf16le(utf16_source.get(), source.size() / 2); 275| 253| std::unique_ptr utf8_output{new char[expected_utf8words]}; 276| 253| size_t utf8words = e->convert_utf16le_to_utf8( 277| 253| utf16_source.get(), source.size() / 2, utf8_output.get()); 278| 253| if (utf8words != 0) { ------------------ | Branch (278:13): [True: 0, False: 253] ------------------ 279| 0| print_input(source, e); 280| 0| abort(); 281| 0| } 282| 253| } 283| 20.5k| } 284| | 285| | /** 286| | * Transcoding from UTF-16BE to UTF-8. 287| | */ 288| 20.5k| { 289| | // Get new source data here as this will allow the fuzzer to optimize it's 290| | // input for UTF16-BE. 291| 20.5k| source = fdp.ConsumeRandomLengthString(kMaxStringSize); 292| 20.5k| std::unique_ptr utf16_source{new char16_t[source.size() / 2]}; 293| 20.5k| if (source.data() != nullptr) { ------------------ | Branch (293:11): [True: 20.5k, False: 0] ------------------ 294| 20.5k| std::memcpy(utf16_source.get(), source.data(), source.size() / 2 * 2); 295| 20.5k| } 296| 20.5k| bool validutf16be = 297| 20.5k| e->validate_utf16be(utf16_source.get(), source.size() / 2); 298| 20.5k| auto rutf16be = e->validate_utf16be_with_errors(utf16_source.get(), 299| 20.5k| source.size() / 2); 300| 20.5k| if (validutf16be != ------------------ | Branch (300:11): [True: 0, False: 20.5k] ------------------ 301| 20.5k| (rutf16be.error == simdutf::SUCCESS)) { // they should agree 302| 0| print_input(source, e); 303| 0| abort(); 304| 0| } 305| 20.5k| if (validutf16be) { ------------------ | Branch (305:11): [True: 20.3k, False: 226] ------------------ 306| | // We need a buffer of size where to write the UTF-16 words. 307| 20.3k| size_t expected_utf8words = 308| 20.3k| e->utf8_length_from_utf16be(utf16_source.get(), source.size() / 2); 309| 20.3k| std::unique_ptr utf8_output{new char[expected_utf8words]}; 310| 20.3k| size_t utf8words = e->convert_utf16be_to_utf8( 311| 20.3k| utf16_source.get(), source.size() / 2, utf8_output.get()); 312| | // It wrote utf16words * sizeof(char16_t) bytes. 313| 20.3k| bool validutf8 = e->validate_utf8(utf8_output.get(), utf8words); 314| 20.3k| if (!validutf8) { ------------------ | Branch (314:13): [True: 0, False: 20.3k] ------------------ 315| 0| print_input(source, e); 316| 0| abort(); 317| 0| } 318| | // convert it back: 319| | // We need a buffer of size where to write the UTF-16 words. 320| 20.3k| size_t expected_utf16words = 321| 20.3k| e->utf16_length_from_utf8(utf8_output.get(), utf8words); 322| 20.3k| std::unique_ptr utf16_output{ 323| 20.3k| new char16_t[expected_utf16words]}; 324| | // convert to UTF-8 325| 20.3k| size_t utf16words = e->convert_utf8_to_utf16be( 326| 20.3k| utf8_output.get(), utf8words, utf16_output.get()); 327| 100k| for (size_t i = 0; i < source.size() / 2; i++) { ------------------ | Branch (327:28): [True: 79.9k, False: 20.3k] ------------------ 328| 79.9k| if (utf16_output.get()[i] != (utf16_source.get())[i]) { ------------------ | Branch (328:15): [True: 0, False: 79.9k] ------------------ 329| 0| print_input(source, e); 330| 0| abort(); 331| 0| } 332| 79.9k| } 333| 20.3k| } else { 334| | // invalid input!!! 335| | // We need a buffer of size where to write the UTF-16 words. 336| 226| size_t expected_utf8words = 337| 226| e->utf8_length_from_utf16be(utf16_source.get(), source.size() / 2); 338| 226| std::unique_ptr utf8_output{new char[expected_utf8words]}; 339| 226| size_t utf8words = e->convert_utf16be_to_utf8( 340| 226| utf16_source.get(), source.size() / 2, utf8_output.get()); 341| 226| if (utf8words != 0) { ------------------ | Branch (341:13): [True: 0, False: 226] ------------------ 342| 0| print_input(source, e); 343| 0| abort(); 344| 0| } 345| 226| } 346| 20.5k| } 347| | 348| | /** 349| | * Transcoding from latin1 to UTF-8. 350| | */ 351| | // Get new source data here as this will allow the fuzzer to optimize it's 352| | // input for latin1. 353| 20.5k| source = fdp.ConsumeRandomLengthString(kMaxStringSize); 354| 20.5k| bool validlatin1 = true; // has to be 355| 20.5k| if (validlatin1) { ------------------ | Branch (355:9): [True: 20.5k, False: 0] ------------------ 356| | // We need a buffer of size where to write the UTF-8 words. 357| 20.5k| size_t expected_utf8words = 358| 20.5k| e->utf8_length_from_latin1(source.c_str(), source.size()); 359| 20.5k| std::unique_ptr utf8_output{new char[expected_utf8words]}; 360| 20.5k| size_t utf8words = e->convert_latin1_to_utf8( 361| 20.5k| source.c_str(), source.size(), utf8_output.get()); 362| | // It wrote utf8words * sizeof(char) bytes. 363| 20.5k| bool validutf8 = e->validate_utf8(utf8_output.get(), utf8words); 364| 20.5k| if (!validutf8) { ------------------ | Branch (364:11): [True: 0, False: 20.5k] ------------------ 365| 0| print_input(source, e); 366| 0| abort(); 367| 0| } 368| | // convert it back: 369| | // We need a buffer of size where to write the latin1 words. 370| 20.5k| size_t expected_latin1words = 371| 20.5k| e->latin1_length_from_utf8(utf8_output.get(), utf8words); 372| 20.5k| std::unique_ptr latin1_output{new char[expected_latin1words]}; 373| | // convert to latin1 374| 20.5k| size_t latin1words = e->convert_utf8_to_latin1( 375| 20.5k| utf8_output.get(), utf8words, latin1_output.get()); 376| 747k| for (size_t i = 0; i < source.size(); i++) { ------------------ | Branch (376:26): [True: 726k, False: 20.5k] ------------------ 377| 726k| if (latin1_output.get()[i] != (source.c_str())[i]) { ------------------ | Branch (377:13): [True: 0, False: 726k] ------------------ 378| 0| print_input(source, e); 379| 0| abort(); 380| 0| } 381| 726k| } 382| 20.5k| } 383| 20.5k| if (validlatin1) { ------------------ | Branch (383:9): [True: 20.5k, False: 0] ------------------ 384| | // We need a buffer of size where to write the UTF-16 words. 385| 20.5k| size_t expected_utf16words = e->utf16_length_from_latin1(source.size()); 386| 20.5k| std::unique_ptr utf16_output{ 387| 20.5k| new char16_t[expected_utf16words]}; 388| 20.5k| size_t utf16words = e->convert_latin1_to_utf16le( 389| 20.5k| source.c_str(), source.size(), utf16_output.get()); 390| | // It wrote utf16words * sizeof(char16_t) bytes. 391| 20.5k| bool validutf16 = e->validate_utf16le(utf16_output.get(), utf16words); 392| 20.5k| if (!validutf16) { ------------------ | Branch (392:11): [True: 0, False: 20.5k] ------------------ 393| 0| print_input(source, e); 394| 0| abort(); 395| 0| } 396| | // convert it back: 397| | // We need a buffer of size where to write the latin1 words. 398| 20.5k| size_t expected_latin1words = e->latin1_length_from_utf16(utf16words); 399| 20.5k| std::unique_ptr latin1_output{new char[expected_latin1words]}; 400| | // convert to latin1 401| 20.5k| size_t latin1words = e->convert_utf16le_to_latin1( 402| 20.5k| utf16_output.get(), utf16words, latin1_output.get()); 403| 747k| for (size_t i = 0; i < source.size(); i++) { ------------------ | Branch (403:26): [True: 726k, False: 20.5k] ------------------ 404| 726k| if (latin1_output.get()[i] != (source.c_str())[i]) { ------------------ | Branch (404:13): [True: 0, False: 726k] ------------------ 405| 0| print_input(source, e); 406| 0| abort(); 407| 0| } 408| 726k| } 409| 20.5k| } 410| 20.5k| if (validlatin1) { ------------------ | Branch (410:9): [True: 20.5k, False: 0] ------------------ 411| | // We need a buffer of size where to write the UTF-16 words. 412| 20.5k| size_t expected_utf16words = e->utf16_length_from_latin1(source.size()); 413| 20.5k| std::unique_ptr utf16_output{ 414| 20.5k| new char16_t[expected_utf16words]}; 415| 20.5k| size_t utf16words = e->convert_latin1_to_utf16be( 416| 20.5k| source.c_str(), source.size(), utf16_output.get()); 417| | // It wrote utf16words * sizeof(char16_t) bytes. 418| 20.5k| bool validutf16 = e->validate_utf16be(utf16_output.get(), utf16words); 419| 20.5k| if (!validutf16) { ------------------ | Branch (419:11): [True: 0, False: 20.5k] ------------------ 420| 0| print_input(source, e); 421| 0| abort(); 422| 0| } 423| | // convert it back: 424| | // We need a buffer of size where to write the latin1 words. 425| 20.5k| size_t expected_latin1words = e->latin1_length_from_utf16(utf16words); 426| 20.5k| std::unique_ptr latin1_output{new char[expected_latin1words]}; 427| | // convert to latin1 428| 20.5k| size_t latin1words = e->convert_utf16be_to_latin1( 429| 20.5k| utf16_output.get(), utf16words, latin1_output.get()); 430| 747k| for (size_t i = 0; i < source.size(); i++) { ------------------ | Branch (430:26): [True: 726k, False: 20.5k] ------------------ 431| 726k| if (latin1_output.get()[i] != (source.c_str())[i]) { ------------------ | Branch (431:13): [True: 0, False: 726k] ------------------ 432| 0| print_input(source, e); 433| 0| abort(); 434| 0| } 435| 726k| } 436| 20.5k| } 437| | 438| 20.5k| if (validlatin1) { ------------------ | Branch (438:9): [True: 20.5k, False: 0] ------------------ 439| | // We need a buffer of size where to write the UTF-16 words. 440| 20.5k| size_t expected_utf32words = e->utf32_length_from_latin1(source.size()); 441| 20.5k| std::unique_ptr utf32_output{ 442| 20.5k| new char32_t[expected_utf32words]}; 443| 20.5k| size_t utf32words = e->convert_latin1_to_utf32( 444| 20.5k| source.c_str(), source.size(), utf32_output.get()); 445| | // It wrote utf16words * sizeof(char16_t) bytes. 446| 20.5k| bool validutf32 = e->validate_utf32(utf32_output.get(), utf32words); 447| 20.5k| if (!validutf32) { ------------------ | Branch (447:11): [True: 0, False: 20.5k] ------------------ 448| 0| print_input(source, e); 449| 0| abort(); 450| 0| } 451| | // convert it back: 452| | // We need a buffer of size where to write the latin1 words. 453| 20.5k| size_t expected_latin1words = e->latin1_length_from_utf32(utf32words); 454| 20.5k| std::unique_ptr latin1_output{new char[expected_latin1words]}; 455| | // convert to latin1 456| 20.5k| size_t latin1words = e->convert_utf32_to_latin1( 457| 20.5k| utf32_output.get(), utf32words, latin1_output.get()); 458| 747k| for (size_t i = 0; i < source.size(); i++) { ------------------ | Branch (458:26): [True: 726k, False: 20.5k] ------------------ 459| 726k| if (latin1_output.get()[i] != (source.c_str())[i]) { ------------------ | Branch (459:13): [True: 0, False: 726k] ------------------ 460| 0| print_input(source, e); 461| 0| abort(); 462| 0| } 463| 726k| } 464| 20.5k| } 465| | 466| | /// Base64 tests. We begin by trying to decode the input, even if we 467| | /// expect it to fail. 468| 20.5k| { 469| 20.5k| size_t max_length_needed = 470| 20.5k| e->maximal_binary_length_from_base64(source.data(), source.size()); 471| 20.5k| std::vector back(max_length_needed); 472| 20.5k| simdutf::result r = 473| 20.5k| e->base64_to_binary(source.data(), source.size(), back.data()); 474| 20.5k| if (r.error == simdutf::error_code::SUCCESS) { ------------------ | Branch (474:11): [True: 18.1k, False: 2.38k] ------------------ 475| | // We expect failure but if we succeed, then we should have a roundtrip. 476| 18.1k| back.resize(r.count); 477| 18.1k| std::vector back2(e->base64_length_from_binary(back.size())); 478| 18.1k| size_t base64size = 479| 18.1k| e->binary_to_base64(back.data(), back.size(), back2.data()); 480| 18.1k| back2.resize(base64size); 481| 18.1k| std::vector back3( 482| 18.1k| e->maximal_binary_length_from_base64(back2.data(), back2.size())); 483| 18.1k| simdutf::result r2 = 484| 18.1k| e->base64_to_binary(back2.data(), back2.size(), back3.data()); 485| 18.1k| if (r2.error != simdutf::error_code::SUCCESS) { ------------------ | Branch (485:13): [True: 0, False: 18.1k] ------------------ 486| 0| print_input(source, e); 487| 0| return false; 488| 0| } 489| 18.1k| if (r2.count != back.size()) { ------------------ | Branch (489:13): [True: 0, False: 18.1k] ------------------ 490| 0| print_input(source, e); 491| 0| return false; 492| 0| } 493| 18.1k| if (back3.size() != back.size()) { ------------------ | Branch (493:13): [True: 0, False: 18.1k] ------------------ 494| 0| print_input(source, e); 495| 0| return false; 496| 0| } 497| 18.1k| } 498| 20.5k| } 499| | 500| | // Same as above, but we use the safe decoder version. 501| 20.5k| { 502| 20.5k| size_t max_length_needed = 503| 20.5k| e->maximal_binary_length_from_base64(source.data(), source.size()); 504| 20.5k| std::vector back(max_length_needed); 505| 20.5k| simdutf::result r = simdutf::base64_to_binary_safe( 506| 20.5k| source.data(), source.size(), back.data(), max_length_needed); 507| 20.5k| if (r.error == simdutf::error_code::SUCCESS) { ------------------ | Branch (507:11): [True: 18.1k, False: 2.38k] ------------------ 508| | // We expect failure but if we succeed, then we should have a roundtrip. 509| 18.1k| back.resize(max_length_needed); 510| 18.1k| std::vector back2(e->base64_length_from_binary(back.size())); 511| 18.1k| size_t base64size = 512| 18.1k| e->binary_to_base64(back.data(), back.size(), back2.data()); 513| 18.1k| back2.resize(base64size); 514| 18.1k| size_t max_length_needed2 = 515| 18.1k| e->maximal_binary_length_from_base64(back2.data(), back2.size()); 516| 18.1k| std::vector back3(max_length_needed2); 517| 18.1k| simdutf::result r2 = simdutf::base64_to_binary_safe( 518| 18.1k| back2.data(), back2.size(), back3.data(), max_length_needed2); 519| 18.1k| if (r2.error != simdutf::error_code::SUCCESS) { ------------------ | Branch (519:13): [True: 0, False: 18.1k] ------------------ 520| 0| print_input(source, e); 521| 0| return false; 522| 0| } 523| 18.1k| if (max_length_needed != back.size()) { ------------------ | Branch (523:13): [True: 0, False: 18.1k] ------------------ 524| 0| print_input(source, e); 525| 0| return false; 526| 0| } 527| 18.1k| if (back3.size() != back.size()) { ------------------ | Branch (527:13): [True: 0, False: 18.1k] ------------------ 528| 0| print_input(source, e); 529| 0| return false; 530| 0| } 531| 18.1k| } 532| 20.5k| } 533| | /// Base64 tests. We encode the content as binary in base64 and we decode 534| | /// it, it should always succeed. 535| 20.5k| { 536| 20.5k| source = fdp.ConsumeRandomLengthString(kMaxStringSize); 537| 20.5k| std::vector base64buffer( 538| 20.5k| e->base64_length_from_binary(source.size())); 539| 20.5k| size_t base64size = e->binary_to_base64(source.data(), source.size(), 540| 20.5k| base64buffer.data()); 541| 20.5k| if (base64size != base64buffer.size()) { ------------------ | Branch (541:11): [True: 0, False: 20.5k] ------------------ 542| 0| print_input(source, e); 543| 0| abort(); 544| 0| } 545| 20.5k| std::vector back(e->maximal_binary_length_from_base64( 546| 20.5k| base64buffer.data(), base64buffer.size())); 547| 20.5k| simdutf::result r = e->base64_to_binary(base64buffer.data(), 548| 20.5k| base64buffer.size(), back.data()); 549| 20.5k| if (r.error != simdutf::error_code::SUCCESS) { ------------------ | Branch (549:11): [True: 0, False: 20.5k] ------------------ 550| 0| print_input(source, e); 551| 0| abort(); 552| 0| } 553| 20.5k| if (r.count != source.size()) { ------------------ | Branch (553:11): [True: 0, False: 20.5k] ------------------ 554| 0| print_input(source, e); 555| 0| abort(); 556| 0| } 557| 381k| for (size_t i = 0; i < source.size(); i++) { ------------------ | Branch (557:26): [True: 361k, False: 20.5k] ------------------ 558| 361k| if (back[i] != (source.c_str())[i]) { ------------------ | Branch (558:13): [True: 0, False: 361k] ------------------ 559| 0| print_input(source, e); 560| 0| abort(); 561| 0| } 562| 361k| } 563| 20.5k| size_t max_length = back.size(); 564| 20.5k| r = simdutf::base64_to_binary_safe( 565| 20.5k| base64buffer.data(), base64buffer.size(), back.data(), max_length); 566| 20.5k| if (r.error != simdutf::error_code::SUCCESS) { ------------------ | Branch (566:11): [True: 0, False: 20.5k] ------------------ 567| 0| printf("base64 round trip failed, error code %d\n", r.error); 568| 0| print_input(source, e); 569| 0| return false; 570| 0| } 571| 20.5k| if (max_length != source.size()) { ------------------ | Branch (571:11): [True: 0, False: 20.5k] ------------------ 572| 0| printf("base64 safe round trip failed, not the same size %zu %zu\n", 573| 0| max_length, source.size()); 574| 0| print_input(source, e); 575| 0| return false; 576| 0| } 577| 381k| for (size_t i = 0; i < source.size(); i++) { ------------------ | Branch (577:26): [True: 361k, False: 20.5k] ------------------ 578| 361k| if (back[i] != (source.c_str())[i]) { ------------------ | Branch (578:13): [True: 0, False: 361k] ------------------ 579| 0| printf("base64 round trip failed, same size, different content\n"); 580| 0| print_input(source, e); 581| 0| return false; 582| 0| } 583| 361k| } 584| 20.5k| } 585| | 586| 20.5k| } // for (auto &e : simdutf::get_available_implementations()) { 587| | 588| 6.85k| return 0; 589| 6.85k|} // extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { _ZN7simdutf26base64_to_binary_safe_implIcEENS_6resultEPKT_mPcRmNS_14base64_optionsENS_27last_chunk_handling_optionsEb: 63| 59.2k| bool decode_up_to_bad_char) noexcept { 64| 59.2k| static_assert(std::is_same::value || 65| 59.2k| std::is_same::value, 66| 59.2k| "Only char and char16_t are supported."); 67| 59.2k| size_t remaining_input_length = length; 68| 59.2k| size_t remaining_output_length = outlen; 69| 59.2k| size_t input_position = 0; 70| 59.2k| size_t output_position = 0; 71| | 72| | // We also do a first pass using the fast path to decode as much as possible 73| 59.2k| size_t safe_input = detail::min( 74| 59.2k| remaining_input_length, 75| 59.2k| base64_length_from_binary(remaining_output_length / 3 * 3, options)); 76| 59.2k| bool done_with_partial = (safe_input == remaining_input_length); 77| 59.2k| simdutf::full_result r; 78| | 79| |#if SIMDUTF_CPLUSPLUS23 80| | if consteval { 81| | r = scalar::base64::base64_to_binary_details_impl( 82| | input + input_position, safe_input, output + output_position, options, 83| | done_with_partial 84| | ? last_chunk_handling_options 85| | : simdutf::last_chunk_handling_options::only_full_chunks); 86| | } else 87| |#endif 88| 59.2k| { 89| 59.2k| r = get_active_implementation()->base64_to_binary_details( 90| 59.2k| input + input_position, safe_input, output + output_position, options, 91| 59.2k| done_with_partial ------------------ | Branch (91:9): [True: 56.0k, False: 3.23k] ------------------ 92| 59.2k| ? last_chunk_handling_options 93| 59.2k| : simdutf::last_chunk_handling_options::only_full_chunks); 94| 59.2k| } 95| 59.2k| simdutf_log_assert(r.input_count <= safe_input, 96| 59.2k| "You should not read more than safe_input"); 97| 59.2k| simdutf_log_assert(r.output_count <= remaining_output_length, 98| 59.2k| "You should not write more than remaining_output_length"); 99| | // Technically redundant, but we want to be explicit about it. 100| 59.2k| input_position += r.input_count; 101| 59.2k| output_position += r.output_count; 102| 59.2k| remaining_input_length -= r.input_count; 103| 59.2k| remaining_output_length -= r.output_count; 104| 59.2k| if (r.error != simdutf::error_code::SUCCESS) { ------------------ | Branch (104:7): [True: 2.07k, False: 57.2k] ------------------ 105| | // There is an error. We return. 106| 2.07k| if (decode_up_to_bad_char && ------------------ | Branch (106:9): [True: 0, False: 2.07k] ------------------ 107| 0| r.error == error_code::INVALID_BASE64_CHARACTER) { ------------------ | Branch (107:9): [True: 0, False: 0] ------------------ 108| 0| return slow_base64_to_binary_safe_impl( 109| 0| input, length, output, outlen, options, last_chunk_handling_options); 110| 0| } 111| 2.07k| outlen = output_position; 112| 2.07k| return {r.error, input_position}; 113| 2.07k| } 114| | 115| 57.2k| if (done_with_partial) { ------------------ | Branch (115:7): [True: 54.7k, False: 2.44k] ------------------ 116| | // We are done. We have decoded everything. 117| 54.7k| outlen = output_position; 118| 54.7k| return {simdutf::error_code::SUCCESS, input_position}; 119| 54.7k| } 120| | // We have decoded some data, but we still have some data to decode. 121| | // We need to decode the rest of the input buffer. 122| 2.44k| r = simdutf::scalar::base64::base64_to_binary_details_safe_impl( 123| 2.44k| input + input_position, remaining_input_length, output + output_position, 124| 2.44k| remaining_output_length, options, last_chunk_handling_options); 125| 2.44k| input_position += r.input_count; 126| 2.44k| output_position += r.output_count; 127| 2.44k| remaining_input_length -= r.input_count; 128| 2.44k| remaining_output_length -= r.output_count; 129| | 130| 2.44k| if (r.error != simdutf::error_code::SUCCESS) { ------------------ | Branch (130:7): [True: 314, False: 2.13k] ------------------ 131| | // There is an error. We return. 132| 314| if (decode_up_to_bad_char && ------------------ | Branch (132:9): [True: 0, False: 314] ------------------ 133| 0| r.error == error_code::INVALID_BASE64_CHARACTER) { ------------------ | Branch (133:9): [True: 0, False: 0] ------------------ 134| 0| return slow_base64_to_binary_safe_impl( 135| 0| input, length, output, outlen, options, last_chunk_handling_options); 136| 0| } 137| 314| outlen = output_position; 138| 314| return {r.error, input_position}; 139| 314| } 140| 2.13k| if (input_position < length) { ------------------ | Branch (140:7): [True: 0, False: 2.13k] ------------------ 141| | // We cannot process the entire input in one go, so we need to 142| | // process it in two steps: first the fast path, then the slow path. 143| | // In some cases, the processing might 'eat up' trailing ignorable 144| | // characters in the fast path, but that can be a problem. 145| | // suppose we have just white space followed by a single base64 character. 146| | // If we first process the white space with the fast path, it will 147| | // eat all of it. But, by the JavaScript standard, we should consume 148| | // no character. See 149| | // https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 150| 0| while (input_position > 0 && ------------------ | Branch (150:12): [True: 0, False: 0] ------------------ 151| 0| base64_ignorable(input[input_position - 1], options)) { ------------------ | Branch (151:12): [True: 0, False: 0] ------------------ 152| 0| input_position--; 153| 0| } 154| 0| } 155| 2.13k| outlen = output_position; 156| 2.13k| return {simdutf::error_code::SUCCESS, input_position}; 157| 2.44k|} _ZN7simdutf6resultC2ENS_10error_codeEm: 87| 193k| : error{err}, count{pos} {} _ZNK7simdutf6result6is_errEv: 93| 10.4k| simdutf_really_inline simdutf_constexpr23 bool is_err() const noexcept { 94| 10.4k| return error != error_code::SUCCESS; 95| 10.4k| } _ZN7simdutf11full_resultC2Ev: 106| 59.2k| : error{error_code::SUCCESS}, input_count{0}, output_count{0} {} _ZN7simdutf11full_resultC2ENS_10error_codeEmm: 111| 120k| : error{err}, input_count{pos_in}, output_count{pos_out} {} _ZN7simdutf11full_resultC2ENS_10error_codeEmmb: 114| 126| : error{err}, input_count{pos_in}, output_count{pos_out}, 115| 126| padding_error{padding_err} {} _ZNK7simdutf11full_resultcvNS_6resultEEv: 117| 59.2k| simdutf_really_inline simdutf_constexpr23 operator result() const noexcept { 118| 59.2k| if (error == error_code::SUCCESS) { ------------------ | Branch (118:9): [True: 56.9k, False: 2.38k] ------------------ 119| 56.9k| return result{error, output_count}; 120| 56.9k| } else { 121| 2.38k| return result{error, input_count}; 122| 2.38k| } 123| 59.2k| } _ZN7simdutf10is_partialENS_27last_chunk_handling_optionsE: 4588| 20.7k|is_partial(last_chunk_handling_options options) { 4589| 20.7k| return (options == stop_before_partial) || (options == only_full_chunks); ------------------ | Branch (4589:10): [True: 0, False: 20.7k] | Branch (4589:46): [True: 3.99k, False: 16.7k] ------------------ 4590| 20.7k|} _ZNK7simdutf14implementation25required_instruction_setsEv: 5613| 27.4k| virtual uint32_t required_instruction_sets() const { 5614| 27.4k| return _required_instruction_sets; 5615| 27.4k| } _ZNK7simdutf14implementation24utf16_length_from_latin1Em: 6818| 41.1k| utf16_length_from_latin1(size_t length) const noexcept { 6819| 41.1k| return length; 6820| 41.1k| } _ZNK7simdutf14implementation24latin1_length_from_utf32Em: 7006| 20.5k| latin1_length_from_utf32(size_t length) const noexcept { 7007| 20.5k| return length; 7008| 20.5k| } _ZNK7simdutf14implementation24latin1_length_from_utf16Em: 7043| 41.1k| latin1_length_from_utf16(size_t length) const noexcept { 7044| 41.1k| return length; 7045| 41.1k| } _ZNK7simdutf14implementation24utf32_length_from_latin1Em: 7076| 20.5k| utf32_length_from_latin1(size_t length) const noexcept { 7077| 20.5k| return length; 7078| 20.5k| } _ZN7simdutf14implementationC2EPKcS2_j: 7514| 5| : _name(name), _description(description), 7515| 5| _required_instruction_sets(required_instruction_sets) {} _ZN7simdutf8internal29available_implementation_listC2Ev: 7546| 1| simdutf_really_inline available_implementation_list() {} _ZN7simdutf25base64_length_from_binaryEmNS_14base64_optionsE: 4899| 59.2k| size_t length, base64_options options = base64_default) noexcept { 4900| 59.2k| return scalar::base64::base64_length_from_binary(length, options); 4901| 59.2k|} _ZN7simdutf16base64_ignorableEcNS_14base64_optionsE: 5314| 3.70k|base64_ignorable(char input, base64_options options = base64_default) noexcept { 5315| 3.70k| return scalar::base64::is_ignorable(input, options); 5316| 3.70k|} simdutf.cpp:_ZN7simdutf6detail12_GLOBAL__N_13minEmm: 59| 59.2k|constexpr std::size_t min(std::size_t a, std::size_t b) { 60| 59.2k| return a < b ? a : b; ------------------ | Branch (60:10): [True: 0, False: 59.2k] ------------------ 61| 59.2k|} _ZN7simdutf8internal10atomic_ptrIKNS_14implementationEEaSEPS3_: 7617| 1| atomic_ptr &operator=(T *_ptr) { 7618| 1| ptr = _ptr; 7619| 1| return *this; 7620| 1| } _ZN7simdutf8internal10atomic_ptrIKNS_14implementationEEcvPS3_Ev: 7614| 1| operator T *() { return ptr.load(); } _ZN7simdutf8internal10atomic_ptrIKNS_14implementationEEC2EPS3_: 7594| 1| atomic_ptr(T *_ptr) : ptr{_ptr} {} _ZN7simdutf8internal10atomic_ptrIKNS_14implementationEEptEv: 7616| 59.2k| T *operator->() { return ptr.load(); } simdutf.cpp:_ZN7simdutf8internalL30detect_supported_architecturesEv: 301| 27.4k|static inline uint32_t detect_supported_architectures() { 302| 27.4k| uint32_t eax; 303| 27.4k| uint32_t ebx = 0; 304| 27.4k| uint32_t ecx = 0; 305| 27.4k| uint32_t edx = 0; 306| 27.4k| uint32_t host_isa = 0x0; 307| | 308| | // EBX for EAX=0x1 309| 27.4k| eax = 0x1; 310| 27.4k| cpuid(&eax, &ebx, &ecx, &edx); 311| | 312| 27.4k| if (ecx & cpuid_bit::sse42) { ------------------ | Branch (312:7): [True: 27.4k, False: 0] ------------------ 313| 27.4k| host_isa |= instruction_set::SSE42; 314| 27.4k| } 315| | 316| 27.4k| if (ecx & cpuid_bit::pclmulqdq) { ------------------ | Branch (316:7): [True: 27.4k, False: 0] ------------------ 317| 27.4k| host_isa |= instruction_set::PCLMULQDQ; 318| 27.4k| } 319| | 320| 27.4k| if ((ecx & cpuid_bit::osxsave) != cpuid_bit::osxsave) { ------------------ | Branch (320:7): [True: 0, False: 27.4k] ------------------ 321| 0| return host_isa; 322| 0| } 323| | 324| | // xgetbv for checking if the OS saves registers 325| 27.4k| uint64_t xcr0 = xgetbv(); 326| | 327| 27.4k| if ((xcr0 & cpuid_bit::xcr0_bit::avx256_saved) == 0) { ------------------ | Branch (327:7): [True: 0, False: 27.4k] ------------------ 328| 0| return host_isa; 329| 0| } 330| | // ECX for EAX=0x7 331| 27.4k| eax = 0x7; 332| 27.4k| ecx = 0x0; // Sub-leaf = 0 333| 27.4k| cpuid(&eax, &ebx, &ecx, &edx); 334| 27.4k| if (ebx & cpuid_bit::ebx::avx2) { ------------------ | Branch (334:7): [True: 27.4k, False: 0] ------------------ 335| 27.4k| host_isa |= instruction_set::AVX2; 336| 27.4k| } 337| 27.4k| if (ebx & cpuid_bit::ebx::bmi1) { ------------------ | Branch (337:7): [True: 27.4k, False: 0] ------------------ 338| 27.4k| host_isa |= instruction_set::BMI1; 339| 27.4k| } 340| 27.4k| if (ebx & cpuid_bit::ebx::bmi2) { ------------------ | Branch (340:7): [True: 27.4k, False: 0] ------------------ 341| 27.4k| host_isa |= instruction_set::BMI2; 342| 27.4k| } 343| 27.4k| if (!((xcr0 & cpuid_bit::xcr0_bit::avx512_saved) == ------------------ | Branch (343:7): [True: 27.4k, False: 0] ------------------ 344| 27.4k| cpuid_bit::xcr0_bit::avx512_saved)) { 345| 27.4k| return host_isa; 346| 27.4k| } 347| 0| if (ebx & cpuid_bit::ebx::avx512f) { ------------------ | Branch (347:7): [True: 0, False: 0] ------------------ 348| 0| host_isa |= instruction_set::AVX512F; 349| 0| } 350| 0| if (ebx & cpuid_bit::ebx::avx512bw) { ------------------ | Branch (350:7): [True: 0, False: 0] ------------------ 351| 0| host_isa |= instruction_set::AVX512BW; 352| 0| } 353| 0| if (ebx & cpuid_bit::ebx::avx512cd) { ------------------ | Branch (353:7): [True: 0, False: 0] ------------------ 354| 0| host_isa |= instruction_set::AVX512CD; 355| 0| } 356| 0| if (ebx & cpuid_bit::ebx::avx512dq) { ------------------ | Branch (356:7): [True: 0, False: 0] ------------------ 357| 0| host_isa |= instruction_set::AVX512DQ; 358| 0| } 359| 0| if (ebx & cpuid_bit::ebx::avx512vl) { ------------------ | Branch (359:7): [True: 0, False: 0] ------------------ 360| 0| host_isa |= instruction_set::AVX512VL; 361| 0| } 362| 0| if (ecx & cpuid_bit::ecx::avx512vbmi2) { ------------------ | Branch (362:7): [True: 0, False: 0] ------------------ 363| 0| host_isa |= instruction_set::AVX512VBMI2; 364| 0| } 365| 0| if (ecx & cpuid_bit::ecx::avx512vpopcnt) { ------------------ | Branch (365:7): [True: 0, False: 0] ------------------ 366| 0| host_isa |= instruction_set::AVX512VPOPCNTDQ; 367| 0| } 368| 0| return host_isa; 369| 27.4k|} simdutf.cpp:_ZN7simdutf8internalL5cpuidEPjS1_S1_S1_: 267| 54.8k| uint32_t *edx) { 268| | #if defined(_MSC_VER) 269| | int cpu_info[4]; 270| | __cpuidex(cpu_info, *eax, *ecx); 271| | *eax = cpu_info[0]; 272| | *ebx = cpu_info[1]; 273| | *ecx = cpu_info[2]; 274| | *edx = cpu_info[3]; 275| | #elif (defined(HAVE_GCC_GET_CPUID) && defined(USE_GCC_GET_CPUID)) || \ 276| | defined(__FILC__) 277| | uint32_t level = *eax; 278| | __get_cpuid(level, eax, ebx, ecx, edx); 279| | #else 280| 54.8k| uint32_t a = *eax, b, c = *ecx, d; 281| 54.8k| asm volatile("cpuid\n\t" : "+a"(a), "=b"(b), "+c"(c), "=d"(d)); 282| 54.8k| *eax = a; 283| 54.8k| *ebx = b; 284| 54.8k| *ecx = c; 285| 54.8k| *edx = d; 286| 54.8k| #endif 287| 54.8k|} simdutf.cpp:_ZN7simdutf8internalL6xgetbvEv: 289| 27.4k|static inline uint64_t xgetbv() { 290| | #if defined(_MSC_VER) 291| | return _xgetbv(0); 292| | #elif defined(__FILC__) 293| | return zxgetbv(); 294| | #else 295| 27.4k| uint32_t xcr0_lo, xcr0_hi; 296| 27.4k| asm volatile("xgetbv\n\t" : "=a"(xcr0_lo), "=d"(xcr0_hi) : "c"(0)); 297| 27.4k| return xcr0_lo | ((uint64_t)xcr0_hi << 32); 298| 27.4k| #endif 299| 27.4k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6425base64_length_from_binaryEmNS_14base64_optionsE: 836| 116k|base64_length_from_binary(size_t length, base64_options options) noexcept { 837| | // By default, we use padding if we are not using the URL variant. 838| | // This is check with ((options & base64_url) == 0) which returns true if we 839| | // are not using the URL variant. However, we also allow 'inversion' of the 840| | // convention with the base64_reverse_padding option. If the 841| | // base64_reverse_padding option is set, we use padding if we are using the 842| | // URL variant, and we omit it if we are not using the URL variant. This is 843| | // checked with 844| | // ((options & base64_reverse_padding) == base64_reverse_padding). 845| 116k| bool use_padding = 846| 116k| ((options & base64_url) == 0) ^ 847| 116k| ((options & base64_reverse_padding) == base64_reverse_padding); 848| 116k| if (!use_padding) { ------------------ | Branch (848:7): [True: 0, False: 116k] ------------------ 849| 0| return length / 3 * 4 + ((length % 3) ? (length % 3) + 1 : 0); ------------------ | Branch (849:30): [True: 0, False: 0] ------------------ 850| 0| } 851| 116k| return (length + 2) / 3 * 852| 116k| 4; // We use padding to make the length a multiple of 4. 853| 116k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6418tail_encode_base64EPcPKcmNS_14base64_optionsE: 648| 20.1k| char *dst, const char *src, size_t srclen, base64_options options) { 649| 20.1k| return tail_encode_base64_impl(dst, src, srclen, options); 650| 20.1k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6423tail_encode_base64_implILb0EEEmPcPKcmNS_14base64_optionsEmm: 418| 56.9k| size_t line_length = simdutf::default_line_length, size_t line_offset = 0) { 419| | if constexpr (use_lines) { 420| | // sanitize line_length and starting_line_offset. 421| | // line_length must be greater than 3. 422| | if (line_length < 4) { 423| | line_length = 4; 424| | } 425| | simdutf_log_assert(line_offset <= line_length, 426| | "line_offset should be less than line_length"); 427| | } 428| | // By default, we use padding if we are not using the URL variant. 429| | // This is check with ((options & base64_url) == 0) which returns true if we 430| | // are not using the URL variant. However, we also allow 'inversion' of the 431| | // convention with the base64_reverse_padding option. If the 432| | // base64_reverse_padding option is set, we use padding if we are using the 433| | // URL variant, and we omit it if we are not using the URL variant. This is 434| | // checked with 435| | // ((options & base64_reverse_padding) == base64_reverse_padding). 436| 56.9k| bool use_padding = 437| 56.9k| ((options & base64_url) == 0) ^ 438| 56.9k| ((options & base64_reverse_padding) == base64_reverse_padding); 439| | // This looks like 3 branches, but we expect the compiler to resolve this to 440| | // a single branch: 441| 56.9k| const char *e0 = (options & base64_url) ? tables::base64::base64_url::e0 ------------------ | Branch (441:20): [True: 0, False: 56.9k] ------------------ 442| 56.9k| : tables::base64::base64_default::e0; 443| 56.9k| const char *e1 = (options & base64_url) ? tables::base64::base64_url::e1 ------------------ | Branch (443:20): [True: 0, False: 56.9k] ------------------ 444| 56.9k| : tables::base64::base64_default::e1; 445| 56.9k| const char *e2 = (options & base64_url) ? tables::base64::base64_url::e2 ------------------ | Branch (445:20): [True: 0, False: 56.9k] ------------------ 446| 56.9k| : tables::base64::base64_default::e2; 447| 56.9k| char *out = dst; 448| 56.9k| size_t i = 0; 449| 56.9k| uint8_t t1, t2, t3; 450| 86.4k| for (; i + 2 < srclen; i += 3) { ------------------ | Branch (450:10): [True: 29.5k, False: 56.9k] ------------------ 451| 29.5k| t1 = uint8_t(src[i]); 452| 29.5k| t2 = uint8_t(src[i + 1]); 453| 29.5k| t3 = uint8_t(src[i + 2]); 454| | if constexpr (use_lines) { 455| | if (line_offset + 3 >= line_length) { 456| | if (line_offset == line_length) { 457| | *out++ = '\n'; 458| | *out++ = e0[t1]; 459| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 460| | *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; 461| | *out++ = e2[t3]; 462| | line_offset = 4; 463| | } else if (line_offset + 1 == line_length) { 464| | *out++ = e0[t1]; 465| | *out++ = '\n'; 466| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 467| | *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; 468| | *out++ = e2[t3]; 469| | line_offset = 3; 470| | } else if (line_offset + 2 == line_length) { 471| | *out++ = e0[t1]; 472| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 473| | *out++ = '\n'; 474| | *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; 475| | *out++ = e2[t3]; 476| | line_offset = 2; 477| | } else if (line_offset + 3 == line_length) { 478| | *out++ = e0[t1]; 479| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 480| | *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; 481| | *out++ = '\n'; 482| | *out++ = e2[t3]; 483| | line_offset = 1; 484| | } 485| | } else { 486| | *out++ = e0[t1]; 487| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 488| | *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; 489| | *out++ = e2[t3]; 490| | line_offset += 4; 491| | } 492| 29.5k| } else { 493| 29.5k| *out++ = e0[t1]; 494| 29.5k| *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 495| 29.5k| *out++ = e1[((t2 & 0x0F) << 2) | ((t3 >> 6) & 0x03)]; 496| 29.5k| *out++ = e2[t3]; 497| 29.5k| } 498| 29.5k| } 499| 56.9k| switch (srclen - i) { 500| 54.7k| case 0: ------------------ | Branch (500:3): [True: 54.7k, False: 2.16k] ------------------ 501| 54.7k| break; 502| 1.29k| case 1: ------------------ | Branch (502:3): [True: 1.29k, False: 55.6k] ------------------ 503| 1.29k| t1 = uint8_t(src[i]); 504| | if constexpr (use_lines) { 505| | if (use_padding) { 506| | if (line_offset + 3 >= line_length) { 507| | if (line_offset == line_length) { 508| | *out++ = '\n'; 509| | *out++ = e0[t1]; 510| | *out++ = e1[(t1 & 0x03) << 4]; 511| | *out++ = '='; 512| | *out++ = '='; 513| | } else if (line_offset + 1 == line_length) { 514| | *out++ = e0[t1]; 515| | *out++ = '\n'; 516| | *out++ = e1[(t1 & 0x03) << 4]; 517| | *out++ = '='; 518| | *out++ = '='; 519| | } else if (line_offset + 2 == line_length) { 520| | *out++ = e0[t1]; 521| | *out++ = e1[(t1 & 0x03) << 4]; 522| | *out++ = '\n'; 523| | *out++ = '='; 524| | *out++ = '='; 525| | } else if (line_offset + 3 == line_length) { 526| | *out++ = e0[t1]; 527| | *out++ = e1[(t1 & 0x03) << 4]; 528| | *out++ = '='; 529| | *out++ = '\n'; 530| | *out++ = '='; 531| | } 532| | } else { 533| | *out++ = e0[t1]; 534| | *out++ = e1[(t1 & 0x03) << 4]; 535| | *out++ = '='; 536| | *out++ = '='; 537| | } 538| | } else { 539| | if (line_offset + 2 >= line_length) { 540| | if (line_offset == line_length) { 541| | *out++ = '\n'; 542| | *out++ = e0[uint8_t(src[i])]; 543| | *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; 544| | } else if (line_offset + 1 == line_length) { 545| | *out++ = e0[uint8_t(src[i])]; 546| | *out++ = '\n'; 547| | *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; 548| | } else { 549| | *out++ = e0[uint8_t(src[i])]; 550| | *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; 551| | // *out++ = '\n'; ==> no newline at the end of the output 552| | } 553| | } else { 554| | *out++ = e0[uint8_t(src[i])]; 555| | *out++ = e1[(uint8_t(src[i]) & 0x03) << 4]; 556| | } 557| | } 558| 1.29k| } else { 559| 1.29k| *out++ = e0[t1]; 560| 1.29k| *out++ = e1[(t1 & 0x03) << 4]; 561| 1.29k| if (use_padding) { ------------------ | Branch (561:11): [True: 1.29k, False: 0] ------------------ 562| 1.29k| *out++ = '='; 563| 1.29k| *out++ = '='; 564| 1.29k| } 565| 1.29k| } 566| 1.29k| break; 567| 870| default: /* case 2 */ ------------------ | Branch (567:3): [True: 870, False: 56.0k] ------------------ 568| 870| t1 = uint8_t(src[i]); 569| 870| t2 = uint8_t(src[i + 1]); 570| | if constexpr (use_lines) { 571| | if (use_padding) { 572| | if (line_offset + 3 >= line_length) { 573| | if (line_offset == line_length) { 574| | *out++ = '\n'; 575| | *out++ = e0[t1]; 576| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 577| | *out++ = e2[(t2 & 0x0F) << 2]; 578| | *out++ = '='; 579| | } else if (line_offset + 1 == line_length) { 580| | *out++ = e0[t1]; 581| | *out++ = '\n'; 582| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 583| | *out++ = e2[(t2 & 0x0F) << 2]; 584| | *out++ = '='; 585| | } else if (line_offset + 2 == line_length) { 586| | *out++ = e0[t1]; 587| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 588| | *out++ = '\n'; 589| | *out++ = e2[(t2 & 0x0F) << 2]; 590| | *out++ = '='; 591| | } else if (line_offset + 3 == line_length) { 592| | *out++ = e0[t1]; 593| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 594| | *out++ = e2[(t2 & 0x0F) << 2]; 595| | *out++ = '\n'; 596| | *out++ = '='; 597| | } 598| | } else { 599| | *out++ = e0[t1]; 600| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 601| | *out++ = e2[(t2 & 0x0F) << 2]; 602| | *out++ = '='; 603| | } 604| | } else { 605| | if (line_offset + 3 >= line_length) { 606| | if (line_offset == line_length) { 607| | *out++ = '\n'; 608| | *out++ = e0[t1]; 609| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 610| | *out++ = e2[(t2 & 0x0F) << 2]; 611| | } else if (line_offset + 1 == line_length) { 612| | *out++ = e0[t1]; 613| | *out++ = '\n'; 614| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 615| | *out++ = e2[(t2 & 0x0F) << 2]; 616| | } else if (line_offset + 2 == line_length) { 617| | *out++ = e0[t1]; 618| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 619| | *out++ = '\n'; 620| | *out++ = e2[(t2 & 0x0F) << 2]; 621| | } else { 622| | *out++ = e0[t1]; 623| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 624| | *out++ = e2[(t2 & 0x0F) << 2]; 625| | // *out++ = '\n'; ==> no newline at the end of the output 626| | } 627| | } else { 628| | *out++ = e0[t1]; 629| | *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 630| | *out++ = e2[(t2 & 0x0F) << 2]; 631| | } 632| | } 633| 870| } else { 634| 870| *out++ = e0[t1]; 635| 870| *out++ = e1[((t1 & 0x03) << 4) | ((t2 >> 4) & 0x0F)]; 636| 870| *out++ = e2[(t2 & 0x0F) << 2]; 637| 870| if (use_padding) { ------------------ | Branch (637:11): [True: 870, False: 0] ------------------ 638| 870| *out++ = '='; 639| 870| } 640| 870| } 641| 56.9k| } 642| 56.9k| return (size_t)(out - dst); 643| 56.9k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6413is_eight_byteIcEEbT_: 19| 318k|template simdutf_constexpr23 bool is_eight_byte(char_type c) { 20| 318k| if constexpr (sizeof(char_type) == 1) { 21| 318k| return true; 22| 318k| } 23| 0| return uint8_t(c) == c; 24| 318k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6433maximal_binary_length_from_base64IPKcEEmT_m: 654| 98.0k|maximal_binary_length_from_base64(InputPtr input, size_t length) noexcept { 655| | // We process the padding characters ('=') at the end to make sure 656| | // that we return an exact result when the input has no ignorable characters 657| | // (e.g., spaces). 658| 98.0k| size_t padding = 0; 659| 98.0k| if (length > 0) { ------------------ | Branch (659:7): [True: 9.37k, False: 88.6k] ------------------ 660| 9.37k| if (input[length - 1] == '=') { ------------------ | Branch (660:9): [True: 2.34k, False: 7.03k] ------------------ 661| 2.34k| padding++; 662| 2.34k| if (length > 1 && input[length - 2] == '=') { ------------------ | Branch (662:11): [True: 2.31k, False: 26] | Branch (662:25): [True: 1.35k, False: 966] ------------------ 663| 1.35k| padding++; 664| 1.35k| } 665| 2.34k| } 666| 9.37k| } 667| | // The input is not otherwise processed for ignorable characters or 668| | // validation, so that the function runs in constant time (very fast). In 669| | // practice, base64 inputs without ignorable characters are common and the 670| | // common case are line separated inputs with relatively long lines (e.g., 76 671| | // characters) which leads this function to a slight (1%) overestimation of 672| | // the output size. 673| | // 674| | // Of course, some inputs might contain an arbitrary number of spaces or 675| | // newlines, which would make this function return a very pessimistic output 676| | // size but systems that produce base64 outputs typically do not do that and 677| | // if they do, they do not care much about minimizing memory usage. 678| | // 679| | // In specialized applications, users may know that their input is line 680| | // separated, which can be checked very quickly by by iterating (e.g., over 76 681| | // character chunks, looking for the linefeed characters only). We could 682| | // provide a specialized function for that, but it is not clear that the added 683| | // complexity is worth it for us. 684| | // 685| 98.0k| size_t actual_length = length - padding; 686| 98.0k| if (actual_length % 4 <= 1) { ------------------ | Branch (686:7): [True: 94.5k, False: 3.49k] ------------------ 687| 94.5k| return actual_length / 4 * 3; 688| 94.5k| } 689| | // if we have a valid input, then the remainder must be 2 or 3 adding one or 690| | // two extra bytes. 691| 3.49k| return actual_length / 4 * 3 + (actual_length % 4) - 1; 692| 98.0k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6412is_ignorableIcEEbT_NS_14base64_optionsE: 28| 3.70k| simdutf::base64_options options) { 29| 3.70k| const uint8_t *to_base64 = 30| 3.70k| (options & base64_default_or_url) ------------------ | Branch (30:7): [True: 0, False: 3.70k] ------------------ 31| 3.70k| ? tables::base64::to_base64_default_or_url_value 32| 3.70k| : ((options & base64_url) ? tables::base64::to_base64_url_value ------------------ | Branch (32:14): [True: 0, False: 3.70k] ------------------ 33| 3.70k| : tables::base64::to_base64_value); 34| 3.70k| const bool ignore_garbage = 35| 3.70k| (options == base64_options::base64_url_accept_garbage) || ------------------ | Branch (35:7): [True: 0, False: 3.70k] ------------------ 36| 3.70k| (options == base64_options::base64_default_accept_garbage) || ------------------ | Branch (36:7): [True: 0, False: 3.70k] ------------------ 37| 3.70k| (options == base64_options::base64_default_or_url_accept_garbage); ------------------ | Branch (37:7): [True: 0, False: 3.70k] ------------------ 38| 3.70k| uint8_t code = to_base64[uint8_t(c)]; 39| 3.70k| if (is_eight_byte(c) && code <= 63) { ------------------ | Branch (39:7): [True: 3.70k, False: 0] | Branch (39:27): [True: 450, False: 3.25k] ------------------ 40| 450| return false; 41| 450| } 42| 3.25k| if (is_eight_byte(c) && code == 64) { ------------------ | Branch (42:7): [True: 3.25k, False: 0] | Branch (42:27): [True: 3.25k, False: 0] ------------------ 43| 3.25k| return true; 44| 3.25k| } 45| 0| return ignore_garbage; 46| 3.25k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base648find_endIcEENS2_13reduced_inputEPKT_mNS_14base64_optionsE: 100| 121k| simdutf::base64_options options) { 101| 121k| const uint8_t *to_base64 = 102| 121k| (options & base64_default_or_url) ------------------ | Branch (102:7): [True: 0, False: 121k] ------------------ 103| 121k| ? tables::base64::to_base64_default_or_url_value 104| 121k| : ((options & base64_url) ? tables::base64::to_base64_url_value ------------------ | Branch (104:14): [True: 0, False: 121k] ------------------ 105| 121k| : tables::base64::to_base64_value); 106| 121k| const bool ignore_garbage = 107| 121k| (options == base64_options::base64_url_accept_garbage) || ------------------ | Branch (107:7): [True: 0, False: 121k] ------------------ 108| 121k| (options == base64_options::base64_default_accept_garbage) || ------------------ | Branch (108:7): [True: 0, False: 121k] ------------------ 109| 121k| (options == base64_options::base64_default_or_url_accept_garbage); ------------------ | Branch (109:7): [True: 0, False: 121k] ------------------ 110| | 111| 121k| size_t equalsigns = 0; 112| | // We intentionally include trailing spaces in the full input length. 113| | // See https://github.com/simdutf/simdutf/issues/824 114| 121k| size_t full_input_length = srclen; 115| | // skip trailing spaces 116| 130k| while (!ignore_garbage && srclen > 0 && ------------------ | Branch (116:10): [True: 130k, False: 0] | Branch (116:29): [True: 22.0k, False: 107k] ------------------ 117| 22.0k| scalar::base64::is_eight_byte(src[srclen - 1]) && ------------------ | Branch (117:10): [True: 22.0k, False: 0] ------------------ 118| 22.0k| to_base64[uint8_t(src[srclen - 1])] == 64) { ------------------ | Branch (118:10): [True: 9.03k, False: 13.0k] ------------------ 119| 9.03k| srclen--; 120| 9.03k| } 121| 121k| size_t equallocation = 122| 121k| srclen; // location of the first padding character if any 123| 121k| if (ignore_garbage) { ------------------ | Branch (123:7): [True: 0, False: 121k] ------------------ 124| | // Technically, we don't need to find the first padding character, we can 125| | // just change our algorithms, but it adds substantial complexity. 126| 0| auto it = simdutf::find(src, src + srclen, '='); 127| 0| if (it != src + srclen) { ------------------ | Branch (127:9): [True: 0, False: 0] ------------------ 128| 0| equallocation = it - src; 129| 0| equalsigns = 1; 130| 0| srclen = equallocation; 131| 0| full_input_length = equallocation + 1; 132| 0| } 133| 0| return {equalsigns, equallocation, srclen, full_input_length}; 134| 0| } 135| 121k| if (!ignore_garbage && srclen > 0 && src[srclen - 1] == '=') { ------------------ | Branch (135:7): [True: 121k, False: 0] | Branch (135:26): [True: 13.0k, False: 107k] | Branch (135:40): [True: 3.89k, False: 9.15k] ------------------ 136| | // This is the last '=' sign. 137| 3.89k| equallocation = srclen - 1; 138| 3.89k| srclen--; 139| 3.89k| equalsigns = 1; 140| | // skip trailing spaces 141| 4.45k| while (srclen > 0 && scalar::base64::is_eight_byte(src[srclen - 1]) && ------------------ | Branch (141:12): [True: 4.40k, False: 47] | Branch (141:26): [True: 4.40k, False: 0] ------------------ 142| 4.40k| to_base64[uint8_t(src[srclen - 1])] == 64) { ------------------ | Branch (142:12): [True: 559, False: 3.84k] ------------------ 143| 559| srclen--; 144| 559| } 145| 3.89k| if (srclen > 0 && src[srclen - 1] == '=') { ------------------ | Branch (145:9): [True: 3.84k, False: 47] | Branch (145:23): [True: 2.30k, False: 1.53k] ------------------ 146| | // This is the second '=' sign. 147| 2.30k| equallocation = srclen - 1; 148| 2.30k| srclen--; 149| 2.30k| equalsigns = 2; 150| 2.30k| } 151| 3.89k| } 152| 121k| return {equalsigns, equallocation, srclen, full_input_length}; 153| 121k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6423base64_tail_decode_safeIcEENS_11full_resultEPcmPKT_mmNS_14base64_optionsENS_27last_chunk_handling_optionsE: 385| 2.40k| base64_options options, last_chunk_handling_options last_chunk_options) { 386| 2.40k| return base64_tail_decode_impl(dst, outlen, src, length, 387| 2.40k| padding_characters, options, 388| 2.40k| last_chunk_options); 389| 2.40k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6423base64_tail_decode_implILb1EcEENS_11full_resultEPcmPKT0_mmNS_14base64_optionsENS_27last_chunk_handling_optionsE: 164| 2.40k| base64_options options, last_chunk_handling_options last_chunk_options) { 165| 2.40k| char *dstend = dst + outlen; 166| 2.40k| (void)dstend; 167| | // This looks like 10 branches, but we expect the compiler to resolve this to 168| | // two branches (easily predicted): 169| 2.40k| const uint8_t *to_base64 = 170| 2.40k| (options & base64_default_or_url) ------------------ | Branch (170:7): [True: 0, False: 2.40k] ------------------ 171| 2.40k| ? tables::base64::to_base64_default_or_url_value 172| 2.40k| : ((options & base64_url) ? tables::base64::to_base64_url_value ------------------ | Branch (172:14): [True: 0, False: 2.40k] ------------------ 173| 2.40k| : tables::base64::to_base64_value); 174| 2.40k| const uint32_t *d0 = 175| 2.40k| (options & base64_default_or_url) ------------------ | Branch (175:7): [True: 0, False: 2.40k] ------------------ 176| 2.40k| ? tables::base64::base64_default_or_url::d0 177| 2.40k| : ((options & base64_url) ? tables::base64::base64_url::d0 ------------------ | Branch (177:14): [True: 0, False: 2.40k] ------------------ 178| 2.40k| : tables::base64::base64_default::d0); 179| 2.40k| const uint32_t *d1 = 180| 2.40k| (options & base64_default_or_url) ------------------ | Branch (180:7): [True: 0, False: 2.40k] ------------------ 181| 2.40k| ? tables::base64::base64_default_or_url::d1 182| 2.40k| : ((options & base64_url) ? tables::base64::base64_url::d1 ------------------ | Branch (182:14): [True: 0, False: 2.40k] ------------------ 183| 2.40k| : tables::base64::base64_default::d1); 184| 2.40k| const uint32_t *d2 = 185| 2.40k| (options & base64_default_or_url) ------------------ | Branch (185:7): [True: 0, False: 2.40k] ------------------ 186| 2.40k| ? tables::base64::base64_default_or_url::d2 187| 2.40k| : ((options & base64_url) ? tables::base64::base64_url::d2 ------------------ | Branch (187:14): [True: 0, False: 2.40k] ------------------ 188| 2.40k| : tables::base64::base64_default::d2); 189| 2.40k| const uint32_t *d3 = 190| 2.40k| (options & base64_default_or_url) ------------------ | Branch (190:7): [True: 0, False: 2.40k] ------------------ 191| 2.40k| ? tables::base64::base64_default_or_url::d3 192| 2.40k| : ((options & base64_url) ? tables::base64::base64_url::d3 ------------------ | Branch (192:14): [True: 0, False: 2.40k] ------------------ 193| 2.40k| : tables::base64::base64_default::d3); 194| 2.40k| const bool ignore_garbage = 195| 2.40k| (options == base64_options::base64_url_accept_garbage) || ------------------ | Branch (195:7): [True: 0, False: 2.40k] ------------------ 196| 2.40k| (options == base64_options::base64_default_accept_garbage) || ------------------ | Branch (196:7): [True: 0, False: 2.40k] ------------------ 197| 2.40k| (options == base64_options::base64_default_or_url_accept_garbage); ------------------ | Branch (197:7): [True: 0, False: 2.40k] ------------------ 198| | 199| 2.40k| const char_type *srcend = src + length; 200| 2.40k| const char_type *srcinit = src; 201| 2.40k| const char *dstinit = dst; 202| | 203| 2.40k| uint32_t x; 204| 2.40k| size_t idx; 205| 2.40k| uint8_t buffer[4]; 206| 2.45k| while (true) { ------------------ | Branch (206:10): [True: 2.45k, Folded] ------------------ 207| 2.51k| while (srcend - src >= 4 && is_eight_byte(src[0]) && ------------------ | Branch (207:12): [True: 247, False: 2.27k] | Branch (207:33): [True: 247, False: 0] ------------------ 208| 247| is_eight_byte(src[1]) && is_eight_byte(src[2]) && ------------------ | Branch (208:12): [True: 247, False: 0] | Branch (208:37): [True: 247, False: 0] ------------------ 209| 247| is_eight_byte(src[3]) && ------------------ | Branch (209:12): [True: 247, False: 0] ------------------ 210| 247| (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | ------------------ | Branch (210:12): [True: 64, False: 183] ------------------ 211| 247| d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { 212| 64| if (check_capacity && dstend - dst < 3) { ------------------ | Branch (212:11): [True: 64, Folded] | Branch (212:29): [True: 0, False: 64] ------------------ 213| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), 214| 0| size_t(dst - dstinit)}; 215| 0| } 216| 64| *dst++ = static_cast(x & 0xFF); 217| 64| *dst++ = static_cast((x >> 8) & 0xFF); 218| 64| *dst++ = static_cast((x >> 16) & 0xFF); 219| 64| src += 4; 220| 64| } 221| 2.45k| const char_type *srccur = src; 222| 2.45k| idx = 0; 223| | // we need at least four characters. 224| 2.45k|#ifdef __clang__ 225| | // If possible, we read four characters at a time. (It is an optimization.) 226| 2.45k| if (ignore_garbage && src + 4 <= srcend) { ------------------ | Branch (226:9): [True: 0, False: 2.45k] | Branch (226:27): [True: 0, False: 0] ------------------ 227| 0| char_type c0 = src[0]; 228| 0| char_type c1 = src[1]; 229| 0| char_type c2 = src[2]; 230| 0| char_type c3 = src[3]; 231| | 232| 0| uint8_t code0 = to_base64[uint8_t(c0)]; 233| 0| uint8_t code1 = to_base64[uint8_t(c1)]; 234| 0| uint8_t code2 = to_base64[uint8_t(c2)]; 235| 0| uint8_t code3 = to_base64[uint8_t(c3)]; 236| | 237| 0| buffer[idx] = code0; 238| 0| idx += (is_eight_byte(c0) && code0 <= 63); ------------------ | Branch (238:15): [True: 0, False: 0] | Branch (238:36): [True: 0, False: 0] ------------------ 239| 0| buffer[idx] = code1; 240| 0| idx += (is_eight_byte(c1) && code1 <= 63); ------------------ | Branch (240:15): [True: 0, False: 0] | Branch (240:36): [True: 0, False: 0] ------------------ 241| 0| buffer[idx] = code2; 242| 0| idx += (is_eight_byte(c2) && code2 <= 63); ------------------ | Branch (242:15): [True: 0, False: 0] | Branch (242:36): [True: 0, False: 0] ------------------ 243| 0| buffer[idx] = code3; 244| 0| idx += (is_eight_byte(c3) && code3 <= 63); ------------------ | Branch (244:15): [True: 0, False: 0] | Branch (244:36): [True: 0, False: 0] ------------------ 245| 0| src += 4; 246| 0| } 247| 2.45k|#endif 248| 14.7k| while ((idx < 4) && (src < srcend)) { ------------------ | Branch (248:12): [True: 14.6k, False: 54] | Branch (248:25): [True: 12.4k, False: 2.21k] ------------------ 249| 12.4k| char_type c = *src; 250| | 251| 12.4k| uint8_t code = to_base64[uint8_t(c)]; 252| 12.4k| buffer[idx] = uint8_t(code); 253| 12.4k| if (is_eight_byte(c) && code <= 63) { ------------------ | Branch (253:11): [True: 12.4k, False: 0] | Branch (253:31): [True: 5.23k, False: 7.24k] ------------------ 254| 5.23k| idx++; 255| 7.24k| } else if (!ignore_garbage && ------------------ | Branch (255:18): [True: 7.24k, False: 0] ------------------ 256| 7.24k| (code > 64 || !scalar::base64::is_eight_byte(c))) { ------------------ | Branch (256:19): [True: 189, False: 7.05k] | Branch (256:32): [True: 0, False: 7.05k] ------------------ 257| 189| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 258| 189| size_t(dst - dstinit)}; 259| 7.05k| } else { 260| | // We have a space or a newline or garbage. We ignore it. 261| 7.05k| } 262| 12.2k| src++; 263| 12.2k| } 264| 2.26k| if (idx != 4) { ------------------ | Branch (264:9): [True: 2.21k, False: 54] ------------------ 265| 2.21k| simdutf_log_assert(idx < 4, "idx should be less than 4"); 266| | // We never should have that the number of base64 characters + the 267| | // number of padding characters is more than 4. 268| 2.21k| if (!ignore_garbage && (idx + padding_characters > 4)) { ------------------ | Branch (268:11): [True: 2.21k, False: 0] | Branch (268:30): [True: 9, False: 2.20k] ------------------ 269| 9| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 270| 9| size_t(dst - dstinit), true}; 271| 9| } 272| | 273| | // The idea here is that in loose mode, 274| | // if there is padding at all, it must be used 275| | // to form 4-wise chunk. However, in loose mode, 276| | // we do accept no padding at all. 277| 2.20k| if (!ignore_garbage && ------------------ | Branch (277:11): [True: 2.20k, False: 0] ------------------ 278| 2.20k| last_chunk_options == last_chunk_handling_options::loose && ------------------ | Branch (278:11): [True: 2.20k, False: 0] ------------------ 279| 2.20k| (idx >= 2) && padding_characters > 0 && ------------------ | Branch (279:11): [True: 2.02k, False: 182] | Branch (279:25): [True: 1.85k, False: 170] ------------------ 280| 1.85k| ((idx + padding_characters) & 3) != 0) { ------------------ | Branch (280:11): [True: 15, False: 1.83k] ------------------ 281| 15| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 282| 15| size_t(dst - dstinit), true}; 283| 15| } else 284| | 285| | // The idea here is that in strict mode, we do not want to accept 286| | // incomplete base64 chunks. So if the chunk was otherwise valid, we 287| | // return BASE64_INPUT_REMAINDER. 288| 2.18k| if (!ignore_garbage && ------------------ | Branch (288:13): [True: 2.18k, False: 0] ------------------ 289| 2.18k| last_chunk_options == last_chunk_handling_options::strict && ------------------ | Branch (289:13): [True: 0, False: 2.18k] ------------------ 290| 0| (idx >= 2) && ((idx + padding_characters) & 3) != 0) { ------------------ | Branch (290:13): [True: 0, False: 0] | Branch (290:27): [True: 0, False: 0] ------------------ 291| | // The partial chunk was at src - idx 292| 0| return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), 293| 0| size_t(dst - dstinit), true}; 294| 0| } else 295| | // If there is a partial chunk with insufficient padding, with 296| | // stop_before_partial, we need to just ignore it. In "only full" 297| | // mode, skip the minute there are padding characters. 298| 2.18k| if ((last_chunk_options == ------------------ | Branch (298:16): [True: 0, False: 2.18k] ------------------ 299| 2.18k| last_chunk_handling_options::stop_before_partial && 300| 0| (padding_characters + idx < 4) && (idx != 0) && ------------------ | Branch (300:16): [True: 0, False: 0] | Branch (300:50): [True: 0, False: 0] ------------------ 301| 0| (idx >= 2 || padding_characters == 0)) || ------------------ | Branch (301:17): [True: 0, False: 0] | Branch (301:29): [True: 0, False: 0] ------------------ 302| 2.18k| (last_chunk_options == ------------------ | Branch (302:16): [True: 0, False: 2.18k] ------------------ 303| 2.18k| last_chunk_handling_options::only_full_chunks && 304| 0| (idx >= 2 || padding_characters == 0))) { ------------------ | Branch (304:17): [True: 0, False: 0] | Branch (304:29): [True: 0, False: 0] ------------------ 305| | // partial means that we are *not* going to consume the read 306| | // characters. We need to rewind the src pointer. 307| 0| src = srccur; 308| 0| return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; 309| 2.18k| } else { 310| 2.18k| if (idx == 2) { ------------------ | Branch (310:17): [True: 1.22k, False: 959] ------------------ 311| 1.22k| uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + 312| 1.22k| (uint32_t(buffer[1]) << 2 * 6); 313| 1.22k| if (!ignore_garbage && ------------------ | Branch (313:19): [True: 1.22k, False: 0] ------------------ 314| 1.22k| (last_chunk_options == last_chunk_handling_options::strict) && ------------------ | Branch (314:19): [True: 0, False: 1.22k] ------------------ 315| 0| (triple & 0xffff)) { ------------------ | Branch (315:19): [True: 0, False: 0] ------------------ 316| 0| return {BASE64_EXTRA_BITS, size_t(src - srcinit), 317| 0| size_t(dst - dstinit)}; 318| 0| } 319| 1.22k| if (check_capacity && dstend - dst < 1) { ------------------ | Branch (319:19): [True: 1.22k, Folded] | Branch (319:37): [True: 0, False: 1.22k] ------------------ 320| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), 321| 0| size_t(dst - dstinit)}; 322| 0| } 323| 1.22k| *dst++ = static_cast((triple >> 16) & 0xFF); 324| 1.22k| } else if (idx == 3) { ------------------ | Branch (324:24): [True: 777, False: 182] ------------------ 325| 777| uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + 326| 777| (uint32_t(buffer[1]) << 2 * 6) + 327| 777| (uint32_t(buffer[2]) << 1 * 6); 328| 777| if (!ignore_garbage && ------------------ | Branch (328:19): [True: 777, False: 0] ------------------ 329| 777| (last_chunk_options == last_chunk_handling_options::strict) && ------------------ | Branch (329:19): [True: 0, False: 777] ------------------ 330| 0| (triple & 0xff)) { ------------------ | Branch (330:19): [True: 0, False: 0] ------------------ 331| 0| return {BASE64_EXTRA_BITS, size_t(src - srcinit), 332| 0| size_t(dst - dstinit)}; 333| 0| } 334| 777| if (check_capacity && dstend - dst < 2) { ------------------ | Branch (334:19): [True: 777, Folded] | Branch (334:37): [True: 0, False: 777] ------------------ 335| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), 336| 0| size_t(dst - dstinit)}; 337| 0| } 338| 777| *dst++ = static_cast((triple >> 16) & 0xFF); 339| 777| *dst++ = static_cast((triple >> 8) & 0xFF); 340| 777| } else if (!ignore_garbage && idx == 1 && ------------------ | Branch (340:24): [True: 182, False: 0] | Branch (340:43): [True: 76, False: 106] ------------------ 341| 76| (!is_partial(last_chunk_options) || ------------------ | Branch (341:25): [True: 76, False: 0] ------------------ 342| 0| (is_partial(last_chunk_options) && ------------------ | Branch (342:26): [True: 0, False: 0] ------------------ 343| 76| padding_characters > 0))) { ------------------ | Branch (343:26): [True: 0, False: 0] ------------------ 344| 76| return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), 345| 76| size_t(dst - dstinit)}; 346| 106| } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { ------------------ | Branch (346:24): [True: 106, False: 0] | Branch (346:43): [True: 106, False: 0] | Branch (346:55): [True: 8, False: 98] ------------------ 347| 8| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 348| 8| size_t(dst - dstinit), true}; 349| 8| } 350| 2.10k| return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; 351| 2.18k| } 352| 2.20k| } 353| 54| if (check_capacity && dstend - dst < 3) { ------------------ | Branch (353:9): [True: 54, Folded] | Branch (353:27): [True: 0, False: 54] ------------------ 354| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), 355| 0| size_t(dst - dstinit)}; 356| 0| } 357| 54| uint32_t triple = 358| 54| (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + 359| 54| (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); 360| 54| *dst++ = static_cast((triple >> 16) & 0xFF); 361| 54| *dst++ = static_cast((triple >> 8) & 0xFF); 362| 54| *dst++ = static_cast(triple & 0xFF); 363| 54| } 364| 2.40k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6417patch_tail_resultENS_11full_resultEmmmmNS_27last_chunk_handling_optionsE: 394| 9.99k| last_chunk_handling_options last_chunk_options) { 395| 9.99k| r.input_count += previous_input; 396| 9.99k| r.output_count += previous_output; 397| 9.99k| if (r.padding_error) { ------------------ | Branch (397:7): [True: 96, False: 9.89k] ------------------ 398| 96| r.input_count = equallocation; 399| 96| } 400| | 401| 9.99k| if (r.error == error_code::SUCCESS) { ------------------ | Branch (401:7): [True: 7.74k, False: 2.25k] ------------------ 402| 7.74k| if (!is_partial(last_chunk_options)) { ------------------ | Branch (402:9): [True: 6.00k, False: 1.74k] ------------------ 403| | // A success when we are not in stop_before_partial mode. 404| | // means that we have consumed the whole input buffer. 405| 6.00k| r.input_count = full_input_length; 406| 6.00k| } else if (r.output_count % 3 != 0) { ------------------ | Branch (406:16): [True: 0, False: 1.74k] ------------------ 407| 0| r.input_count = full_input_length; 408| 0| } 409| 7.74k| } 410| 9.99k| return r; 411| 9.99k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6434base64_to_binary_details_safe_implIcEENS_11full_resultEPKT_mPcmNS_14base64_optionsENS_27last_chunk_handling_optionsE: 783| 2.44k| last_chunk_handling_options last_chunk_options) noexcept { 784| 2.44k| const bool ignore_garbage = 785| 2.44k| (options == base64_options::base64_url_accept_garbage) || ------------------ | Branch (785:7): [True: 0, False: 2.44k] ------------------ 786| 2.44k| (options == base64_options::base64_default_accept_garbage) || ------------------ | Branch (786:7): [True: 0, False: 2.44k] ------------------ 787| 2.44k| (options == base64_options::base64_default_or_url_accept_garbage); ------------------ | Branch (787:7): [True: 0, False: 2.44k] ------------------ 788| 2.44k| auto ri = simdutf::scalar::base64::find_end(input, length, options); 789| 2.44k| size_t equallocation = ri.equallocation; 790| 2.44k| size_t equalsigns = ri.equalsigns; 791| 2.44k| length = ri.srclen; 792| 2.44k| size_t full_input_length = ri.full_input_length; 793| 2.44k| if (length == 0) { ------------------ | Branch (793:7): [True: 44, False: 2.40k] ------------------ 794| 44| if (!ignore_garbage && equalsigns > 0) { ------------------ | Branch (794:9): [True: 44, False: 0] | Branch (794:28): [True: 17, False: 27] ------------------ 795| 17| return {INVALID_BASE64_CHARACTER, equallocation, 0}; 796| 17| } 797| 27| return {SUCCESS, full_input_length, 0}; 798| 44| } 799| 2.40k| full_result r = scalar::base64::base64_tail_decode_safe( 800| 2.40k| output, outlen, input, length, equalsigns, options, last_chunk_options); 801| 2.40k| r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, 802| 2.40k| full_input_length, last_chunk_options); 803| 2.40k| if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && ------------------ | Branch (803:7): [True: 2.40k, False: 0] | Branch (803:42): [True: 2.10k, False: 297] ------------------ 804| 2.10k| equalsigns > 0 && !ignore_garbage) { ------------------ | Branch (804:7): [True: 1.83k, False: 268] | Branch (804:25): [True: 1.83k, False: 0] ------------------ 805| | // additional checks 806| 1.83k| if ((r.output_count % 3 == 0) || ------------------ | Branch (806:9): [True: 0, False: 1.83k] ------------------ 807| 1.83k| ((r.output_count % 3) + 1 + equalsigns != 4)) { ------------------ | Branch (807:9): [True: 0, False: 1.83k] ------------------ 808| 0| return {INVALID_BASE64_CHARACTER, equallocation, r.output_count}; 809| 0| } 810| 1.83k| } 811| | 812| | // When is_partial(last_chunk_options) is true, we must either end with 813| | // the end of the stream (beyond whitespace) or right after a non-ignorable 814| | // character or at the very beginning of the stream. 815| | // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 816| 2.40k| if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && ------------------ | Branch (816:7): [True: 0, False: 2.40k] | Branch (816:41): [True: 0, False: 0] ------------------ 817| 0| r.input_count < full_input_length) { ------------------ | Branch (817:7): [True: 0, False: 0] ------------------ 818| | // First check if we can extend the input to the end of the stream 819| 0| while (r.input_count < full_input_length && ------------------ | Branch (819:12): [True: 0, False: 0] ------------------ 820| 0| base64_ignorable(*(input + r.input_count), options)) { ------------------ | Branch (820:12): [True: 0, False: 0] ------------------ 821| 0| r.input_count++; 822| 0| } 823| | // If we are still not at the end of the stream, then we must backtrack 824| | // to the last non-ignorable character. 825| 0| if (r.input_count < full_input_length) { ------------------ | Branch (825:9): [True: 0, False: 0] ------------------ 826| 0| while (r.input_count > 0 && ------------------ | Branch (826:14): [True: 0, False: 0] ------------------ 827| 0| base64_ignorable(*(input + r.input_count - 1), options)) { ------------------ | Branch (827:14): [True: 0, False: 0] ------------------ 828| 0| r.input_count--; 829| 0| } 830| 0| } 831| 0| } 832| 2.40k| return r; 833| 2.40k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6429base64_to_binary_details_implIcEENS_11full_resultEPKT_mPcNS_14base64_optionsENS_27last_chunk_handling_optionsE: 727| 20.3k| last_chunk_handling_options last_chunk_options) noexcept { 728| 20.3k| const bool ignore_garbage = 729| 20.3k| (options == base64_options::base64_url_accept_garbage) || ------------------ | Branch (729:7): [True: 0, False: 20.3k] ------------------ 730| 20.3k| (options == base64_options::base64_default_accept_garbage) || ------------------ | Branch (730:7): [True: 0, False: 20.3k] ------------------ 731| 20.3k| (options == base64_options::base64_default_or_url_accept_garbage); ------------------ | Branch (731:7): [True: 0, False: 20.3k] ------------------ 732| 20.3k| auto ri = simdutf::scalar::base64::find_end(input, length, options); 733| 20.3k| size_t equallocation = ri.equallocation; 734| 20.3k| size_t equalsigns = ri.equalsigns; 735| 20.3k| length = ri.srclen; 736| 20.3k| size_t full_input_length = ri.full_input_length; 737| 20.3k| if (length == 0) { ------------------ | Branch (737:7): [True: 20.0k, False: 369] ------------------ 738| 20.0k| if (!ignore_garbage && equalsigns > 0) { ------------------ | Branch (738:9): [True: 20.0k, False: 0] | Branch (738:28): [True: 3, False: 19.9k] ------------------ 739| 3| return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; 740| 3| } 741| 19.9k| return {SUCCESS, full_input_length, 0}; 742| 20.0k| } 743| 369| full_result r = scalar::base64::base64_tail_decode( 744| 369| output, input, length, equalsigns, options, last_chunk_options); 745| 369| r = scalar::base64::patch_tail_result(r, 0, 0, equallocation, 746| 369| full_input_length, last_chunk_options); 747| 369| if (!is_partial(last_chunk_options) && r.error == error_code::SUCCESS && ------------------ | Branch (747:7): [True: 369, False: 0] | Branch (747:42): [True: 179, False: 190] ------------------ 748| 179| equalsigns > 0 && !ignore_garbage) { ------------------ | Branch (748:7): [True: 113, False: 66] | Branch (748:25): [True: 113, False: 0] ------------------ 749| | // additional checks 750| 113| if ((r.output_count % 3 == 0) || ------------------ | Branch (750:9): [True: 0, False: 113] ------------------ 751| 113| ((r.output_count % 3) + 1 + equalsigns != 4)) { ------------------ | Branch (751:9): [True: 0, False: 113] ------------------ 752| 0| return {INVALID_BASE64_CHARACTER, equallocation, r.output_count, true}; 753| 0| } 754| 113| } 755| | // When is_partial(last_chunk_options) is true, we must either end with 756| | // the end of the stream (beyond whitespace) or right after a non-ignorable 757| | // character or at the very beginning of the stream. 758| | // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 759| 369| if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && ------------------ | Branch (759:7): [True: 0, False: 369] | Branch (759:41): [True: 0, False: 0] ------------------ 760| 0| r.input_count < full_input_length) { ------------------ | Branch (760:7): [True: 0, False: 0] ------------------ 761| | // First check if we can extend the input to the end of the stream 762| 0| while (r.input_count < full_input_length && ------------------ | Branch (762:12): [True: 0, False: 0] ------------------ 763| 0| base64_ignorable(*(input + r.input_count), options)) { ------------------ | Branch (763:12): [True: 0, False: 0] ------------------ 764| 0| r.input_count++; 765| 0| } 766| | // If we are still not at the end of the stream, then we must backtrack 767| | // to the last non-ignorable character. 768| 0| if (r.input_count < full_input_length) { ------------------ | Branch (768:9): [True: 0, False: 0] ------------------ 769| 0| while (r.input_count > 0 && ------------------ | Branch (769:14): [True: 0, False: 0] ------------------ 770| 0| base64_ignorable(*(input + r.input_count - 1), options)) { ------------------ | Branch (770:14): [True: 0, False: 0] ------------------ 771| 0| r.input_count--; 772| 0| } 773| 0| } 774| 0| } 775| 369| return r; 776| 369|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6418base64_tail_decodeIcEENS_11full_resultEPcPKT_mmNS_14base64_optionsENS_27last_chunk_handling_optionsE: 371| 7.59k| base64_options options, last_chunk_handling_options last_chunk_options) { 372| 7.59k| return base64_tail_decode_impl(dst, 0, src, length, padding_characters, 373| 7.59k| options, last_chunk_options); 374| 7.59k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16base6423base64_tail_decode_implILb0EcEENS_11full_resultEPcmPKT0_mmNS_14base64_optionsENS_27last_chunk_handling_optionsE: 164| 7.59k| base64_options options, last_chunk_handling_options last_chunk_options) { 165| 7.59k| char *dstend = dst + outlen; 166| 7.59k| (void)dstend; 167| | // This looks like 10 branches, but we expect the compiler to resolve this to 168| | // two branches (easily predicted): 169| 7.59k| const uint8_t *to_base64 = 170| 7.59k| (options & base64_default_or_url) ------------------ | Branch (170:7): [True: 0, False: 7.59k] ------------------ 171| 7.59k| ? tables::base64::to_base64_default_or_url_value 172| 7.59k| : ((options & base64_url) ? tables::base64::to_base64_url_value ------------------ | Branch (172:14): [True: 0, False: 7.59k] ------------------ 173| 7.59k| : tables::base64::to_base64_value); 174| 7.59k| const uint32_t *d0 = 175| 7.59k| (options & base64_default_or_url) ------------------ | Branch (175:7): [True: 0, False: 7.59k] ------------------ 176| 7.59k| ? tables::base64::base64_default_or_url::d0 177| 7.59k| : ((options & base64_url) ? tables::base64::base64_url::d0 ------------------ | Branch (177:14): [True: 0, False: 7.59k] ------------------ 178| 7.59k| : tables::base64::base64_default::d0); 179| 7.59k| const uint32_t *d1 = 180| 7.59k| (options & base64_default_or_url) ------------------ | Branch (180:7): [True: 0, False: 7.59k] ------------------ 181| 7.59k| ? tables::base64::base64_default_or_url::d1 182| 7.59k| : ((options & base64_url) ? tables::base64::base64_url::d1 ------------------ | Branch (182:14): [True: 0, False: 7.59k] ------------------ 183| 7.59k| : tables::base64::base64_default::d1); 184| 7.59k| const uint32_t *d2 = 185| 7.59k| (options & base64_default_or_url) ------------------ | Branch (185:7): [True: 0, False: 7.59k] ------------------ 186| 7.59k| ? tables::base64::base64_default_or_url::d2 187| 7.59k| : ((options & base64_url) ? tables::base64::base64_url::d2 ------------------ | Branch (187:14): [True: 0, False: 7.59k] ------------------ 188| 7.59k| : tables::base64::base64_default::d2); 189| 7.59k| const uint32_t *d3 = 190| 7.59k| (options & base64_default_or_url) ------------------ | Branch (190:7): [True: 0, False: 7.59k] ------------------ 191| 7.59k| ? tables::base64::base64_default_or_url::d3 192| 7.59k| : ((options & base64_url) ? tables::base64::base64_url::d3 ------------------ | Branch (192:14): [True: 0, False: 7.59k] ------------------ 193| 7.59k| : tables::base64::base64_default::d3); 194| 7.59k| const bool ignore_garbage = 195| 7.59k| (options == base64_options::base64_url_accept_garbage) || ------------------ | Branch (195:7): [True: 0, False: 7.59k] ------------------ 196| 7.59k| (options == base64_options::base64_default_accept_garbage) || ------------------ | Branch (196:7): [True: 0, False: 7.59k] ------------------ 197| 7.59k| (options == base64_options::base64_default_or_url_accept_garbage); ------------------ | Branch (197:7): [True: 0, False: 7.59k] ------------------ 198| | 199| 7.59k| const char_type *srcend = src + length; 200| 7.59k| const char_type *srcinit = src; 201| 7.59k| const char *dstinit = dst; 202| | 203| 7.59k| uint32_t x; 204| 7.59k| size_t idx; 205| 7.59k| uint8_t buffer[4]; 206| 8.40k| while (true) { ------------------ | Branch (206:10): [True: 8.40k, Folded] ------------------ 207| 62.4k| while (srcend - src >= 4 && is_eight_byte(src[0]) && ------------------ | Branch (207:12): [True: 56.6k, False: 5.79k] | Branch (207:33): [True: 56.6k, False: 0] ------------------ 208| 56.6k| is_eight_byte(src[1]) && is_eight_byte(src[2]) && ------------------ | Branch (208:12): [True: 56.6k, False: 0] | Branch (208:37): [True: 56.6k, False: 0] ------------------ 209| 56.6k| is_eight_byte(src[3]) && ------------------ | Branch (209:12): [True: 56.6k, False: 0] ------------------ 210| 56.6k| (x = d0[uint8_t(src[0])] | d1[uint8_t(src[1])] | ------------------ | Branch (210:12): [True: 54.0k, False: 2.61k] ------------------ 211| 56.6k| d2[uint8_t(src[2])] | d3[uint8_t(src[3])]) < 0x01FFFFFF) { 212| 54.0k| if (check_capacity && dstend - dst < 3) { ------------------ | Branch (212:11): [Folded, False: 54.0k] | Branch (212:29): [True: 0, False: 0] ------------------ 213| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(src - srcinit), 214| 0| size_t(dst - dstinit)}; 215| 0| } 216| 54.0k| *dst++ = static_cast(x & 0xFF); 217| 54.0k| *dst++ = static_cast((x >> 8) & 0xFF); 218| 54.0k| *dst++ = static_cast((x >> 16) & 0xFF); 219| 54.0k| src += 4; 220| 54.0k| } 221| 8.40k| const char_type *srccur = src; 222| 8.40k| idx = 0; 223| | // we need at least four characters. 224| 8.40k|#ifdef __clang__ 225| | // If possible, we read four characters at a time. (It is an optimization.) 226| 8.40k| if (ignore_garbage && src + 4 <= srcend) { ------------------ | Branch (226:9): [True: 0, False: 8.40k] | Branch (226:27): [True: 0, False: 0] ------------------ 227| 0| char_type c0 = src[0]; 228| 0| char_type c1 = src[1]; 229| 0| char_type c2 = src[2]; 230| 0| char_type c3 = src[3]; 231| | 232| 0| uint8_t code0 = to_base64[uint8_t(c0)]; 233| 0| uint8_t code1 = to_base64[uint8_t(c1)]; 234| 0| uint8_t code2 = to_base64[uint8_t(c2)]; 235| 0| uint8_t code3 = to_base64[uint8_t(c3)]; 236| | 237| 0| buffer[idx] = code0; 238| 0| idx += (is_eight_byte(c0) && code0 <= 63); ------------------ | Branch (238:15): [True: 0, False: 0] | Branch (238:36): [True: 0, False: 0] ------------------ 239| 0| buffer[idx] = code1; 240| 0| idx += (is_eight_byte(c1) && code1 <= 63); ------------------ | Branch (240:15): [True: 0, False: 0] | Branch (240:36): [True: 0, False: 0] ------------------ 241| 0| buffer[idx] = code2; 242| 0| idx += (is_eight_byte(c2) && code2 <= 63); ------------------ | Branch (242:15): [True: 0, False: 0] | Branch (242:36): [True: 0, False: 0] ------------------ 243| 0| buffer[idx] = code3; 244| 0| idx += (is_eight_byte(c3) && code3 <= 63); ------------------ | Branch (244:15): [True: 0, False: 0] | Branch (244:36): [True: 0, False: 0] ------------------ 245| 0| src += 4; 246| 0| } 247| 8.40k|#endif 248| 30.4k| while ((idx < 4) && (src < srcend)) { ------------------ | Branch (248:12): [True: 29.6k, False: 810] | Branch (248:25): [True: 23.7k, False: 5.90k] ------------------ 249| 23.7k| char_type c = *src; 250| | 251| 23.7k| uint8_t code = to_base64[uint8_t(c)]; 252| 23.7k| buffer[idx] = uint8_t(code); 253| 23.7k| if (is_eight_byte(c) && code <= 63) { ------------------ | Branch (253:11): [True: 23.7k, False: 0] | Branch (253:31): [True: 10.3k, False: 13.3k] ------------------ 254| 10.3k| idx++; 255| 13.3k| } else if (!ignore_garbage && ------------------ | Branch (255:18): [True: 13.3k, False: 0] ------------------ 256| 13.3k| (code > 64 || !scalar::base64::is_eight_byte(c))) { ------------------ | Branch (256:19): [True: 1.68k, False: 11.7k] | Branch (256:32): [True: 0, False: 11.7k] ------------------ 257| 1.68k| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 258| 1.68k| size_t(dst - dstinit)}; 259| 11.7k| } else { 260| | // We have a space or a newline or garbage. We ignore it. 261| 11.7k| } 262| 22.0k| src++; 263| 22.0k| } 264| 6.71k| if (idx != 4) { ------------------ | Branch (264:9): [True: 5.90k, False: 810] ------------------ 265| 5.90k| simdutf_log_assert(idx < 4, "idx should be less than 4"); 266| | // We never should have that the number of base64 characters + the 267| | // number of padding characters is more than 4. 268| 5.90k| if (!ignore_garbage && (idx + padding_characters > 4)) { ------------------ | Branch (268:11): [True: 5.90k, False: 0] | Branch (268:30): [True: 17, False: 5.89k] ------------------ 269| 17| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 270| 17| size_t(dst - dstinit), true}; 271| 17| } 272| | 273| | // The idea here is that in loose mode, 274| | // if there is padding at all, it must be used 275| | // to form 4-wise chunk. However, in loose mode, 276| | // we do accept no padding at all. 277| 5.89k| if (!ignore_garbage && ------------------ | Branch (277:11): [True: 5.89k, False: 0] ------------------ 278| 5.89k| last_chunk_options == last_chunk_handling_options::loose && ------------------ | Branch (278:11): [True: 4.14k, False: 1.74k] ------------------ 279| 4.14k| (idx >= 2) && padding_characters > 0 && ------------------ | Branch (279:11): [True: 2.35k, False: 1.79k] | Branch (279:25): [True: 1.86k, False: 490] ------------------ 280| 1.86k| ((idx + padding_characters) & 3) != 0) { ------------------ | Branch (280:11): [True: 23, False: 1.84k] ------------------ 281| 23| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 282| 23| size_t(dst - dstinit), true}; 283| 23| } else 284| | 285| | // The idea here is that in strict mode, we do not want to accept 286| | // incomplete base64 chunks. So if the chunk was otherwise valid, we 287| | // return BASE64_INPUT_REMAINDER. 288| 5.86k| if (!ignore_garbage && ------------------ | Branch (288:13): [True: 5.86k, False: 0] ------------------ 289| 5.86k| last_chunk_options == last_chunk_handling_options::strict && ------------------ | Branch (289:13): [True: 0, False: 5.86k] ------------------ 290| 0| (idx >= 2) && ((idx + padding_characters) & 3) != 0) { ------------------ | Branch (290:13): [True: 0, False: 0] | Branch (290:27): [True: 0, False: 0] ------------------ 291| | // The partial chunk was at src - idx 292| 0| return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), 293| 0| size_t(dst - dstinit), true}; 294| 0| } else 295| | // If there is a partial chunk with insufficient padding, with 296| | // stop_before_partial, we need to just ignore it. In "only full" 297| | // mode, skip the minute there are padding characters. 298| 5.86k| if ((last_chunk_options == ------------------ | Branch (298:16): [True: 0, False: 5.86k] ------------------ 299| 5.86k| last_chunk_handling_options::stop_before_partial && 300| 0| (padding_characters + idx < 4) && (idx != 0) && ------------------ | Branch (300:16): [True: 0, False: 0] | Branch (300:50): [True: 0, False: 0] ------------------ 301| 0| (idx >= 2 || padding_characters == 0)) || ------------------ | Branch (301:17): [True: 0, False: 0] | Branch (301:29): [True: 0, False: 0] ------------------ 302| 5.86k| (last_chunk_options == ------------------ | Branch (302:16): [True: 1.74k, False: 4.12k] ------------------ 303| 5.86k| last_chunk_handling_options::only_full_chunks && 304| 1.74k| (idx >= 2 || padding_characters == 0))) { ------------------ | Branch (304:17): [True: 198, False: 1.54k] | Branch (304:29): [True: 1.54k, False: 6] ------------------ 305| | // partial means that we are *not* going to consume the read 306| | // characters. We need to rewind the src pointer. 307| 1.74k| src = srccur; 308| 1.74k| return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; 309| 4.12k| } else { 310| 4.12k| if (idx == 2) { ------------------ | Branch (310:17): [True: 1.36k, False: 2.76k] ------------------ 311| 1.36k| uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + 312| 1.36k| (uint32_t(buffer[1]) << 2 * 6); 313| 1.36k| if (!ignore_garbage && ------------------ | Branch (313:19): [True: 1.36k, False: 0] ------------------ 314| 1.36k| (last_chunk_options == last_chunk_handling_options::strict) && ------------------ | Branch (314:19): [True: 0, False: 1.36k] ------------------ 315| 0| (triple & 0xffff)) { ------------------ | Branch (315:19): [True: 0, False: 0] ------------------ 316| 0| return {BASE64_EXTRA_BITS, size_t(src - srcinit), 317| 0| size_t(dst - dstinit)}; 318| 0| } 319| 1.36k| if (check_capacity && dstend - dst < 1) { ------------------ | Branch (319:19): [Folded, False: 1.36k] | Branch (319:37): [True: 0, False: 0] ------------------ 320| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), 321| 0| size_t(dst - dstinit)}; 322| 0| } 323| 1.36k| *dst++ = static_cast((triple >> 16) & 0xFF); 324| 2.76k| } else if (idx == 3) { ------------------ | Branch (324:24): [True: 963, False: 1.79k] ------------------ 325| 963| uint32_t triple = (uint32_t(buffer[0]) << 3 * 6) + 326| 963| (uint32_t(buffer[1]) << 2 * 6) + 327| 963| (uint32_t(buffer[2]) << 1 * 6); 328| 963| if (!ignore_garbage && ------------------ | Branch (328:19): [True: 963, False: 0] ------------------ 329| 963| (last_chunk_options == last_chunk_handling_options::strict) && ------------------ | Branch (329:19): [True: 0, False: 963] ------------------ 330| 0| (triple & 0xff)) { ------------------ | Branch (330:19): [True: 0, False: 0] ------------------ 331| 0| return {BASE64_EXTRA_BITS, size_t(src - srcinit), 332| 0| size_t(dst - dstinit)}; 333| 0| } 334| 963| if (check_capacity && dstend - dst < 2) { ------------------ | Branch (334:19): [Folded, False: 963] | Branch (334:37): [True: 0, False: 0] ------------------ 335| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), 336| 0| size_t(dst - dstinit)}; 337| 0| } 338| 963| *dst++ = static_cast((triple >> 16) & 0xFF); 339| 963| *dst++ = static_cast((triple >> 8) & 0xFF); 340| 1.79k| } else if (!ignore_garbage && idx == 1 && ------------------ | Branch (340:24): [True: 1.79k, False: 0] | Branch (340:43): [True: 205, False: 1.59k] ------------------ 341| 205| (!is_partial(last_chunk_options) || ------------------ | Branch (341:25): [True: 199, False: 6] ------------------ 342| 6| (is_partial(last_chunk_options) && ------------------ | Branch (342:26): [True: 6, False: 0] ------------------ 343| 205| padding_characters > 0))) { ------------------ | Branch (343:26): [True: 6, False: 0] ------------------ 344| 205| return {BASE64_INPUT_REMAINDER, size_t(src - srcinit), 345| 205| size_t(dst - dstinit)}; 346| 1.59k| } else if (!ignore_garbage && idx == 0 && padding_characters > 0) { ------------------ | Branch (346:24): [True: 1.59k, False: 0] | Branch (346:43): [True: 1.59k, False: 0] | Branch (346:55): [True: 24, False: 1.56k] ------------------ 347| 24| return {INVALID_BASE64_CHARACTER, size_t(src - srcinit), 348| 24| size_t(dst - dstinit), true}; 349| 24| } 350| 3.90k| return {SUCCESS, size_t(src - srcinit), size_t(dst - dstinit)}; 351| 4.12k| } 352| 5.89k| } 353| 810| if (check_capacity && dstend - dst < 3) { ------------------ | Branch (353:9): [Folded, False: 810] | Branch (353:27): [True: 0, False: 0] ------------------ 354| 0| return {OUTPUT_BUFFER_TOO_SMALL, size_t(srccur - srcinit), 355| 0| size_t(dst - dstinit)}; 356| 0| } 357| 810| uint32_t triple = 358| 810| (uint32_t(buffer[0]) << 3 * 6) + (uint32_t(buffer[1]) << 2 * 6) + 359| 810| (uint32_t(buffer[2]) << 1 * 6) + (uint32_t(buffer[3]) << 0 * 6); 360| 810| *dst++ = static_cast((triple >> 16) & 0xFF); 361| 810| *dst++ = static_cast((triple >> 8) & 0xFF); 362| 810| *dst++ = static_cast(triple & 0xFF); 363| 810| } 364| 7.59k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_16latin123utf8_length_from_latin1EPKcm: 10| 14.3k| size_t len) { 11| 14.3k| const uint8_t *c = reinterpret_cast(buf); 12| 14.3k| size_t answer = 0; 13| 32.5k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (13:22): [True: 18.2k, False: 14.3k] ------------------ 14| 18.2k| if ((c[i] >> 7)) { ------------------ | Branch (14:9): [True: 6.43k, False: 11.7k] ------------------ 15| 6.43k| answer++; 16| 6.43k| } 17| 18.2k| } 18| 14.3k| return answer + len; 19| 14.3k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_115latin1_to_utf167convertILNS_10endiannessE0EPKcQsr7simdutf6detailE22indexes_into_byte_likeIT0_EEEmS7_mPDs: 14| 8.42k| char16_t *utf16_output) { 15| 8.42k| size_t pos = 0; 16| 8.42k| char16_t *start{utf16_output}; 17| | 18| 99.9k| while (pos < len) { ------------------ | Branch (18:10): [True: 91.4k, False: 8.42k] ------------------ 19| 91.4k| uint16_t word = 20| 91.4k| uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point 21| 91.4k| *utf16_output++ = 22| 91.4k| char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); ------------------ | Branch (22:18): [True: 91.4k, Folded] ------------------ 23| 91.4k| pos++; 24| 91.4k| } 25| | 26| 8.42k| return utf16_output - start; 27| 8.42k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_115latin1_to_utf167convertILNS_10endiannessE1EPKcQsr7simdutf6detailE22indexes_into_byte_likeIT0_EEEmS7_mPDs: 14| 8.42k| char16_t *utf16_output) { 15| 8.42k| size_t pos = 0; 16| 8.42k| char16_t *start{utf16_output}; 17| | 18| 99.9k| while (pos < len) { ------------------ | Branch (18:10): [True: 91.4k, False: 8.42k] ------------------ 19| 91.4k| uint16_t word = 20| 91.4k| uint8_t(data[pos]); // extend Latin-1 char to 16-bit Unicode code point 21| 91.4k| *utf16_output++ = 22| 91.4k| char16_t(match_system(big_endian) ? word : u16_swap_bytes(word)); ------------------ | Branch (22:18): [Folded, False: 91.4k] ------------------ 23| 91.4k| pos++; 24| 91.4k| } 25| | 26| 8.42k| return utf16_output - start; 27| 8.42k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_115latin1_to_utf327convertIPKcQsr7simdutf6detailE22indexes_into_byte_likeIT_EEEmS6_mPDi: 14| 8.37k| char32_t *utf32_output) { 15| 8.37k| char32_t *start{utf32_output}; 16| 97.6k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (16:22): [True: 89.3k, False: 8.37k] ------------------ 17| 89.3k| *utf32_output++ = uint8_t(data[i]); 18| 89.3k| } 19| 8.37k| return utf32_output - start; 20| 8.37k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_114latin1_to_utf87convertEPKcmPc: 66| 10.0k| char *utf8_output) { 67| 10.0k| return convert(reinterpret_cast(buf), len, 68| 10.0k| utf8_output); 69| 10.0k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_114latin1_to_utf87convertIPKhPcQaasr7simdutf6detailE22indexes_into_byte_likeIT_Esr7simdutf6detailE26index_assignable_from_charIT0_EEEmS7_mS8_: 17| 10.0k| OutputPtr utf8_output) { 18| | // const unsigned char *data = reinterpret_cast(buf); 19| 10.0k| size_t pos = 0; 20| 10.0k| size_t utf8_pos = 0; 21| | 22| 91.3k| while (pos < len) { ------------------ | Branch (22:10): [True: 81.3k, False: 10.0k] ------------------ 23| |#if SIMDUTF_CPLUSPLUS23 24| | if !consteval 25| |#endif 26| 81.3k| { 27| | // try to convert the next block of 16 ASCII bytes 28| 81.3k| if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that ------------------ | Branch (28:11): [True: 54.4k, False: 26.8k] ------------------ 29| | // they are ascii 30| 54.4k| uint64_t v1; 31| 54.4k| ::memcpy(&v1, data + pos, sizeof(uint64_t)); 32| 54.4k| uint64_t v2; 33| 54.4k| ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 34| 54.4k| uint64_t v{v1 | 35| 54.4k| v2}; // We are only interested in these bits: 1000 1000 1000 36| | // 1000, so it makes sense to concatenate everything 37| 54.4k| if ((v & 0x8080808080808080) == ------------------ | Branch (37:13): [True: 4.76k, False: 49.7k] ------------------ 38| 54.4k| 0) { // if NONE of these are set, e.g. all of them are zero, then 39| | // everything is ASCII 40| 4.76k| size_t final_pos = pos + 16; 41| 80.9k| while (pos < final_pos) { ------------------ | Branch (41:18): [True: 76.2k, False: 4.76k] ------------------ 42| 76.2k| utf8_output[utf8_pos++] = char(data[pos]); 43| 76.2k| pos++; 44| 76.2k| } 45| 4.76k| continue; 46| 4.76k| } 47| 54.4k| } // if (pos + 16 <= len) 48| 81.3k| } // !consteval scope 49| | 50| 76.5k| unsigned char byte = data[pos]; 51| 76.5k| if ((byte & 0x80) == 0) { // if ASCII ------------------ | Branch (51:9): [True: 31.1k, False: 45.4k] ------------------ 52| | // will generate one UTF-8 bytes 53| 31.1k| utf8_output[utf8_pos++] = char(byte); 54| 31.1k| pos++; 55| 45.4k| } else { 56| | // will generate two UTF-8 bytes 57| 45.4k| utf8_output[utf8_pos++] = char((byte >> 6) | 0b11000000); 58| 45.4k| utf8_output[utf8_pos++] = char((byte & 0b111111) | 0b10000000); 59| 45.4k| pos++; 60| 45.4k| } 61| 76.5k| } // while 62| 10.0k| return utf8_pos; 63| 10.0k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_114latin1_to_utf823utf8_length_from_latin1IPKcQsr7simdutf6detailE22indexes_into_byte_likeIT_EEEmS6_m: 199| 7.00k|utf8_length_from_latin1(InputPtr input, size_t length) noexcept { 200| 7.00k| size_t answer = length; 201| 7.00k| size_t i = 0; 202| | 203| |#if SIMDUTF_CPLUSPLUS23 204| | if !consteval 205| |#endif 206| 7.00k| { 207| 7.00k| auto pop = [](uint64_t v) { 208| 7.00k| return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * 209| 7.00k| UINT64_C(0x0101010101010101) >> 210| 7.00k| 56); 211| 7.00k| }; 212| 10.5k| for (; i + 32 <= length; i += 32) { ------------------ | Branch (212:12): [True: 3.53k, False: 7.00k] ------------------ 213| 3.53k| uint64_t v; 214| 3.53k| memcpy(&v, input + i, 8); 215| 3.53k| answer += pop(v); 216| 3.53k| memcpy(&v, input + i + 8, sizeof(v)); 217| 3.53k| answer += pop(v); 218| 3.53k| memcpy(&v, input + i + 16, sizeof(v)); 219| 3.53k| answer += pop(v); 220| 3.53k| memcpy(&v, input + i + 24, sizeof(v)); 221| 3.53k| answer += pop(v); 222| 3.53k| } 223| 7.28k| for (; i + 8 <= length; i += 8) { ------------------ | Branch (223:12): [True: 281, False: 7.00k] ------------------ 224| 281| uint64_t v; 225| 281| memcpy(&v, input + i, sizeof(v)); 226| 281| answer += pop(v); 227| 281| } 228| 7.00k| } // !consteval scope 229| 7.84k| for (; i + 1 <= length; i += 1) { ------------------ | Branch (229:10): [True: 836, False: 7.00k] ------------------ 230| 836| answer += static_cast(input[i]) >> 7; 231| 836| } 232| 7.00k| return answer; 233| 7.00k|} simdutf.cpp:_ZZN7simdutf6scalar12_GLOBAL__N_114latin1_to_utf823utf8_length_from_latin1IPKcQsr7simdutf6detailE22indexes_into_byte_likeIT_EEEmS6_mENKUlmE_clEm: 207| 14.4k| auto pop = [](uint64_t v) { 208| 14.4k| return (size_t)(((v >> 7) & UINT64_C(0x0101010101010101)) * 209| | UINT64_C(0x0101010101010101) >> 210| 14.4k| 56); 211| 14.4k| }; _ZN7simdutf6scalar14u16_swap_bytesEt: 8| 876k|u16_swap_bytes(const uint16_t word) { 9| 876k| return uint16_t((word >> 8) | (word << 8)); 10| 876k|} _ZN7simdutf6scalar14u32_swap_bytesEj: 13| 6.50k|u32_swap_bytes(const uint32_t word) { 14| 6.50k| return ((word >> 24) & 0xff) | // move byte 3 to byte 0 15| 6.50k| ((word << 8) & 0xff0000) | // move byte 1 to byte 2 16| 6.50k| ((word >> 8) & 0xff00) | // move byte 2 to byte 1 17| 6.50k| ((word << 24) & 0xff000000); // byte 0 to byte 3 18| 6.50k|} _ZN7simdutf6scalar5utf1614swap_if_neededILNS_10endiannessE0EEEtt: 27| 474k|template constexpr uint16_t swap_if_needed(uint16_t c) { 28| 474k| return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; ------------------ | Branch (28:10): [Folded, False: 474k] ------------------ 29| 474k|} _ZN7simdutf6scalar5utf1614swap_if_neededILNS_10endiannessE1EEEtt: 27| 468k|template constexpr uint16_t swap_if_needed(uint16_t c) { 28| 468k| return !match_system(big_endian) ? scalar::u16_swap_bytes(c) : c; ------------------ | Branch (28:10): [True: 468k, Folded] ------------------ 29| 468k|} _ZN7simdutf6scalar5utf168validateILNS_10endiannessE0EEEbPKDsm: 22| 25.5k|validate(const char16_t *data, size_t len) noexcept { 23| 25.5k| uint64_t pos = 0; 24| 333k| while (pos < len) { ------------------ | Branch (24:10): [True: 307k, False: 25.3k] ------------------ 25| 307k| char16_t word = scalar::utf16::swap_if_needed(data[pos]); 26| 307k| if ((word & 0xF800) == 0xD800) { ------------------ | Branch (26:9): [True: 2.59k, False: 305k] ------------------ 27| 2.59k| if (pos + 1 >= len) { ------------------ | Branch (27:11): [True: 34, False: 2.56k] ------------------ 28| 34| return false; 29| 34| } 30| 2.56k| char16_t diff = char16_t(word - 0xD800); 31| 2.56k| if (diff > 0x3FF) { ------------------ | Branch (31:11): [True: 54, False: 2.51k] ------------------ 32| 54| return false; 33| 54| } 34| 2.51k| char16_t next_word = !match_system(big_endian) ------------------ | Branch (34:28): [Folded, False: 2.51k] ------------------ 35| 2.51k| ? u16_swap_bytes(data[pos + 1]) 36| 2.51k| : data[pos + 1]; 37| 2.51k| char16_t diff2 = char16_t(next_word - 0xDC00); 38| 2.51k| if (diff2 > 0x3FF) { ------------------ | Branch (38:11): [True: 58, False: 2.45k] ------------------ 39| 58| return false; 40| 58| } 41| 2.45k| pos += 2; 42| 305k| } else { 43| 305k| pos++; 44| 305k| } 45| 307k| } 46| 25.3k| return true; 47| 25.5k|} _ZN7simdutf6scalar5utf168validateILNS_10endiannessE1EEEbPKDsm: 22| 25.3k|validate(const char16_t *data, size_t len) noexcept { 23| 25.3k| uint64_t pos = 0; 24| 330k| while (pos < len) { ------------------ | Branch (24:10): [True: 305k, False: 25.2k] ------------------ 25| 305k| char16_t word = scalar::utf16::swap_if_needed(data[pos]); 26| 305k| if ((word & 0xF800) == 0xD800) { ------------------ | Branch (26:9): [True: 3.47k, False: 302k] ------------------ 27| 3.47k| if (pos + 1 >= len) { ------------------ | Branch (27:11): [True: 25, False: 3.44k] ------------------ 28| 25| return false; 29| 25| } 30| 3.44k| char16_t diff = char16_t(word - 0xD800); 31| 3.44k| if (diff > 0x3FF) { ------------------ | Branch (31:11): [True: 37, False: 3.40k] ------------------ 32| 37| return false; 33| 37| } 34| 3.40k| char16_t next_word = !match_system(big_endian) ------------------ | Branch (34:28): [True: 3.40k, Folded] ------------------ 35| 3.40k| ? u16_swap_bytes(data[pos + 1]) 36| 3.40k| : data[pos + 1]; 37| 3.40k| char16_t diff2 = char16_t(next_word - 0xDC00); 38| 3.40k| if (diff2 > 0x3FF) { ------------------ | Branch (38:11): [True: 68, False: 3.34k] ------------------ 39| 68| return false; 40| 68| } 41| 3.34k| pos += 2; 42| 302k| } else { 43| 302k| pos++; 44| 302k| } 45| 305k| } 46| 25.2k| return true; 47| 25.3k|} _ZN7simdutf6scalar5utf1620validate_with_errorsILNS_10endiannessE0EEENS_6resultEPKDsm: 51| 7.68k|validate_with_errors(const char16_t *data, size_t len) noexcept { 52| 7.68k| size_t pos = 0; 53| 52.4k| while (pos < len) { ------------------ | Branch (53:10): [True: 45.0k, False: 7.42k] ------------------ 54| 45.0k| char16_t word = scalar::utf16::swap_if_needed(data[pos]); 55| 45.0k| if ((word & 0xF800) == 0xD800) { ------------------ | Branch (55:9): [True: 965, False: 44.0k] ------------------ 56| 965| if (pos + 1 >= len) { ------------------ | Branch (56:11): [True: 34, False: 931] ------------------ 57| 34| return result(error_code::SURROGATE, pos); 58| 34| } 59| 931| char16_t diff = char16_t(word - 0xD800); 60| 931| if (diff > 0x3FF) { ------------------ | Branch (60:11): [True: 102, False: 829] ------------------ 61| 102| return result(error_code::SURROGATE, pos); 62| 102| } 63| 829| char16_t next_word = !match_system(big_endian) ------------------ | Branch (63:28): [Folded, False: 829] ------------------ 64| 829| ? u16_swap_bytes(data[pos + 1]) 65| 829| : data[pos + 1]; 66| 829| char16_t diff2 = uint16_t(next_word - 0xDC00); 67| 829| if (diff2 > 0x3FF) { ------------------ | Branch (67:11): [True: 117, False: 712] ------------------ 68| 117| return result(error_code::SURROGATE, pos); 69| 117| } 70| 712| pos += 2; 71| 44.0k| } else { 72| 44.0k| pos++; 73| 44.0k| } 74| 45.0k| } 75| 7.42k| return result(error_code::SUCCESS, pos); 76| 7.68k|} _ZN7simdutf6scalar5utf1620validate_with_errorsILNS_10endiannessE1EEENS_6resultEPKDsm: 51| 7.54k|validate_with_errors(const char16_t *data, size_t len) noexcept { 52| 7.54k| size_t pos = 0; 53| 50.0k| while (pos < len) { ------------------ | Branch (53:10): [True: 42.7k, False: 7.32k] ------------------ 54| 42.7k| char16_t word = scalar::utf16::swap_if_needed(data[pos]); 55| 42.7k| if ((word & 0xF800) == 0xD800) { ------------------ | Branch (55:9): [True: 1.80k, False: 40.9k] ------------------ 56| 1.80k| if (pos + 1 >= len) { ------------------ | Branch (56:11): [True: 25, False: 1.77k] ------------------ 57| 25| return result(error_code::SURROGATE, pos); 58| 25| } 59| 1.77k| char16_t diff = char16_t(word - 0xD800); 60| 1.77k| if (diff > 0x3FF) { ------------------ | Branch (60:11): [True: 77, False: 1.70k] ------------------ 61| 77| return result(error_code::SURROGATE, pos); 62| 77| } 63| 1.70k| char16_t next_word = !match_system(big_endian) ------------------ | Branch (63:28): [True: 1.70k, Folded] ------------------ 64| 1.70k| ? u16_swap_bytes(data[pos + 1]) 65| 1.70k| : data[pos + 1]; 66| 1.70k| char16_t diff2 = uint16_t(next_word - 0xDC00); 67| 1.70k| if (diff2 > 0x3FF) { ------------------ | Branch (67:11): [True: 124, False: 1.57k] ------------------ 68| 124| return result(error_code::SURROGATE, pos); 69| 124| } 70| 1.57k| pos += 2; 71| 40.9k| } else { 72| 40.9k| pos++; 73| 40.9k| } 74| 42.7k| } 75| 7.32k| return result(error_code::SUCCESS, pos); 76| 7.54k|} _ZN7simdutf6scalar5utf1622utf8_length_from_utf16ILNS_10endiannessE0EEEmPKDsm: 91| 39.4k| size_t len) { 92| | // We are not BOM aware. 93| 39.4k| size_t counter{0}; 94| 127k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (94:22): [True: 88.4k, False: 39.4k] ------------------ 95| 88.4k| char16_t word = scalar::utf16::swap_if_needed(p[i]); 96| 88.4k| counter++; // ASCII 97| 88.4k| counter += static_cast( 98| 88.4k| word > 99| 88.4k| 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes 100| 88.4k| counter += static_cast((word > 0x7FF && word <= 0xD7FF) || ------------------ | Branch (100:37): [True: 34.4k, False: 54.0k] | Branch (100:53): [True: 28.1k, False: 6.33k] ------------------ 101| 60.3k| (word >= 0xE000)); // three-byte ------------------ | Branch (101:36): [True: 3.97k, False: 56.3k] ------------------ 102| 88.4k| } 103| 39.4k| return counter; 104| 39.4k|} _ZN7simdutf6scalar5utf1622utf8_length_from_utf16ILNS_10endiannessE1EEEmPKDsm: 91| 39.4k| size_t len) { 92| | // We are not BOM aware. 93| 39.4k| size_t counter{0}; 94| 127k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (94:22): [True: 88.0k, False: 39.4k] ------------------ 95| 88.0k| char16_t word = scalar::utf16::swap_if_needed(p[i]); 96| 88.0k| counter++; // ASCII 97| 88.0k| counter += static_cast( 98| 88.0k| word > 99| 88.0k| 0x7F); // non-ASCII is at least 2 bytes, surrogates are 2*2 == 4 bytes 100| 88.0k| counter += static_cast((word > 0x7FF && word <= 0xD7FF) || ------------------ | Branch (100:37): [True: 32.9k, False: 55.0k] | Branch (100:53): [True: 24.0k, False: 8.96k] ------------------ 101| 64.0k| (word >= 0xE000)); // three-byte ------------------ | Branch (101:36): [True: 3.76k, False: 60.2k] ------------------ 102| 88.0k| } 103| 39.4k| return counter; 104| 39.4k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_115utf16_to_latin17convertILNS_10endiannessE0EPKDsPcQaasr7simdutf6detailE18indexes_into_utf16IT0_Esr7simdutf6detailE26index_assignable_from_charIT1_EEEmS8_mS9_: 17| 8.41k| OutputPtr latin_output) { 18| 8.41k| if (len == 0) { ------------------ | Branch (18:7): [True: 6.62k, False: 1.79k] ------------------ 19| 6.62k| return 0; 20| 6.62k| } 21| 1.79k| size_t pos = 0; 22| 1.79k| const auto latin_output_start = latin_output; 23| 1.79k| uint16_t word = 0; 24| 1.79k| uint16_t too_large = 0; 25| | 26| 97.4k| while (pos < len) { ------------------ | Branch (26:10): [True: 95.6k, False: 1.79k] ------------------ 27| 95.6k| word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; ------------------ | Branch (27:12): [Folded, False: 95.6k] ------------------ 28| 95.6k| too_large |= word; 29| 95.6k| *latin_output++ = char(word & 0xFF); 30| 95.6k| pos++; 31| 95.6k| } 32| 1.79k| if ((too_large & 0xFF00) != 0) { ------------------ | Branch (32:7): [True: 0, False: 1.79k] ------------------ 33| 0| return 0; 34| 0| } 35| | 36| 1.79k| return latin_output - latin_output_start; 37| 1.79k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_115utf16_to_latin17convertILNS_10endiannessE1EPKDsPcQaasr7simdutf6detailE18indexes_into_utf16IT0_Esr7simdutf6detailE26index_assignable_from_charIT1_EEEmS8_mS9_: 17| 8.41k| OutputPtr latin_output) { 18| 8.41k| if (len == 0) { ------------------ | Branch (18:7): [True: 6.62k, False: 1.79k] ------------------ 19| 6.62k| return 0; 20| 6.62k| } 21| 1.79k| size_t pos = 0; 22| 1.79k| const auto latin_output_start = latin_output; 23| 1.79k| uint16_t word = 0; 24| 1.79k| uint16_t too_large = 0; 25| | 26| 97.4k| while (pos < len) { ------------------ | Branch (26:10): [True: 95.6k, False: 1.79k] ------------------ 27| 95.6k| word = !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; ------------------ | Branch (27:12): [True: 95.6k, Folded] ------------------ 28| 95.6k| too_large |= word; 29| 95.6k| *latin_output++ = char(word & 0xFF); 30| 95.6k| pos++; 31| 95.6k| } 32| 1.79k| if ((too_large & 0xFF00) != 0) { ------------------ | Branch (32:7): [True: 0, False: 1.79k] ------------------ 33| 0| return 0; 34| 0| } 35| | 36| 1.79k| return latin_output - latin_output_start; 37| 1.79k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_113utf16_to_utf87convertILNS_10endiannessE0EPKDsPcQsr7simdutf6detailE18indexes_into_utf16IT0_EEEmS8_mT1_: 17| 15.8k| OutputPtr utf8_output) { 18| 15.8k| size_t pos = 0; 19| 15.8k| const auto start = utf8_output; 20| 71.7k| while (pos < len) { ------------------ | Branch (20:10): [True: 56.0k, False: 15.6k] ------------------ 21| |#if SIMDUTF_CPLUSPLUS23 22| | if !consteval 23| |#endif 24| 56.0k| { 25| | // try to convert the next block of 8 bytes 26| 56.0k| if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that ------------------ | Branch (26:11): [True: 51.5k, False: 4.50k] ------------------ 27| | // they are ascii 28| 51.5k| uint64_t v; 29| 51.5k| ::memcpy(&v, data + pos, sizeof(uint64_t)); 30| | if constexpr (!match_system(big_endian)) { 31| | v = (v >> 8) | (v << (64 - 8)); 32| | } 33| 51.5k| if ((v & 0xFF80FF80FF80FF80) == 0) { ------------------ | Branch (33:13): [True: 13.8k, False: 37.7k] ------------------ 34| 13.8k| size_t final_pos = pos + 4; 35| 69.0k| while (pos < final_pos) { ------------------ | Branch (35:18): [True: 55.2k, False: 13.8k] ------------------ 36| 55.2k| *utf8_output++ = !match_system(big_endian) ------------------ | Branch (36:30): [Folded, False: 55.2k] ------------------ 37| 55.2k| ? char(u16_swap_bytes(data[pos])) 38| 55.2k| : char(data[pos]); 39| 55.2k| pos++; 40| 55.2k| } 41| 13.8k| continue; 42| 13.8k| } 43| 51.5k| } 44| 56.0k| } 45| 42.2k| uint16_t word = 46| 42.2k| !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; ------------------ | Branch (46:9): [Folded, False: 42.2k] ------------------ 47| 42.2k| if ((word & 0xFF80) == 0) { ------------------ | Branch (47:9): [True: 5.94k, False: 36.3k] ------------------ 48| | // will generate one UTF-8 bytes 49| 5.94k| *utf8_output++ = char(word); 50| 5.94k| pos++; 51| 36.3k| } else if ((word & 0xF800) == 0) { ------------------ | Branch (51:16): [True: 3.25k, False: 33.0k] ------------------ 52| | // will generate two UTF-8 bytes 53| | // we have 0b110XXXXX 0b10XXXXXX 54| 3.25k| *utf8_output++ = char((word >> 6) | 0b11000000); 55| 3.25k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 56| 3.25k| pos++; 57| 33.0k| } else if ((word & 0xF800) != 0xD800) { ------------------ | Branch (57:16): [True: 31.5k, False: 1.47k] ------------------ 58| | // will generate three UTF-8 bytes 59| | // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX 60| 31.5k| *utf8_output++ = char((word >> 12) | 0b11100000); 61| 31.5k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 62| 31.5k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 63| 31.5k| pos++; 64| 31.5k| } else { 65| | // must be a surrogate pair 66| 1.47k| if (pos + 1 >= len) { ------------------ | Branch (66:11): [True: 34, False: 1.43k] ------------------ 67| 34| return 0; 68| 34| } 69| 1.43k| uint16_t diff = uint16_t(word - 0xD800); 70| 1.43k| if (diff > 0x3FF) { ------------------ | Branch (70:11): [True: 45, False: 1.39k] ------------------ 71| 45| return 0; 72| 45| } 73| 1.39k| uint16_t next_word = !match_system(big_endian) ------------------ | Branch (73:28): [Folded, False: 1.39k] ------------------ 74| 1.39k| ? u16_swap_bytes(data[pos + 1]) 75| 1.39k| : data[pos + 1]; 76| 1.39k| uint16_t diff2 = uint16_t(next_word - 0xDC00); 77| 1.39k| if (diff2 > 0x3FF) { ------------------ | Branch (77:11): [True: 51, False: 1.34k] ------------------ 78| 51| return 0; 79| 51| } 80| 1.34k| uint32_t value = (diff << 10) + diff2 + 0x10000; 81| | // will generate four UTF-8 bytes 82| | // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX 83| 1.34k| *utf8_output++ = char((value >> 18) | 0b11110000); 84| 1.34k| *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); 85| 1.34k| *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); 86| 1.34k| *utf8_output++ = char((value & 0b111111) | 0b10000000); 87| 1.34k| pos += 2; 88| 1.34k| } 89| 42.2k| } 90| 15.6k| return utf8_output - start; 91| 15.8k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_113utf16_to_utf87convertILNS_10endiannessE1EPKDsPcQsr7simdutf6detailE18indexes_into_utf16IT0_EEEmS8_mT1_: 17| 15.6k| OutputPtr utf8_output) { 18| 15.6k| size_t pos = 0; 19| 15.6k| const auto start = utf8_output; 20| 70.0k| while (pos < len) { ------------------ | Branch (20:10): [True: 54.4k, False: 15.5k] ------------------ 21| |#if SIMDUTF_CPLUSPLUS23 22| | if !consteval 23| |#endif 24| 54.4k| { 25| | // try to convert the next block of 8 bytes 26| 54.4k| if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that ------------------ | Branch (26:11): [True: 50.2k, False: 4.16k] ------------------ 27| | // they are ascii 28| 50.2k| uint64_t v; 29| 50.2k| ::memcpy(&v, data + pos, sizeof(uint64_t)); 30| 50.2k| if constexpr (!match_system(big_endian)) { 31| 50.2k| v = (v >> 8) | (v << (64 - 8)); 32| 50.2k| } 33| 50.2k| if ((v & 0xFF80FF80FF80FF80) == 0) { ------------------ | Branch (33:13): [True: 13.9k, False: 36.3k] ------------------ 34| 13.9k| size_t final_pos = pos + 4; 35| 69.5k| while (pos < final_pos) { ------------------ | Branch (35:18): [True: 55.6k, False: 13.9k] ------------------ 36| 55.6k| *utf8_output++ = !match_system(big_endian) ------------------ | Branch (36:30): [True: 55.6k, Folded] ------------------ 37| 55.6k| ? char(u16_swap_bytes(data[pos])) 38| 55.6k| : char(data[pos]); 39| 55.6k| pos++; 40| 55.6k| } 41| 13.9k| continue; 42| 13.9k| } 43| 50.2k| } 44| 54.4k| } 45| 40.5k| uint16_t word = 46| 40.5k| !match_system(big_endian) ? u16_swap_bytes(data[pos]) : data[pos]; ------------------ | Branch (46:9): [True: 40.5k, Folded] ------------------ 47| 40.5k| if ((word & 0xFF80) == 0) { ------------------ | Branch (47:9): [True: 6.62k, False: 33.8k] ------------------ 48| | // will generate one UTF-8 bytes 49| 6.62k| *utf8_output++ = char(word); 50| 6.62k| pos++; 51| 33.8k| } else if ((word & 0xF800) == 0) { ------------------ | Branch (51:16): [True: 3.17k, False: 30.7k] ------------------ 52| | // will generate two UTF-8 bytes 53| | // we have 0b110XXXXX 0b10XXXXXX 54| 3.17k| *utf8_output++ = char((word >> 6) | 0b11000000); 55| 3.17k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 56| 3.17k| pos++; 57| 30.7k| } else if ((word & 0xF800) != 0xD800) { ------------------ | Branch (57:16): [True: 28.3k, False: 2.38k] ------------------ 58| | // will generate three UTF-8 bytes 59| | // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX 60| 28.3k| *utf8_output++ = char((word >> 12) | 0b11100000); 61| 28.3k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 62| 28.3k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 63| 28.3k| pos++; 64| 28.3k| } else { 65| | // must be a surrogate pair 66| 2.38k| if (pos + 1 >= len) { ------------------ | Branch (66:11): [True: 25, False: 2.35k] ------------------ 67| 25| return 0; 68| 25| } 69| 2.35k| uint16_t diff = uint16_t(word - 0xD800); 70| 2.35k| if (diff > 0x3FF) { ------------------ | Branch (70:11): [True: 28, False: 2.33k] ------------------ 71| 28| return 0; 72| 28| } 73| 2.33k| uint16_t next_word = !match_system(big_endian) ------------------ | Branch (73:28): [True: 2.33k, Folded] ------------------ 74| 2.33k| ? u16_swap_bytes(data[pos + 1]) 75| 2.33k| : data[pos + 1]; 76| 2.33k| uint16_t diff2 = uint16_t(next_word - 0xDC00); 77| 2.33k| if (diff2 > 0x3FF) { ------------------ | Branch (77:11): [True: 62, False: 2.26k] ------------------ 78| 62| return 0; 79| 62| } 80| 2.26k| uint32_t value = (diff << 10) + diff2 + 0x10000; 81| | // will generate four UTF-8 bytes 82| | // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX 83| 2.26k| *utf8_output++ = char((value >> 18) | 0b11110000); 84| 2.26k| *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); 85| 2.26k| *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); 86| 2.26k| *utf8_output++ = char((value & 0b111111) | 0b10000000); 87| 2.26k| pos += 2; 88| 2.26k| } 89| 40.5k| } 90| 15.5k| return utf8_output - start; 91| 15.6k|} _ZN7simdutf6scalar5utf328validateEPKDim: 25| 17.9k| size_t len) noexcept { 26| 17.9k| return validate(reinterpret_cast(buf), len); 27| 17.9k|} _ZN7simdutf6scalar5utf3222utf8_length_from_utf32EPKDim: 54| 18.9k| size_t len) { 55| | // We are not BOM aware. 56| 18.9k| size_t counter{0}; 57| 63.6k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (57:22): [True: 44.7k, False: 18.9k] ------------------ 58| | // credit: @ttsugriy for the vectorizable approach 59| 44.7k| counter++; // ASCII 60| 44.7k| counter += static_cast(p[i] > 0x7F); // two-byte 61| 44.7k| counter += static_cast(p[i] > 0x7FF); // three-byte 62| 44.7k| counter += static_cast(p[i] > 0xFFFF); // four-bytes 63| 44.7k| } 64| 18.9k| return counter; 65| 18.9k|} _ZN7simdutf6scalar5utf328validateIPKjQsr7simdutf6detailE19indexes_into_uint32IT_EEEbS5_m: 13| 17.9k| size_t len) noexcept { 14| 17.9k| uint64_t pos = 0; 15| 160k| for (; pos < len; pos++) { ------------------ | Branch (15:10): [True: 142k, False: 17.9k] ------------------ 16| 142k| uint32_t word = data[pos]; 17| 142k| if (word > 0x10FFFF || (word >= 0xD800 && word <= 0xDFFF)) { ------------------ | Branch (17:9): [True: 0, False: 142k] | Branch (17:29): [True: 1.27k, False: 140k] | Branch (17:47): [True: 0, False: 1.27k] ------------------ 18| 0| return false; 19| 0| } 20| 142k| } 21| 17.9k| return true; 22| 17.9k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_115utf32_to_latin17convertEPKDimPc: 12| 8.45k| char *latin1_output) { 13| 8.45k| char *start = latin1_output; 14| 8.45k| uint32_t utf32_char; 15| 8.45k| size_t pos = 0; 16| 8.45k| uint32_t too_large = 0; 17| | 18| 105k| while (pos < len) { ------------------ | Branch (18:10): [True: 97.0k, False: 8.45k] ------------------ 19| 97.0k| utf32_char = (uint32_t)data[pos]; 20| 97.0k| too_large |= utf32_char; 21| 97.0k| *latin1_output++ = (char)(utf32_char & 0xFF); 22| 97.0k| pos++; 23| 97.0k| } 24| 8.45k| if ((too_large & 0xFFFFFF00) != 0) { ------------------ | Branch (24:7): [True: 0, False: 8.45k] ------------------ 25| 0| return 0; 26| 0| } 27| 8.45k| return latin1_output - start; 28| 8.45k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_113utf32_to_utf87convertIPKDiPcQaasr7simdutf6detailE18indexes_into_utf32IT_Esr7simdutf6detailE26index_assignable_from_charIT0_EEEmS7_mS8_: 17| 8.26k| OutputPtr utf8_output) { 18| 8.26k| size_t pos = 0; 19| 8.26k| auto start = utf8_output; 20| 41.7k| while (pos < len) { ------------------ | Branch (20:10): [True: 33.4k, False: 8.26k] ------------------ 21| |#if SIMDUTF_CPLUSPLUS23 22| | if !consteval 23| |#endif 24| 33.4k| { // try to convert the next block of 2 ASCII characters 25| 33.4k| if (pos + 2 <= len) { // if it is safe to read 8 more bytes, check that ------------------ | Branch (25:11): [True: 32.1k, False: 1.32k] ------------------ 26| | // they are ascii 27| 32.1k| uint64_t v; 28| 32.1k| ::memcpy(&v, data + pos, sizeof(uint64_t)); 29| 32.1k| if ((v & 0xFFFFFF80FFFFFF80) == 0) { ------------------ | Branch (29:13): [True: 27.4k, False: 4.70k] ------------------ 30| 27.4k| *utf8_output++ = char(data[pos]); 31| 27.4k| *utf8_output++ = char(data[pos + 1]); 32| 27.4k| pos += 2; 33| 27.4k| continue; 34| 27.4k| } 35| 32.1k| } 36| 33.4k| } 37| | 38| 6.03k| uint32_t word = data[pos]; 39| 6.03k| if ((word & 0xFFFFFF80) == 0) { ------------------ | Branch (39:9): [True: 2.01k, False: 4.01k] ------------------ 40| | // will generate one UTF-8 bytes 41| 2.01k| *utf8_output++ = char(word); 42| 2.01k| pos++; 43| 4.01k| } else if ((word & 0xFFFFF800) == 0) { ------------------ | Branch (43:16): [True: 1.68k, False: 2.33k] ------------------ 44| | // will generate two UTF-8 bytes 45| | // we have 0b110XXXXX 0b10XXXXXX 46| 1.68k| *utf8_output++ = char((word >> 6) | 0b11000000); 47| 1.68k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 48| 1.68k| pos++; 49| 2.33k| } else if ((word & 0xFFFF0000) == 0) { ------------------ | Branch (49:16): [True: 1.16k, False: 1.16k] ------------------ 50| | // will generate three UTF-8 bytes 51| | // we have 0b1110XXXX 0b10XXXXXX 0b10XXXXXX 52| 1.16k| if (word >= 0xD800 && word <= 0xDFFF) { ------------------ | Branch (52:11): [True: 731, False: 436] | Branch (52:29): [True: 0, False: 731] ------------------ 53| 0| return 0; 54| 0| } 55| 1.16k| *utf8_output++ = char((word >> 12) | 0b11100000); 56| 1.16k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 57| 1.16k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 58| 1.16k| pos++; 59| 1.16k| } else { 60| | // will generate four UTF-8 bytes 61| | // we have 0b11110XXX 0b10XXXXXX 0b10XXXXXX 0b10XXXXXX 62| 1.16k| if (word > 0x10FFFF) { ------------------ | Branch (62:11): [True: 0, False: 1.16k] ------------------ 63| 0| return 0; 64| 0| } 65| 1.16k| *utf8_output++ = char((word >> 18) | 0b11110000); 66| 1.16k| *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); 67| 1.16k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 68| 1.16k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 69| 1.16k| pos++; 70| 1.16k| } 71| 6.03k| } 72| 8.26k| return utf8_output - start; 73| 8.26k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf88validateEPKcm: 113| 27.3k| size_t len) noexcept { 114| 27.3k| return validate(reinterpret_cast(buf), len); 115| 27.3k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf88validateIPKhEEbT_m: 14| 27.3k| size_t len) noexcept { 15| 27.3k| static_assert( 16| 27.3k| std::is_same::type, uint8_t>::value, 17| 27.3k| "dereferencing the data pointer must result in a uint8_t"); 18| 27.3k| uint64_t pos = 0; 19| 27.3k| uint32_t code_point = 0; 20| 108k| while (pos < len) { ------------------ | Branch (20:10): [True: 82.0k, False: 26.8k] ------------------ 21| 82.0k| uint64_t next_pos; 22| |#if SIMDUTF_CPLUSPLUS23 23| | if !consteval 24| |#endif 25| 82.0k| { // check if the next 16 bytes are ascii. 26| 82.0k| next_pos = pos + 16; 27| 82.0k| if (next_pos <= len) { // if it is safe to read 16 more bytes, check ------------------ | Branch (27:11): [True: 79.8k, False: 2.19k] ------------------ 28| | // that they are ascii 29| 79.8k| uint64_t v1{}; 30| 79.8k| std::memcpy(&v1, data + pos, sizeof(uint64_t)); 31| 79.8k| uint64_t v2{}; 32| 79.8k| std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 33| 79.8k| uint64_t v{v1 | v2}; 34| 79.8k| if ((v & 0x8080808080808080) == 0) { ------------------ | Branch (34:13): [True: 4.91k, False: 74.9k] ------------------ 35| 4.91k| pos = next_pos; 36| 4.91k| continue; 37| 4.91k| } 38| 79.8k| } 39| 82.0k| } 40| | 41| 77.1k| unsigned char byte = data[pos]; 42| | 43| 102k| while (byte < 0b10000000) { ------------------ | Branch (43:12): [True: 25.9k, False: 76.8k] ------------------ 44| 25.9k| if (++pos == len) { ------------------ | Branch (44:11): [True: 292, False: 25.6k] ------------------ 45| 292| return true; 46| 292| } 47| 25.6k| byte = data[pos]; 48| 25.6k| } 49| | 50| 76.8k| if ((byte & 0b11100000) == 0b11000000) { ------------------ | Branch (50:9): [True: 36.6k, False: 40.2k] ------------------ 51| 36.6k| next_pos = pos + 2; 52| 36.6k| if (next_pos > len) { ------------------ | Branch (52:11): [True: 9, False: 36.6k] ------------------ 53| 9| return false; 54| 9| } 55| 36.6k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (55:11): [True: 24, False: 36.5k] ------------------ 56| 24| return false; 57| 24| } 58| | // range check 59| 36.5k| code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); 60| 36.5k| if (code_point < 0x80) { ------------------ | Branch (60:11): [True: 5, False: 36.5k] ------------------ 61| 5| return false; 62| 5| } 63| 40.2k| } else if ((byte & 0b11110000) == 0b11100000) { ------------------ | Branch (63:16): [True: 38.9k, False: 1.23k] ------------------ 64| 38.9k| next_pos = pos + 3; 65| 38.9k| if (next_pos > len) { ------------------ | Branch (65:11): [True: 8, False: 38.9k] ------------------ 66| 8| return false; 67| 8| } 68| 38.9k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (68:11): [True: 6, False: 38.9k] ------------------ 69| 6| return false; 70| 6| } 71| 38.9k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (71:11): [True: 5, False: 38.9k] ------------------ 72| 5| return false; 73| 5| } 74| | // range check 75| 38.9k| code_point = (byte & 0b00001111) << 12 | 76| 38.9k| (data[pos + 1] & 0b00111111) << 6 | 77| 38.9k| (data[pos + 2] & 0b00111111); 78| 38.9k| if ((code_point < 0x800) || ------------------ | Branch (78:11): [True: 9, False: 38.9k] ------------------ 79| 38.9k| (0xd7ff < code_point && code_point < 0xe000)) { ------------------ | Branch (79:12): [True: 4.74k, False: 34.2k] | Branch (79:35): [True: 3, False: 4.74k] ------------------ 80| 12| return false; 81| 12| } 82| 38.9k| } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 ------------------ | Branch (82:16): [True: 1.13k, False: 99] ------------------ 83| 1.13k| next_pos = pos + 4; 84| 1.13k| if (next_pos > len) { ------------------ | Branch (84:11): [True: 5, False: 1.12k] ------------------ 85| 5| return false; 86| 5| } 87| 1.12k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (87:11): [True: 9, False: 1.11k] ------------------ 88| 9| return false; 89| 9| } 90| 1.11k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (90:11): [True: 8, False: 1.11k] ------------------ 91| 8| return false; 92| 8| } 93| 1.11k| if ((data[pos + 3] & 0b11000000) != 0b10000000) { ------------------ | Branch (93:11): [True: 5, False: 1.10k] ------------------ 94| 5| return false; 95| 5| } 96| | // range check 97| 1.10k| code_point = 98| 1.10k| (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | 99| 1.10k| (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); 100| 1.10k| if (code_point <= 0xffff || 0x10ffff < code_point) { ------------------ | Branch (100:11): [True: 7, False: 1.09k] | Branch (100:35): [True: 4, False: 1.09k] ------------------ 101| 11| return false; 102| 11| } 103| 1.10k| } else { 104| | // we may have a continuation 105| 99| return false; 106| 99| } 107| 76.6k| pos = next_pos; 108| 76.6k| } 109| 26.8k| return true; 110| 27.3k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf820validate_with_errorsEPKcm: 221| 8.17k|validate_with_errors(const char *buf, size_t len) noexcept { 222| 8.17k| return validate_with_errors(reinterpret_cast(buf), len); 223| 8.17k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf820validate_with_errorsIPKhEENS_6resultET_m: 119| 8.17k|validate_with_errors(BytePtr data, size_t len) noexcept { 120| 8.17k| static_assert( 121| 8.17k| std::is_same::type, uint8_t>::value, 122| 8.17k| "dereferencing the data pointer must result in a uint8_t"); 123| 8.17k| size_t pos = 0; 124| 8.17k| uint32_t code_point = 0; 125| 15.9k| while (pos < len) { ------------------ | Branch (125:10): [True: 9.44k, False: 6.51k] ------------------ 126| | // check of the next 16 bytes are ascii. 127| 9.44k| size_t next_pos = pos + 16; 128| 9.44k| if (next_pos <= ------------------ | Branch (128:9): [True: 8.23k, False: 1.21k] ------------------ 129| 9.44k| len) { // if it is safe to read 16 more bytes, check that they are ascii 130| 8.23k| uint64_t v1; 131| 8.23k| std::memcpy(&v1, data + pos, sizeof(uint64_t)); 132| 8.23k| uint64_t v2; 133| 8.23k| std::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 134| 8.23k| uint64_t v{v1 | v2}; 135| 8.23k| if ((v & 0x8080808080808080) == 0) { ------------------ | Branch (135:11): [True: 2.74k, False: 5.48k] ------------------ 136| 2.74k| pos = next_pos; 137| 2.74k| continue; 138| 2.74k| } 139| 8.23k| } 140| 6.69k| unsigned char byte = data[pos]; 141| | 142| 17.4k| while (byte < 0b10000000) { ------------------ | Branch (142:12): [True: 10.8k, False: 6.56k] ------------------ 143| 10.8k| if (++pos == len) { ------------------ | Branch (143:11): [True: 133, False: 10.7k] ------------------ 144| 133| return result(error_code::SUCCESS, len); 145| 133| } 146| 10.7k| byte = data[pos]; 147| 10.7k| } 148| | 149| 6.56k| if ((byte & 0b11100000) == 0b11000000) { ------------------ | Branch (149:9): [True: 3.09k, False: 3.47k] ------------------ 150| 3.09k| next_pos = pos + 2; 151| 3.09k| if (next_pos > len) { ------------------ | Branch (151:11): [True: 51, False: 3.04k] ------------------ 152| 51| return result(error_code::TOO_SHORT, pos); 153| 51| } 154| 3.04k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (154:11): [True: 198, False: 2.84k] ------------------ 155| 198| return result(error_code::TOO_SHORT, pos); 156| 198| } 157| | // range check 158| 2.84k| code_point = (byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); 159| 2.84k| if (code_point < 0x80) { ------------------ | Branch (159:11): [True: 31, False: 2.81k] ------------------ 160| 31| return result(error_code::OVERLONG, pos); 161| 31| } 162| 3.47k| } else if ((byte & 0b11110000) == 0b11100000) { ------------------ | Branch (162:16): [True: 1.68k, False: 1.79k] ------------------ 163| 1.68k| next_pos = pos + 3; 164| 1.68k| if (next_pos > len) { ------------------ | Branch (164:11): [True: 48, False: 1.63k] ------------------ 165| 48| return result(error_code::TOO_SHORT, pos); 166| 48| } 167| 1.63k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (167:11): [True: 107, False: 1.52k] ------------------ 168| 107| return result(error_code::TOO_SHORT, pos); 169| 107| } 170| 1.52k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (170:11): [True: 58, False: 1.46k] ------------------ 171| 58| return result(error_code::TOO_SHORT, pos); 172| 58| } 173| | // range check 174| 1.46k| code_point = (byte & 0b00001111) << 12 | 175| 1.46k| (data[pos + 1] & 0b00111111) << 6 | 176| 1.46k| (data[pos + 2] & 0b00111111); 177| 1.46k| if (code_point < 0x800) { ------------------ | Branch (177:11): [True: 23, False: 1.44k] ------------------ 178| 23| return result(error_code::OVERLONG, pos); 179| 23| } 180| 1.44k| if (0xd7ff < code_point && code_point < 0xe000) { ------------------ | Branch (180:11): [True: 895, False: 551] | Branch (180:34): [True: 20, False: 875] ------------------ 181| 20| return result(error_code::SURROGATE, pos); 182| 20| } 183| 1.79k| } else if ((byte & 0b11111000) == 0b11110000) { // 0b11110000 ------------------ | Branch (183:16): [True: 1.05k, False: 736] ------------------ 184| 1.05k| next_pos = pos + 4; 185| 1.05k| if (next_pos > len) { ------------------ | Branch (185:11): [True: 36, False: 1.02k] ------------------ 186| 36| return result(error_code::TOO_SHORT, pos); 187| 36| } 188| 1.02k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (188:11): [True: 68, False: 952] ------------------ 189| 68| return result(error_code::TOO_SHORT, pos); 190| 68| } 191| 952| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (191:11): [True: 54, False: 898] ------------------ 192| 54| return result(error_code::TOO_SHORT, pos); 193| 54| } 194| 898| if ((data[pos + 3] & 0b11000000) != 0b10000000) { ------------------ | Branch (194:11): [True: 60, False: 838] ------------------ 195| 60| return result(error_code::TOO_SHORT, pos); 196| 60| } 197| | // range check 198| 838| code_point = 199| 838| (byte & 0b00000111) << 18 | (data[pos + 1] & 0b00111111) << 12 | 200| 838| (data[pos + 2] & 0b00111111) << 6 | (data[pos + 3] & 0b00111111); 201| 838| if (code_point <= 0xffff) { ------------------ | Branch (201:11): [True: 19, False: 819] ------------------ 202| 19| return result(error_code::OVERLONG, pos); 203| 19| } 204| 819| if (0x10ffff < code_point) { ------------------ | Branch (204:11): [True: 18, False: 801] ------------------ 205| 18| return result(error_code::TOO_LARGE, pos); 206| 18| } 207| 819| } else { 208| | // we either have too many continuation bytes or an invalid leading byte 209| 736| if ((byte & 0b11000000) == 0b10000000) { ------------------ | Branch (209:11): [True: 444, False: 292] ------------------ 210| 444| return result(error_code::TOO_LONG, pos); 211| 444| } else { 212| 292| return result(error_code::HEADER_BITS, pos); 213| 292| } 214| 736| } 215| 5.03k| pos = next_pos; 216| 5.03k| } 217| 6.51k| return result(error_code::SUCCESS, len); 218| 8.17k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf831rewind_and_validate_with_errorsEPKcS4_m: 232| 1.42k| const char *start, const char *buf, size_t len) noexcept { 233| | // First check that we start with a leading byte 234| 1.42k| if ((*start & 0b11000000) == 0b10000000) { ------------------ | Branch (234:7): [True: 101, False: 1.32k] ------------------ 235| 101| return result(error_code::TOO_LONG, 0); 236| 101| } 237| 1.32k| size_t extra_len{0}; 238| | // A leading byte cannot be further than 4 bytes away 239| 1.42k| for (int i = 0; i < 5; i++) { ------------------ | Branch (239:19): [True: 1.42k, False: 0] ------------------ 240| 1.42k| unsigned char byte = *buf; 241| 1.42k| if ((byte & 0b11000000) != 0b10000000) { ------------------ | Branch (241:9): [True: 1.32k, False: 100] ------------------ 242| 1.32k| break; 243| 1.32k| } else { 244| 100| buf--; 245| 100| extra_len++; 246| 100| } 247| 1.42k| } 248| | 249| 1.32k| result res = validate_with_errors(buf, len + extra_len); 250| 1.32k| res.count -= extra_len; // Might underflow 251| 1.32k| return res; 252| 1.42k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf817count_code_pointsIPKcQsr7simdutf6detailE22indexes_into_byte_likeIT_EEEmS6_m: 454| 61.6k|simdutf_constexpr23 size_t count_code_points(InputPtr data, size_t len) { 455| 61.6k| size_t counter{0}; 456| 612k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (456:22): [True: 550k, False: 61.6k] ------------------ 457| | // -65 is 0b10111111, anything larger in two-complement's should start a new 458| | // code point. 459| 550k| if (int8_t(data[i]) > -65) { ------------------ | Branch (459:9): [True: 426k, False: 124k] ------------------ 460| 426k| counter++; 461| 426k| } 462| 550k| } 463| 61.6k| return counter; 464| 61.6k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_14utf822utf16_length_from_utf8IPKcQsr7simdutf6detailE22indexes_into_byte_likeIT_EEEmS6_m: 470| 81.7k|simdutf_constexpr23 size_t utf16_length_from_utf8(InputPtr data, size_t len) { 471| 81.7k| size_t counter{0}; 472| 458k| for (size_t i = 0; i < len; i++) { ------------------ | Branch (472:22): [True: 377k, False: 81.7k] ------------------ 473| 377k| if (int8_t(data[i]) > -65) { ------------------ | Branch (473:9): [True: 247k, False: 130k] ------------------ 474| 247k| counter++; 475| 247k| } 476| 377k| if (uint8_t(data[i]) >= 240) { ------------------ | Branch (476:9): [True: 10.7k, False: 366k] ------------------ 477| 10.7k| counter++; 478| 10.7k| } 479| 377k| } 480| 81.7k| return counter; 481| 81.7k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_114utf8_to_latin17convertIPKcPcQaasr7simdutf6detailE22indexes_into_byte_likeIT_Esr7simdutf6detailE22indexes_into_byte_likeIT0_EEEmS7_mS8_: 17| 39.7k| OutputPtr latin_output) { 18| 39.7k| size_t pos = 0; 19| 39.7k| auto start = latin_output; 20| | 21| 154k| while (pos < len) { ------------------ | Branch (21:10): [True: 116k, False: 37.5k] ------------------ 22| |#if SIMDUTF_CPLUSPLUS23 23| | if !consteval 24| |#endif 25| 116k| { 26| | // try to convert the next block of 16 ASCII bytes 27| 116k| if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that ------------------ | Branch (27:11): [True: 94.1k, False: 22.7k] ------------------ 28| | // they are ascii 29| 94.1k| uint64_t v1; 30| 94.1k| ::memcpy(&v1, data + pos, sizeof(uint64_t)); 31| 94.1k| uint64_t v2; 32| 94.1k| ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 33| 94.1k| uint64_t v{v1 | v2}; // We are only interested in these bits: 1000 1000 34| | // 1000 1000 .... etc 35| 94.1k| if ((v & 0x8080808080808080) == ------------------ | Branch (35:13): [True: 6.12k, False: 88.0k] ------------------ 36| 94.1k| 0) { // if NONE of these are set, e.g. all of them are zero, then 37| | // everything is ASCII 38| 6.12k| size_t final_pos = pos + 16; 39| 104k| while (pos < final_pos) { ------------------ | Branch (39:18): [True: 97.9k, False: 6.12k] ------------------ 40| 97.9k| *latin_output++ = char(data[pos]); 41| 97.9k| pos++; 42| 97.9k| } 43| 6.12k| continue; 44| 6.12k| } 45| 94.1k| } 46| 116k| } 47| | 48| | // suppose it is not an all ASCII byte sequence 49| 110k| uint8_t leading_byte = data[pos]; // leading byte 50| 110k| if (leading_byte < 0b10000000) { ------------------ | Branch (50:9): [True: 47.2k, False: 63.6k] ------------------ 51| | // converting one ASCII byte !!! 52| 47.2k| *latin_output++ = char(leading_byte); 53| 47.2k| pos++; 54| 63.6k| } else if ((leading_byte & 0b11100000) == ------------------ | Branch (54:16): [True: 61.7k, False: 1.87k] ------------------ 55| 63.6k| 0b11000000) { // the first three bits indicate: 56| | // We have a two-byte UTF-8 57| 61.7k| if (pos + 1 >= len) { ------------------ | Branch (57:11): [True: 32, False: 61.7k] ------------------ 58| 32| return 0; 59| 32| } // minimal bound checking 60| 61.7k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (60:11): [True: 108, False: 61.6k] ------------------ 61| 108| return 0; 62| 108| } // checks if the next byte is a valid continuation byte in UTF-8. A 63| | // valid continuation byte starts with 10. 64| | // range check - 65| 61.6k| uint32_t code_point = 66| 61.6k| (leading_byte & 0b00011111) << 6 | 67| 61.6k| (data[pos + 1] & 68| 61.6k| 0b00111111); // assembles the Unicode code point from the two bytes. 69| | // It does this by discarding the leading 110 and 10 70| | // bits from the two bytes, shifting the remaining bits 71| | // of the first byte, and then combining the results 72| | // with a bitwise OR operation. 73| 61.6k| if (code_point < 0x80 || 0xFF < code_point) { ------------------ | Branch (73:11): [True: 30, False: 61.5k] | Branch (73:32): [True: 213, False: 61.3k] ------------------ 74| 243| return 0; // We only care about the range 129-255 which is Non-ASCII 75| | // latin1 characters. A code_point beneath 0x80 is invalid as 76| | // it is already covered by bytes whose leading bit is zero. 77| 243| } 78| 61.3k| *latin_output++ = char(code_point); 79| 61.3k| pos += 2; 80| 61.3k| } else { 81| 1.87k| return 0; 82| 1.87k| } 83| 110k| } 84| 37.5k| return latin_output - start; 85| 39.7k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_113utf8_to_utf167convertILNS_10endiannessE0EPKcQsr7simdutf6detailE22indexes_into_byte_likeIT0_EEEmS7_mPDs: 16| 40.1k| char16_t *utf16_output) { 17| 40.1k| size_t pos = 0; 18| 40.1k| char16_t *start{utf16_output}; 19| 104k| while (pos < len) { ------------------ | Branch (19:10): [True: 65.6k, False: 39.0k] ------------------ 20| |#if SIMDUTF_CPLUSPLUS23 21| | if !consteval 22| |#endif 23| | // try to convert the next block of 16 ASCII bytes 24| 65.6k| { 25| 65.6k| if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that ------------------ | Branch (25:11): [True: 51.2k, False: 14.3k] ------------------ 26| | // they are ascii 27| 51.2k| uint64_t v1; 28| 51.2k| ::memcpy(&v1, data + pos, sizeof(uint64_t)); 29| 51.2k| uint64_t v2; 30| 51.2k| ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 31| 51.2k| uint64_t v{v1 | v2}; 32| 51.2k| if ((v & 0x8080808080808080) == 0) { ------------------ | Branch (32:13): [True: 3.52k, False: 47.7k] ------------------ 33| 3.52k| size_t final_pos = pos + 16; 34| 59.8k| while (pos < final_pos) { ------------------ | Branch (34:18): [True: 56.3k, False: 3.52k] ------------------ 35| 56.3k| *utf16_output++ = !match_system(big_endian) ------------------ | Branch (35:31): [Folded, False: 56.3k] ------------------ 36| 56.3k| ? char16_t(u16_swap_bytes(data[pos])) 37| 56.3k| : char16_t(data[pos]); 38| 56.3k| pos++; 39| 56.3k| } 40| 3.52k| continue; 41| 3.52k| } 42| 51.2k| } 43| 65.6k| } 44| | 45| 62.1k| uint8_t leading_byte = data[pos]; // leading byte 46| 62.1k| if (leading_byte < 0b10000000) { ------------------ | Branch (46:9): [True: 24.7k, False: 37.3k] ------------------ 47| | // converting one ASCII byte !!! 48| 24.7k| *utf16_output++ = !match_system(big_endian) ------------------ | Branch (48:25): [Folded, False: 24.7k] ------------------ 49| 24.7k| ? char16_t(u16_swap_bytes(leading_byte)) 50| 24.7k| : char16_t(leading_byte); 51| 24.7k| pos++; 52| 37.3k| } else if ((leading_byte & 0b11100000) == 0b11000000) { ------------------ | Branch (52:16): [True: 5.54k, False: 31.8k] ------------------ 53| | // We have a two-byte UTF-8, it should become 54| | // a single UTF-16 word. 55| 5.54k| if (pos + 1 >= len) { ------------------ | Branch (55:11): [True: 51, False: 5.49k] ------------------ 56| 51| return 0; 57| 51| } // minimal bound checking 58| 5.49k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (58:11): [True: 155, False: 5.33k] ------------------ 59| 155| return 0; 60| 155| } 61| | // range check 62| 5.33k| uint32_t code_point = 63| 5.33k| (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); 64| 5.33k| if (code_point < 0x80) { ------------------ | Branch (64:11): [True: 31, False: 5.30k] ------------------ 65| 31| return 0; 66| 31| } 67| | if constexpr (!match_system(big_endian)) { 68| | code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); 69| | } 70| 5.30k| *utf16_output++ = char16_t(code_point); 71| 5.30k| pos += 2; 72| 31.8k| } else if ((leading_byte & 0b11110000) == 0b11100000) { ------------------ | Branch (72:16): [True: 28.6k, False: 3.14k] ------------------ 73| | // We have a three-byte UTF-8, it should become 74| | // a single UTF-16 word. 75| 28.6k| if (pos + 2 >= len) { ------------------ | Branch (75:11): [True: 48, False: 28.6k] ------------------ 76| 48| return 0; 77| 48| } // minimal bound checking 78| | 79| 28.6k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (79:11): [True: 50, False: 28.5k] ------------------ 80| 50| return 0; 81| 50| } 82| 28.5k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (82:11): [True: 33, False: 28.5k] ------------------ 83| 33| return 0; 84| 33| } 85| | // range check 86| 28.5k| uint32_t code_point = (leading_byte & 0b00001111) << 12 | 87| 28.5k| (data[pos + 1] & 0b00111111) << 6 | 88| 28.5k| (data[pos + 2] & 0b00111111); 89| 28.5k| if (code_point < 0x800 || (0xd7ff < code_point && code_point < 0xe000)) { ------------------ | Branch (89:11): [True: 20, False: 28.5k] | Branch (89:34): [True: 5.22k, False: 23.3k] | Branch (89:57): [True: 16, False: 5.20k] ------------------ 90| 36| return 0; 91| 36| } 92| | if constexpr (!match_system(big_endian)) { 93| | code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); 94| | } 95| 28.5k| *utf16_output++ = char16_t(code_point); 96| 28.5k| pos += 3; 97| 28.5k| } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 ------------------ | Branch (97:16): [True: 2.57k, False: 567] ------------------ 98| | // we have a 4-byte UTF-8 word. 99| 2.57k| if (pos + 3 >= len) { ------------------ | Branch (99:11): [True: 36, False: 2.53k] ------------------ 100| 36| return 0; 101| 36| } // minimal bound checking 102| 2.53k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (102:11): [True: 38, False: 2.50k] ------------------ 103| 38| return 0; 104| 38| } 105| 2.50k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (105:11): [True: 33, False: 2.46k] ------------------ 106| 33| return 0; 107| 33| } 108| 2.46k| if ((data[pos + 3] & 0b11000000) != 0b10000000) { ------------------ | Branch (108:11): [True: 45, False: 2.42k] ------------------ 109| 45| return 0; 110| 45| } 111| | 112| | // range check 113| 2.42k| uint32_t code_point = (leading_byte & 0b00000111) << 18 | 114| 2.42k| (data[pos + 1] & 0b00111111) << 12 | 115| 2.42k| (data[pos + 2] & 0b00111111) << 6 | 116| 2.42k| (data[pos + 3] & 0b00111111); 117| 2.42k| if (code_point <= 0xffff || 0x10ffff < code_point) { ------------------ | Branch (117:11): [True: 18, False: 2.40k] | Branch (117:35): [True: 18, False: 2.38k] ------------------ 118| 36| return 0; 119| 36| } 120| 2.38k| code_point -= 0x10000; 121| | // The mask is a no-op (code_point < 0x100000). It stops clang 18's SLP 122| | // vectorizer on RISC-V V from narrowing code_point to 16 bits before 123| | // the shift, which drops bits 16-19 of the code point. 124| 2.38k| uint16_t high_surrogate = uint16_t(0xD800 + ((code_point >> 10) & 0x3FF)); 125| 2.38k| uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); 126| | if constexpr (!match_system(big_endian)) { 127| | high_surrogate = u16_swap_bytes(high_surrogate); 128| | low_surrogate = u16_swap_bytes(low_surrogate); 129| | } 130| 2.38k| *utf16_output++ = char16_t(high_surrogate); 131| 2.38k| *utf16_output++ = char16_t(low_surrogate); 132| 2.38k| pos += 4; 133| 2.38k| } else { 134| 567| return 0; 135| 567| } 136| 62.1k| } 137| 39.0k| return utf16_output - start; 138| 40.1k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_113utf8_to_utf167convertILNS_10endiannessE1EPKcQsr7simdutf6detailE22indexes_into_byte_likeIT0_EEEmS7_mPDs: 16| 40.2k| char16_t *utf16_output) { 17| 40.2k| size_t pos = 0; 18| 40.2k| char16_t *start{utf16_output}; 19| 102k| while (pos < len) { ------------------ | Branch (19:10): [True: 62.9k, False: 39.0k] ------------------ 20| |#if SIMDUTF_CPLUSPLUS23 21| | if !consteval 22| |#endif 23| | // try to convert the next block of 16 ASCII bytes 24| 62.9k| { 25| 62.9k| if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that ------------------ | Branch (25:11): [True: 49.1k, False: 13.8k] ------------------ 26| | // they are ascii 27| 49.1k| uint64_t v1; 28| 49.1k| ::memcpy(&v1, data + pos, sizeof(uint64_t)); 29| 49.1k| uint64_t v2; 30| 49.1k| ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 31| 49.1k| uint64_t v{v1 | v2}; 32| 49.1k| if ((v & 0x8080808080808080) == 0) { ------------------ | Branch (32:13): [True: 3.53k, False: 45.6k] ------------------ 33| 3.53k| size_t final_pos = pos + 16; 34| 60.1k| while (pos < final_pos) { ------------------ | Branch (34:18): [True: 56.6k, False: 3.53k] ------------------ 35| 56.6k| *utf16_output++ = !match_system(big_endian) ------------------ | Branch (35:31): [True: 56.6k, Folded] ------------------ 36| 56.6k| ? char16_t(u16_swap_bytes(data[pos])) 37| 56.6k| : char16_t(data[pos]); 38| 56.6k| pos++; 39| 56.6k| } 40| 3.53k| continue; 41| 3.53k| } 42| 49.1k| } 43| 62.9k| } 44| | 45| 59.4k| uint8_t leading_byte = data[pos]; // leading byte 46| 59.4k| if (leading_byte < 0b10000000) { ------------------ | Branch (46:9): [True: 24.8k, False: 34.5k] ------------------ 47| | // converting one ASCII byte !!! 48| 24.8k| *utf16_output++ = !match_system(big_endian) ------------------ | Branch (48:25): [True: 24.8k, Folded] ------------------ 49| 24.8k| ? char16_t(u16_swap_bytes(leading_byte)) 50| 24.8k| : char16_t(leading_byte); 51| 24.8k| pos++; 52| 34.5k| } else if ((leading_byte & 0b11100000) == 0b11000000) { ------------------ | Branch (52:16): [True: 5.46k, False: 29.0k] ------------------ 53| | // We have a two-byte UTF-8, it should become 54| | // a single UTF-16 word. 55| 5.46k| if (pos + 1 >= len) { ------------------ | Branch (55:11): [True: 51, False: 5.41k] ------------------ 56| 51| return 0; 57| 51| } // minimal bound checking 58| 5.41k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (58:11): [True: 155, False: 5.25k] ------------------ 59| 155| return 0; 60| 155| } 61| | // range check 62| 5.25k| uint32_t code_point = 63| 5.25k| (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); 64| 5.25k| if (code_point < 0x80) { ------------------ | Branch (64:11): [True: 31, False: 5.22k] ------------------ 65| 31| return 0; 66| 31| } 67| 5.22k| if constexpr (!match_system(big_endian)) { 68| 5.22k| code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); 69| 5.22k| } 70| 5.22k| *utf16_output++ = char16_t(code_point); 71| 5.22k| pos += 2; 72| 29.0k| } else if ((leading_byte & 0b11110000) == 0b11100000) { ------------------ | Branch (72:16): [True: 26.0k, False: 3.06k] ------------------ 73| | // We have a three-byte UTF-8, it should become 74| | // a single UTF-16 word. 75| 26.0k| if (pos + 2 >= len) { ------------------ | Branch (75:11): [True: 48, False: 25.9k] ------------------ 76| 48| return 0; 77| 48| } // minimal bound checking 78| | 79| 25.9k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (79:11): [True: 50, False: 25.9k] ------------------ 80| 50| return 0; 81| 50| } 82| 25.9k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (82:11): [True: 33, False: 25.9k] ------------------ 83| 33| return 0; 84| 33| } 85| | // range check 86| 25.9k| uint32_t code_point = (leading_byte & 0b00001111) << 12 | 87| 25.9k| (data[pos + 1] & 0b00111111) << 6 | 88| 25.9k| (data[pos + 2] & 0b00111111); 89| 25.9k| if (code_point < 0x800 || (0xd7ff < code_point && code_point < 0xe000)) { ------------------ | Branch (89:11): [True: 20, False: 25.8k] | Branch (89:34): [True: 4.52k, False: 21.3k] | Branch (89:57): [True: 16, False: 4.50k] ------------------ 90| 36| return 0; 91| 36| } 92| 25.8k| if constexpr (!match_system(big_endian)) { 93| 25.8k| code_point = uint32_t(u16_swap_bytes(uint16_t(code_point))); 94| 25.8k| } 95| 25.8k| *utf16_output++ = char16_t(code_point); 96| 25.8k| pos += 3; 97| 25.8k| } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 ------------------ | Branch (97:16): [True: 2.49k, False: 567] ------------------ 98| | // we have a 4-byte UTF-8 word. 99| 2.49k| if (pos + 3 >= len) { ------------------ | Branch (99:11): [True: 36, False: 2.45k] ------------------ 100| 36| return 0; 101| 36| } // minimal bound checking 102| 2.45k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (102:11): [True: 38, False: 2.41k] ------------------ 103| 38| return 0; 104| 38| } 105| 2.41k| if ((data[pos + 2] & 0b11000000) != 0b10000000) { ------------------ | Branch (105:11): [True: 33, False: 2.38k] ------------------ 106| 33| return 0; 107| 33| } 108| 2.38k| if ((data[pos + 3] & 0b11000000) != 0b10000000) { ------------------ | Branch (108:11): [True: 45, False: 2.34k] ------------------ 109| 45| return 0; 110| 45| } 111| | 112| | // range check 113| 2.34k| uint32_t code_point = (leading_byte & 0b00000111) << 18 | 114| 2.34k| (data[pos + 1] & 0b00111111) << 12 | 115| 2.34k| (data[pos + 2] & 0b00111111) << 6 | 116| 2.34k| (data[pos + 3] & 0b00111111); 117| 2.34k| if (code_point <= 0xffff || 0x10ffff < code_point) { ------------------ | Branch (117:11): [True: 18, False: 2.32k] | Branch (117:35): [True: 18, False: 2.30k] ------------------ 118| 36| return 0; 119| 36| } 120| 2.30k| code_point -= 0x10000; 121| | // The mask is a no-op (code_point < 0x100000). It stops clang 18's SLP 122| | // vectorizer on RISC-V V from narrowing code_point to 16 bits before 123| | // the shift, which drops bits 16-19 of the code point. 124| 2.30k| uint16_t high_surrogate = uint16_t(0xD800 + ((code_point >> 10) & 0x3FF)); 125| 2.30k| uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); 126| 2.30k| if constexpr (!match_system(big_endian)) { 127| 2.30k| high_surrogate = u16_swap_bytes(high_surrogate); 128| 2.30k| low_surrogate = u16_swap_bytes(low_surrogate); 129| 2.30k| } 130| 2.30k| *utf16_output++ = char16_t(high_surrogate); 131| 2.30k| *utf16_output++ = char16_t(low_surrogate); 132| 2.30k| pos += 4; 133| 2.30k| } else { 134| 567| return 0; 135| 567| } 136| 59.4k| } 137| 39.0k| return utf16_output - start; 138| 40.2k|} simdutf.cpp:_ZN7simdutf6scalar12_GLOBAL__N_113utf8_to_utf327convertIPKcQsr7simdutf6detailE22indexes_into_byte_likeIT_EEEmS6_mPDi: 16| 19.8k| char32_t *utf32_output) { 17| 19.8k| size_t pos = 0; 18| 19.8k| char32_t *start{utf32_output}; 19| 52.9k| while (pos < len) { ------------------ | Branch (19:10): [True: 34.2k, False: 18.7k] ------------------ 20| |#if SIMDUTF_CPLUSPLUS23 21| | if !consteval 22| |#endif 23| 34.2k| { 24| | // try to convert the next block of 16 ASCII bytes 25| 34.2k| if (pos + 16 <= len) { // if it is safe to read 16 more bytes, check that ------------------ | Branch (25:11): [True: 23.3k, False: 10.9k] ------------------ 26| | // they are ascii 27| 23.3k| uint64_t v1; 28| 23.3k| ::memcpy(&v1, data + pos, sizeof(uint64_t)); 29| 23.3k| uint64_t v2; 30| 23.3k| ::memcpy(&v2, data + pos + sizeof(uint64_t), sizeof(uint64_t)); 31| 23.3k| uint64_t v{v1 | v2}; 32| 23.3k| if ((v & 0x8080808080808080) == 0) { ------------------ | Branch (32:13): [True: 3.39k, False: 19.9k] ------------------ 33| 3.39k| size_t final_pos = pos + 16; 34| 57.7k| while (pos < final_pos) { ------------------ | Branch (34:18): [True: 54.3k, False: 3.39k] ------------------ 35| 54.3k| *utf32_output++ = uint8_t(data[pos]); 36| 54.3k| pos++; 37| 54.3k| } 38| 3.39k| continue; 39| 3.39k| } 40| 23.3k| } 41| 34.2k| } 42| 30.8k| auto leading_byte = uint8_t(data[pos]); // leading byte 43| 30.8k| if (leading_byte < 0b10000000) { ------------------ | Branch (43:9): [True: 21.7k, False: 9.07k] ------------------ 44| | // converting one ASCII byte !!! 45| 21.7k| *utf32_output++ = char32_t(leading_byte); 46| 21.7k| pos++; 47| 21.7k| } else if ((leading_byte & 0b11100000) == 0b11000000) { ------------------ | Branch (47:16): [True: 4.00k, False: 5.07k] ------------------ 48| | // We have a two-byte UTF-8 49| 4.00k| if (pos + 1 >= len) { ------------------ | Branch (49:11): [True: 51, False: 3.95k] ------------------ 50| 51| return 0; 51| 51| } // minimal bound checking 52| 3.95k| if ((data[pos + 1] & 0b11000000) != 0b10000000) { ------------------ | Branch (52:11): [True: 155, False: 3.79k] ------------------ 53| 155| return 0; 54| 155| } 55| | // range check 56| 3.79k| uint32_t code_point = (leading_byte & 0b00011111) << 6 | 57| 3.79k| (uint8_t(data[pos + 1]) & 0b00111111); 58| 3.79k| if (code_point < 0x80) { ------------------ | Branch (58:11): [True: 31, False: 3.76k] ------------------ 59| 31| return 0; 60| 31| } 61| 3.76k| *utf32_output++ = char32_t(code_point); 62| 3.76k| pos += 2; 63| 5.07k| } else if ((leading_byte & 0b11110000) == 0b11100000) { ------------------ | Branch (63:16): [True: 2.41k, False: 2.66k] ------------------ 64| | // We have a three-byte UTF-8 65| 2.41k| if (pos + 2 >= len) { ------------------ | Branch (65:11): [True: 48, False: 2.36k] ------------------ 66| 48| return 0; 67| 48| } // minimal bound checking 68| | 69| 2.36k| if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { ------------------ | Branch (69:11): [True: 50, False: 2.31k] ------------------ 70| 50| return 0; 71| 50| } 72| 2.31k| if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { ------------------ | Branch (72:11): [True: 33, False: 2.27k] ------------------ 73| 33| return 0; 74| 33| } 75| | // range check 76| 2.27k| uint32_t code_point = (leading_byte & 0b00001111) << 12 | 77| 2.27k| (uint8_t(data[pos + 1]) & 0b00111111) << 6 | 78| 2.27k| (uint8_t(data[pos + 2]) & 0b00111111); 79| 2.27k| if (code_point < 0x800 || (0xd7ff < code_point && code_point < 0xe000)) { ------------------ | Branch (79:11): [True: 20, False: 2.25k] | Branch (79:34): [True: 1.29k, False: 961] | Branch (79:57): [True: 16, False: 1.28k] ------------------ 80| 36| return 0; 81| 36| } 82| 2.24k| *utf32_output++ = char32_t(code_point); 83| 2.24k| pos += 3; 84| 2.66k| } else if ((leading_byte & 0b11111000) == 0b11110000) { // 0b11110000 ------------------ | Branch (84:16): [True: 2.09k, False: 567] ------------------ 85| | // we have a 4-byte UTF-8 word. 86| 2.09k| if (pos + 3 >= len) { ------------------ | Branch (86:11): [True: 36, False: 2.05k] ------------------ 87| 36| return 0; 88| 36| } // minimal bound checking 89| 2.05k| if ((uint8_t(data[pos + 1]) & 0b11000000) != 0b10000000) { ------------------ | Branch (89:11): [True: 38, False: 2.01k] ------------------ 90| 38| return 0; 91| 38| } 92| 2.01k| if ((uint8_t(data[pos + 2]) & 0b11000000) != 0b10000000) { ------------------ | Branch (92:11): [True: 33, False: 1.98k] ------------------ 93| 33| return 0; 94| 33| } 95| 1.98k| if ((uint8_t(data[pos + 3]) & 0b11000000) != 0b10000000) { ------------------ | Branch (95:11): [True: 45, False: 1.94k] ------------------ 96| 45| return 0; 97| 45| } 98| | 99| | // range check 100| 1.94k| uint32_t code_point = (leading_byte & 0b00000111) << 18 | 101| 1.94k| (uint8_t(data[pos + 1]) & 0b00111111) << 12 | 102| 1.94k| (uint8_t(data[pos + 2]) & 0b00111111) << 6 | 103| 1.94k| (uint8_t(data[pos + 3]) & 0b00111111); 104| 1.94k| if (code_point <= 0xffff || 0x10ffff < code_point) { ------------------ | Branch (104:11): [True: 18, False: 1.92k] | Branch (104:35): [True: 18, False: 1.90k] ------------------ 105| 36| return 0; 106| 36| } 107| 1.90k| *utf32_output++ = char32_t(code_point); 108| 1.90k| pos += 4; 109| 1.90k| } else { 110| 567| return 0; 111| 567| } 112| 30.8k| } 113| 18.7k| return utf32_output - start; 114| 19.8k|} _ZNK7simdutf8fallback14implementation13validate_utf8EPKcm: 37| 27.3k|implementation::validate_utf8(const char *buf, size_t len) const noexcept { 38| 27.3k| return scalar::utf8::validate(buf, len); 39| 27.3k|} _ZNK7simdutf8fallback14implementation25validate_utf8_with_errorsEPKcm: 44| 6.85k| const char *buf, size_t len) const noexcept { 45| 6.85k| return scalar::utf8::validate_with_errors(buf, len); 46| 6.85k|} _ZNK7simdutf8fallback14implementation16validate_utf16leEPKDsm: 77| 20.3k| size_t len) const noexcept { 78| 20.3k| return scalar::utf16::validate(buf, len); 79| 20.3k|} _ZNK7simdutf8fallback14implementation16validate_utf16beEPKDsm: 85| 20.3k| size_t len) const noexcept { 86| 20.3k| return scalar::utf16::validate(buf, len); 87| 20.3k|} _ZNK7simdutf8fallback14implementation28validate_utf16le_with_errorsEPKDsm: 90| 6.85k| const char16_t *buf, size_t len) const noexcept { 91| 6.85k| return scalar::utf16::validate_with_errors(buf, len); 92| 6.85k|} _ZNK7simdutf8fallback14implementation28validate_utf16be_with_errorsEPKDsm: 95| 6.85k| const char16_t *buf, size_t len) const noexcept { 96| 6.85k| return scalar::utf16::validate_with_errors(buf, len); 97| 6.85k|} _ZNK7simdutf8fallback14implementation14validate_utf32EPKDim: 114| 13.5k|implementation::validate_utf32(const char32_t *buf, size_t len) const noexcept { 115| 13.5k| return scalar::utf32::validate(buf, len); 116| 13.5k|} _ZNK7simdutf8fallback14implementation22convert_latin1_to_utf8EPKcmPc: 128| 7.00k| const char *buf, size_t len, char *utf8_output) const noexcept { 129| 7.00k| return scalar::latin1_to_utf8::convert(buf, len, utf8_output); 130| 7.00k|} _ZNK7simdutf8fallback14implementation25convert_latin1_to_utf16leEPKcmPDs: 135| 6.85k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 136| 6.85k| return scalar::latin1_to_utf16::convert(buf, len, 137| 6.85k| utf16_output); 138| 6.85k|} _ZNK7simdutf8fallback14implementation25convert_latin1_to_utf16beEPKcmPDs: 141| 6.85k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 142| 6.85k| return scalar::latin1_to_utf16::convert(buf, len, 143| 6.85k| utf16_output); 144| 6.85k|} _ZNK7simdutf8fallback14implementation23convert_latin1_to_utf32EPKcmPDi: 149| 6.85k| const char *buf, size_t len, char32_t *utf32_output) const noexcept { 150| 6.85k| return scalar::latin1_to_utf32::convert(buf, len, utf32_output); 151| 6.85k|} _ZNK7simdutf8fallback14implementation22convert_utf8_to_latin1EPKcmPc: 156| 13.7k| const char *buf, size_t len, char *latin1_output) const noexcept { 157| 13.7k| return scalar::utf8_to_latin1::convert(buf, len, latin1_output); 158| 13.7k|} _ZNK7simdutf8fallback14implementation23convert_utf8_to_utf16leEPKcmPDs: 173| 13.6k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 174| 13.6k| return scalar::utf8_to_utf16::convert(buf, len, 175| 13.6k| utf16_output); 176| 13.6k|} _ZNK7simdutf8fallback14implementation23convert_utf8_to_utf16beEPKcmPDs: 179| 13.6k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 180| 13.6k| return scalar::utf8_to_utf16::convert(buf, len, 181| 13.6k| utf16_output); 182| 13.6k|} _ZNK7simdutf8fallback14implementation21convert_utf8_to_utf32EPKcmPDi: 211| 6.85k| const char *buf, size_t len, char32_t *utf32_output) const noexcept { 212| 6.85k| return scalar::utf8_to_utf32::convert(buf, len, utf32_output); 213| 6.85k|} _ZNK7simdutf8fallback14implementation25convert_utf16le_to_latin1EPKDsmPc: 228| 6.85k| const char16_t *buf, size_t len, char *latin1_output) const noexcept { 229| 6.85k| return scalar::utf16_to_latin1::convert(buf, len, 230| 6.85k| latin1_output); 231| 6.85k|} _ZNK7simdutf8fallback14implementation25convert_utf16be_to_latin1EPKDsmPc: 234| 6.85k| const char16_t *buf, size_t len, char *latin1_output) const noexcept { 235| 6.85k| return scalar::utf16_to_latin1::convert(buf, len, 236| 6.85k| latin1_output); 237| 6.85k|} _ZNK7simdutf8fallback14implementation23convert_utf16le_to_utf8EPKDsmPc: 268| 13.5k| const char16_t *buf, size_t len, char *utf8_output) const noexcept { 269| 13.5k| return scalar::utf16_to_utf8::convert(buf, len, 270| 13.5k| utf8_output); 271| 13.5k|} _ZNK7simdutf8fallback14implementation23convert_utf16be_to_utf8EPKDsmPc: 274| 13.5k| const char16_t *buf, size_t len, char *utf8_output) const noexcept { 275| 13.5k| return scalar::utf16_to_utf8::convert(buf, len, utf8_output); 276| 13.5k|} _ZNK7simdutf8fallback14implementation23convert_utf32_to_latin1EPKDimPc: 305| 6.85k| const char32_t *buf, size_t len, char *latin1_output) const noexcept { 306| 6.85k| return scalar::utf32_to_latin1::convert(buf, len, latin1_output); 307| 6.85k|} _ZNK7simdutf8fallback14implementation21convert_utf32_to_utf8EPKDimPc: 322| 6.64k| const char32_t *buf, size_t len, char *utf8_output) const noexcept { 323| 6.64k| return scalar::utf32_to_utf8::convert(buf, len, utf8_output); 324| 6.64k|} _ZNK7simdutf8fallback14implementation23latin1_length_from_utf8EPKcm: 443| 13.7k| const char *buf, size_t len) const noexcept { 444| 13.7k| return scalar::utf8::count_code_points(buf, len); 445| 13.7k|} _ZNK7simdutf8fallback14implementation23utf8_length_from_latin1EPKcm: 450| 7.00k| const char *input, size_t length) const noexcept { 451| 7.00k| return scalar::latin1_to_utf8::utf8_length_from_latin1(input, length); 452| 7.00k|} _ZNK7simdutf8fallback14implementation24utf8_length_from_utf16leEPKDsm: 457| 13.5k| const char16_t *input, size_t length) const noexcept { 458| 13.5k| return scalar::utf16::utf8_length_from_utf16(input, 459| 13.5k| length); 460| 13.5k|} _ZNK7simdutf8fallback14implementation24utf8_length_from_utf16beEPKDsm: 463| 13.5k| const char16_t *input, size_t length) const noexcept { 464| 13.5k| return scalar::utf16::utf8_length_from_utf16(input, length); 465| 13.5k|} _ZNK7simdutf8fallback14implementation22utf16_length_from_utf8EPKcm: 483| 27.3k| const char *input, size_t length) const noexcept { 484| 27.3k| return scalar::utf8::utf16_length_from_utf8(input, length); 485| 27.3k|} _ZNK7simdutf8fallback14implementation22utf8_length_from_utf32EPKDim: 518| 6.64k| const char32_t *input, size_t length) const noexcept { 519| 6.64k| return scalar::utf32::utf8_length_from_utf32(input, length); 520| 6.64k|} _ZNK7simdutf8fallback14implementation22utf32_length_from_utf8EPKcm: 532| 6.85k| const char *input, size_t length) const noexcept { 533| 6.85k| return scalar::utf8::count_code_points(input, length); 534| 6.85k|} _ZNK7simdutf8fallback14implementation16base64_to_binaryEPKcmPcNS_14base64_optionsENS_27last_chunk_handling_optionsE: 541| 20.3k| last_chunk_handling_options last_chunk_options) const noexcept { 542| 20.3k| return simdutf::scalar::base64::base64_to_binary_details_impl( 543| 20.3k| input, length, output, options, last_chunk_options); 544| 20.3k|} _ZNK7simdutf8fallback14implementation16binary_to_base64EPKcmPcNS_14base64_optionsE: 569| 20.1k| base64_options options) const noexcept { 570| 20.1k| return scalar::base64::tail_encode_base64(output, input, length, options); 571| 20.1k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_16base6422compress_decode_base64ILb0ELb0ELb0EcEENS_11full_resultEPcPKT2_mNS_14base64_optionsENS_27last_chunk_handling_optionsE: 45| 78.5k| last_chunk_handling_options last_chunk_options) { 46| 78.5k| const uint8_t *to_base64 = 47| 78.5k| default_or_url ? tables::base64::to_base64_default_or_url_value ------------------ | Branch (47:7): [Folded, False: 78.5k] ------------------ 48| 78.5k| : (base64_url ? tables::base64::to_base64_url_value ------------------ | Branch (48:25): [Folded, False: 78.5k] ------------------ 49| 78.5k| : tables::base64::to_base64_value); 50| 78.5k| auto ri = 51| 78.5k| ignore_garbage ------------------ | Branch (51:7): [Folded, False: 78.5k] ------------------ 52| 78.5k| ? simdutf::scalar::base64::reduced_input{0, srclen, srclen, srclen} 53| 78.5k| : simdutf::scalar::base64::find_end(src, srclen, options); 54| 78.5k| size_t equallocation = ri.equallocation; 55| 78.5k| size_t equalsigns = ri.equalsigns; 56| 78.5k| srclen = ri.srclen; 57| 78.5k| size_t full_input_length = ri.full_input_length; 58| 78.5k| if (srclen == 0) { ------------------ | Branch (58:7): [True: 70.2k, False: 8.29k] ------------------ 59| 70.2k| if (!ignore_garbage && equalsigns > 0) { ------------------ | Branch (59:9): [True: 70.2k, Folded] | Branch (59:28): [True: 22, False: 70.2k] ------------------ 60| 22| return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; 61| 22| } 62| 70.2k| return {SUCCESS, full_input_length, 0}; 63| 70.2k| } 64| 8.29k| const chartype *const srcinit = src; 65| 8.29k| const char *const dstinit = dst; 66| 8.29k| const chartype *srcend = src + srclen; 67| | 68| 8.29k| constexpr size_t block_size = 6; 69| 8.29k| static_assert(block_size >= 2, "block_size must be at least two"); 70| 8.29k| char buffer[block_size * 64]; 71| 8.29k| char *bufferptr = buffer; 72| | // A wide block store writes 4 bytes past its 48. That is safe when the next 73| | // store starts 48 bytes later. Pairs of clean blocks wide-store the first 74| | // and end the second on byte 48. Anything shorter, or a block that is not 75| | // clean, uses the exact store. The pair attempt is only worth it on a long 76| | // input: a short one often fails the first block and then repeats that work. 77| 8.29k| if (srclen >= 512) { ------------------ | Branch (77:7): [True: 1.88k, False: 6.41k] ------------------ 78| 1.88k| const chartype *const srcend128 = src + srclen - 128; 79| 6.73k| while (bufferptr == buffer && src <= srcend128) { ------------------ | Branch (79:12): [True: 6.73k, False: 0] | Branch (79:35): [True: 6.14k, False: 592] ------------------ 80| 6.14k| block64 b0(src); 81| 6.14k| uint64_t e0 = 0; 82| 6.14k| const uint64_t m0 = 83| 6.14k| b0.to_base64_mask(&e0); 84| 6.14k| if ((!ignore_garbage && e0) || m0 != 0) { ------------------ | Branch (84:12): [True: 6.14k, Folded] | Branch (84:31): [True: 467, False: 5.67k] | Branch (84:38): [True: 349, False: 5.33k] ------------------ 85| 816| break; 86| 816| } 87| 5.33k| block64 b1(src + 64); 88| 5.33k| uint64_t e1 = 0; 89| 5.33k| const uint64_t m1 = 90| 5.33k| b1.to_base64_mask(&e1); 91| 5.33k| if ((!ignore_garbage && e1) || m1 != 0) { ------------------ | Branch (91:12): [True: 5.33k, Folded] | Branch (91:31): [True: 229, False: 5.10k] | Branch (91:38): [True: 248, False: 4.85k] ------------------ 92| 477| break; 93| 477| } 94| 4.85k| b0.base64_decode_block(dst); 95| 4.85k| b1.template base64_decode_block(dst + 48); 96| 4.85k| src += 128; 97| 4.85k| dst += 96; 98| 4.85k| } 99| 1.88k| } 100| 8.29k| if (srclen >= 64) { ------------------ | Branch (100:7): [True: 4.58k, False: 3.71k] ------------------ 101| 4.58k| const chartype *const srcend64 = srcinit + srclen - 64; 102| 11.7k| while (src <= srcend64) { ------------------ | Branch (102:12): [True: 9.05k, False: 2.69k] ------------------ 103| 9.05k| block64 b(src); 104| 9.05k| uint64_t error = 0; 105| 9.05k| const uint64_t badcharmask = 106| 9.05k| b.to_base64_mask(&error); 107| 9.05k| if (!ignore_garbage && error) { ------------------ | Branch (107:11): [True: 9.05k, Folded] | Branch (107:30): [True: 1.89k, False: 7.15k] ------------------ 108| 1.89k| const size_t error_offset = trailing_zeroes(error); 109| 1.89k| return {error_code::INVALID_BASE64_CHARACTER, 110| 1.89k| size_t(src - srcinit + error_offset), size_t(dst - dstinit)}; 111| 1.89k| } 112| 7.15k| if (ignore_garbage && badcharmask != 0) { ------------------ | Branch (112:11): [Folded, False: 7.15k] | Branch (112:29): [True: 0, False: 0] ------------------ 113| 0| const uint64_t equals = block64(src).equal_sign_mask(); 114| 0| if (equals != 0) { ------------------ | Branch (114:13): [True: 0, False: 0] ------------------ 115| 0| const size_t offset = trailing_zeroes(equals); 116| 0| equallocation = size_t(src - srcinit) + offset; 117| 0| equalsigns = 1; 118| 0| srclen = equallocation; 119| 0| full_input_length = equallocation + 1; 120| 0| if (srclen == 0) { ------------------ | Branch (120:15): [True: 0, False: 0] ------------------ 121| 0| return {SUCCESS, full_input_length, 0}; 122| 0| } 123| 0| srcend = srcinit + srclen; 124| 0| src = srcend; 125| 0| bufferptr += b.compress_block(badcharmask | (~uint64_t(0) << offset), 126| 0| bufferptr); 127| 0| break; 128| 0| } 129| 0| } 130| 7.15k| src += 64; 131| 7.15k| if (badcharmask != 0) { ------------------ | Branch (131:11): [True: 2.81k, False: 4.34k] ------------------ 132| 2.81k| bufferptr += b.compress_block(badcharmask, bufferptr); 133| 4.34k| } else if (bufferptr != buffer) { ------------------ | Branch (133:18): [True: 337, False: 4.00k] ------------------ 134| 337| b.copy_block(bufferptr); 135| 337| bufferptr += 64; 136| 4.00k| } else { 137| 4.00k| b.template base64_decode_block(dst); 138| 4.00k| dst += 48; 139| 4.00k| } 140| 7.15k| if (bufferptr >= (block_size - 1) * 64 + buffer) { ------------------ | Branch (140:11): [True: 228, False: 6.93k] ------------------ 141| 1.14k| for (size_t i = 0; i < (block_size - 2); i++) { ------------------ | Branch (141:28): [True: 912, False: 228] ------------------ 142| 912| base64_decode_block(dst, buffer + i * 64); 143| 912| dst += 48; 144| 912| } 145| 228| base64_decode_block(dst, buffer + (block_size - 2) * 64); 146| 228| dst += 48; 147| 228| std::memcpy(buffer, buffer + (block_size - 1) * 64, 148| 228| 64); // 64 might be too much 149| 228| bufferptr -= (block_size - 1) * 64; 150| 228| } 151| 7.15k| } 152| 4.58k| } 153| | 154| 6.40k| if (ignore_garbage && equalsigns == 0) { ------------------ | Branch (154:7): [Folded, False: 6.40k] | Branch (154:25): [True: 0, False: 0] ------------------ 155| 0| const chartype *equal = src; 156| 0| while (equal < srcend && *equal != '=') { ------------------ | Branch (156:12): [True: 0, False: 0] | Branch (156:30): [True: 0, False: 0] ------------------ 157| 0| equal++; 158| 0| } 159| 0| if (equal < srcend) { ------------------ | Branch (159:9): [True: 0, False: 0] ------------------ 160| 0| equallocation = size_t(equal - srcinit); 161| 0| equalsigns = 1; 162| 0| srclen = equallocation; 163| 0| full_input_length = equallocation + 1; 164| 0| if (srclen == 0) { ------------------ | Branch (164:11): [True: 0, False: 0] ------------------ 165| 0| return {SUCCESS, full_input_length, 0}; 166| 0| } 167| 0| srcend = equal; 168| 0| } 169| 0| } 170| 6.40k| char *buffer_start = buffer; 171| | // Optimization note: if this is almost full, then it is worth our 172| | // time, otherwise, we should just decode directly. 173| 6.40k| int last_block = (int)((bufferptr - buffer_start) % 64); 174| 6.40k| if (last_block != 0 && srcend - src + last_block >= 64) { ------------------ | Branch (174:7): [True: 625, False: 5.78k] | Branch (174:26): [True: 105, False: 520] ------------------ 175| | 176| 1.40k| while ((bufferptr - buffer_start) % 64 != 0 && src < srcend) { ------------------ | Branch (176:12): [True: 1.33k, False: 72] | Branch (176:52): [True: 1.31k, False: 23] ------------------ 177| 1.31k| uint8_t val = to_base64[uint8_t(*src)]; 178| 1.31k| *bufferptr = char(val); 179| 1.31k| if (!ignore_garbage && ------------------ | Branch (179:11): [True: 1.31k, Folded] ------------------ 180| 1.31k| (!scalar::base64::is_eight_byte(*src) || val > 64)) { ------------------ | Branch (180:12): [True: 0, False: 1.31k] | Branch (180:52): [True: 10, False: 1.30k] ------------------ 181| 10| return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), 182| 10| size_t(dst - dstinit)}; 183| 10| } 184| 1.30k| bufferptr += (val <= 63 && scalar::base64::is_eight_byte(*src)); ------------------ | Branch (184:21): [True: 665, False: 639] | Branch (184:34): [True: 665, False: 0] ------------------ 185| 1.30k| src++; 186| 1.30k| } 187| 105| } 188| | 189| 6.94k| for (; buffer_start + 64 <= bufferptr; buffer_start += 64) { ------------------ | Branch (189:10): [True: 553, False: 6.39k] ------------------ 190| | // 8 payload bytes still in the buffer become 6 output bytes, which cover 191| | // the 4-byte tail of a wide store. Fewer than that, and the tail can 192| | // stick out past the real end. 193| 553| const size_t after = size_t(bufferptr - (buffer_start + 64)); 194| 553| if (after >= 8) { ------------------ | Branch (194:9): [True: 468, False: 85] ------------------ 195| 468| base64_decode_block(dst, buffer_start); 196| 468| } else { 197| 85| base64_decode_block(dst, buffer_start); 198| 85| } 199| 553| dst += 48; 200| 553| } 201| 6.39k| if ((bufferptr - buffer_start) % 64 != 0) { ------------------ | Branch (201:7): [True: 543, False: 5.85k] ------------------ 202| 5.13k| while (buffer_start + 4 < bufferptr) { ------------------ | Branch (202:12): [True: 4.58k, False: 543] ------------------ 203| 4.58k| uint32_t triple = ((uint32_t(uint8_t(buffer_start[0])) << 3 * 6) + 204| 4.58k| (uint32_t(uint8_t(buffer_start[1])) << 2 * 6) + 205| 4.58k| (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + 206| 4.58k| (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) 207| 4.58k| << 8; 208| 4.58k|#if !SIMDUTF_IS_BIG_ENDIAN 209| 4.58k| triple = scalar::u32_swap_bytes(triple); 210| 4.58k|#endif 211| 4.58k| std::memcpy(dst, &triple, 3); 212| | 213| 4.58k| dst += 3; 214| 4.58k| buffer_start += 4; 215| 4.58k| } 216| 543| if (buffer_start + 4 <= bufferptr) { ------------------ | Branch (216:9): [True: 152, False: 391] ------------------ 217| 152| uint32_t triple = ((uint32_t(uint8_t(buffer_start[0])) << 3 * 6) + 218| 152| (uint32_t(uint8_t(buffer_start[1])) << 2 * 6) + 219| 152| (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + 220| 152| (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) 221| 152| << 8; 222| 152|#if !SIMDUTF_IS_BIG_ENDIAN 223| 152| triple = scalar::u32_swap_bytes(triple); 224| 152|#endif 225| 152| std::memcpy(dst, &triple, 3); 226| | 227| 152| dst += 3; 228| 152| buffer_start += 4; 229| 152| } 230| | // we may have 1, 2 or 3 bytes left and we need to decode them so let us 231| | // backtrack 232| 543| int leftover = int(bufferptr - buffer_start); 233| 1.40k| while (leftover > 0) { ------------------ | Branch (233:12): [True: 860, False: 543] ------------------ 234| 860| if (!ignore_garbage) { ------------------ | Branch (234:11): [True: 860, Folded] ------------------ 235| 4.47k| while (to_base64[uint8_t(*(src - 1))] == 64) { ------------------ | Branch (235:16): [True: 3.61k, False: 860] ------------------ 236| 3.61k| src--; 237| 3.61k| } 238| 860| } else { 239| 0| while (!scalar::base64::is_eight_byte(*(src - 1)) || ------------------ | Branch (239:16): [True: 0, False: 0] ------------------ 240| 0| to_base64[uint8_t(*(src - 1))] >= 64) { ------------------ | Branch (240:16): [True: 0, False: 0] ------------------ 241| 0| src--; 242| 0| } 243| 0| } 244| 860| src--; 245| 860| leftover--; 246| 860| } 247| 543| } 248| 6.39k| if (src < srcend + equalsigns) { ------------------ | Branch (248:7): [True: 5.98k, False: 408] ------------------ 249| 5.98k| full_result r = scalar::base64::base64_tail_decode( 250| 5.98k| dst, src, srcend - src, equalsigns, options, last_chunk_options); 251| 5.98k| r = scalar::base64::patch_tail_result( 252| 5.98k| r, size_t(src - srcinit), size_t(dst - dstinit), equallocation, 253| 5.98k| full_input_length, last_chunk_options); 254| | // When is_partial(last_chunk_options) is true, we must either end with 255| | // the end of the stream (beyond whitespace) or right after a non-ignorable 256| | // character or at the very beginning of the stream. 257| | // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 258| 5.98k| if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && ------------------ | Branch (258:9): [True: 2.24k, False: 3.74k] | Branch (258:43): [True: 1.74k, False: 507] ------------------ 259| 1.74k| r.input_count < full_input_length) { ------------------ | Branch (259:9): [True: 304, False: 1.43k] ------------------ 260| | // First check if we can extend the input to the end of the stream 261| 1.90k| while (r.input_count < full_input_length && ------------------ | Branch (261:14): [True: 1.87k, False: 34] ------------------ 262| 1.87k| base64_ignorable(*(srcinit + r.input_count), options)) { ------------------ | Branch (262:14): [True: 1.60k, False: 270] ------------------ 263| 1.60k| r.input_count++; 264| 1.60k| } 265| | // If we are still not at the end of the stream, then we must backtrack 266| | // to the last non-ignorable character. 267| 304| if (r.input_count < full_input_length) { ------------------ | Branch (267:11): [True: 270, False: 34] ------------------ 268| 1.91k| while (r.input_count > 0 && ------------------ | Branch (268:16): [True: 1.82k, False: 90] ------------------ 269| 1.82k| base64_ignorable(*(srcinit + r.input_count - 1), options)) { ------------------ | Branch (269:16): [True: 1.64k, False: 180] ------------------ 270| 1.64k| r.input_count--; 271| 1.64k| } 272| 270| } 273| 304| } 274| 5.98k| return r; 275| 5.98k| } 276| 408| if (!ignore_garbage && equalsigns > 0) { ------------------ | Branch (276:7): [True: 408, Folded] | Branch (276:26): [True: 0, False: 408] ------------------ 277| 0| if ((size_t(dst - dstinit) % 3 == 0) || ------------------ | Branch (277:9): [True: 0, False: 0] ------------------ 278| 0| ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { ------------------ | Branch (278:9): [True: 0, False: 0] ------------------ 279| 0| return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), 280| 0| true}; 281| 0| } 282| 0| } 283| 408| return {SUCCESS, srclen, size_t(dst - dstinit)}; 284| 408|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_16base6422compress_decode_base64ILb0ELb0ELb0EcEENS_11full_resultEPcPKT2_mNS_14base64_optionsENS_27last_chunk_handling_optionsE: 45| 19.6k| last_chunk_handling_options last_chunk_options) { 46| 19.6k| const uint8_t *to_base64 = 47| 19.6k| default_or_url ? tables::base64::to_base64_default_or_url_value ------------------ | Branch (47:7): [Folded, False: 19.6k] ------------------ 48| 19.6k| : (base64_url ? tables::base64::to_base64_url_value ------------------ | Branch (48:25): [Folded, False: 19.6k] ------------------ 49| 19.6k| : tables::base64::to_base64_value); 50| 19.6k| auto ri = 51| 19.6k| ignore_garbage ------------------ | Branch (51:7): [Folded, False: 19.6k] ------------------ 52| 19.6k| ? simdutf::scalar::base64::reduced_input{0, srclen, srclen, srclen} 53| 19.6k| : simdutf::scalar::base64::find_end(src, srclen, options); 54| 19.6k| size_t equallocation = ri.equallocation; 55| 19.6k| size_t equalsigns = ri.equalsigns; 56| 19.6k| srclen = ri.srclen; 57| 19.6k| size_t full_input_length = ri.full_input_length; 58| 19.6k| if (srclen == 0) { ------------------ | Branch (58:7): [True: 17.7k, False: 1.93k] ------------------ 59| 17.7k| if (!ignore_garbage && equalsigns > 0) { ------------------ | Branch (59:9): [True: 17.7k, Folded] | Branch (59:28): [True: 5, False: 17.6k] ------------------ 60| 5| return {INVALID_BASE64_CHARACTER, equallocation, 0, true}; 61| 5| } 62| 17.6k| return {SUCCESS, full_input_length, 0}; 63| 17.7k| } 64| 1.93k| const chartype *const srcinit = src; 65| 1.93k| const char *const dstinit = dst; 66| 1.93k| const chartype *srcend = src + srclen; 67| | 68| 1.93k| constexpr size_t block_size = 6; 69| 1.93k| static_assert(block_size >= 2, "block_size must be at least two"); 70| 1.93k| char buffer[block_size * 64]; 71| 1.93k| char *bufferptr = buffer; 72| | // A wide block store writes 4 bytes past its 48. That is safe when the next 73| | // store starts 48 bytes later. Pairs of clean blocks wide-store the first 74| | // and end the second on byte 48. Anything shorter, or a block that is not 75| | // clean, uses the exact store. The pair attempt is only worth it on a long 76| | // input: a short one often fails the first block and then repeats that work. 77| 1.93k| if (srclen >= 512) { ------------------ | Branch (77:7): [True: 547, False: 1.38k] ------------------ 78| 547| const chartype *const srcend128 = src + srclen - 128; 79| 1.71k| while (bufferptr == buffer && src <= srcend128) { ------------------ | Branch (79:12): [True: 1.71k, False: 0] | Branch (79:35): [True: 1.57k, False: 135] ------------------ 80| 1.57k| block64 b0(src); 81| 1.57k| uint64_t e0 = 0; 82| 1.57k| const uint64_t m0 = 83| 1.57k| b0.to_base64_mask(&e0); 84| 1.57k| if ((!ignore_garbage && e0) || m0 != 0) { ------------------ | Branch (84:12): [True: 1.57k, Folded] | Branch (84:31): [True: 145, False: 1.43k] | Branch (84:38): [True: 120, False: 1.31k] ------------------ 85| 265| break; 86| 265| } 87| 1.31k| block64 b1(src + 64); 88| 1.31k| uint64_t e1 = 0; 89| 1.31k| const uint64_t m1 = 90| 1.31k| b1.to_base64_mask(&e1); 91| 1.31k| if ((!ignore_garbage && e1) || m1 != 0) { ------------------ | Branch (91:12): [True: 1.31k, Folded] | Branch (91:31): [True: 74, False: 1.23k] | Branch (91:38): [True: 73, False: 1.16k] ------------------ 92| 147| break; 93| 147| } 94| 1.16k| b0.base64_decode_block(dst); 95| 1.16k| b1.template base64_decode_block(dst + 48); 96| 1.16k| src += 128; 97| 1.16k| dst += 96; 98| 1.16k| } 99| 547| } 100| 1.93k| if (srclen >= 64) { ------------------ | Branch (100:7): [True: 1.11k, False: 819] ------------------ 101| 1.11k| const chartype *const srcend64 = srcinit + srclen - 64; 102| 2.85k| while (src <= srcend64) { ------------------ | Branch (102:12): [True: 2.31k, False: 547] ------------------ 103| 2.31k| block64 b(src); 104| 2.31k| uint64_t error = 0; 105| 2.31k| const uint64_t badcharmask = 106| 2.31k| b.to_base64_mask(&error); 107| 2.31k| if (!ignore_garbage && error) { ------------------ | Branch (107:11): [True: 2.31k, Folded] | Branch (107:30): [True: 570, False: 1.74k] ------------------ 108| 570| const size_t error_offset = trailing_zeroes(error); 109| 570| return {error_code::INVALID_BASE64_CHARACTER, 110| 570| size_t(src - srcinit + error_offset), size_t(dst - dstinit)}; 111| 570| } 112| 1.74k| if (ignore_garbage && badcharmask != 0) { ------------------ | Branch (112:11): [Folded, False: 1.74k] | Branch (112:29): [True: 0, False: 0] ------------------ 113| 0| const uint64_t equals = block64(src).equal_sign_mask(); 114| 0| if (equals != 0) { ------------------ | Branch (114:13): [True: 0, False: 0] ------------------ 115| 0| const size_t offset = trailing_zeroes(equals); 116| 0| equallocation = size_t(src - srcinit) + offset; 117| 0| equalsigns = 1; 118| 0| srclen = equallocation; 119| 0| full_input_length = equallocation + 1; 120| 0| if (srclen == 0) { ------------------ | Branch (120:15): [True: 0, False: 0] ------------------ 121| 0| return {SUCCESS, full_input_length, 0}; 122| 0| } 123| 0| srcend = srcinit + srclen; 124| 0| src = srcend; 125| 0| bufferptr += b.compress_block(badcharmask | (~uint64_t(0) << offset), 126| 0| bufferptr); 127| 0| break; 128| 0| } 129| 0| } 130| 1.74k| src += 64; 131| 1.74k| if (badcharmask != 0) { ------------------ | Branch (131:11): [True: 981, False: 761] ------------------ 132| 981| bufferptr += b.compress_block(badcharmask, bufferptr); 133| 981| } else if (bufferptr != buffer) { ------------------ | Branch (133:18): [True: 88, False: 673] ------------------ 134| 88| b.copy_block(bufferptr); 135| 88| bufferptr += 64; 136| 673| } else { 137| 673| b.template base64_decode_block(dst); 138| 673| dst += 48; 139| 673| } 140| 1.74k| if (bufferptr >= (block_size - 1) * 64 + buffer) { ------------------ | Branch (140:11): [True: 72, False: 1.67k] ------------------ 141| 360| for (size_t i = 0; i < (block_size - 2); i++) { ------------------ | Branch (141:28): [True: 288, False: 72] ------------------ 142| 288| base64_decode_block(dst, buffer + i * 64); 143| 288| dst += 48; 144| 288| } 145| 72| base64_decode_block(dst, buffer + (block_size - 2) * 64); 146| 72| dst += 48; 147| 72| std::memcpy(buffer, buffer + (block_size - 1) * 64, 148| 72| 64); // 64 might be too much 149| 72| bufferptr -= (block_size - 1) * 64; 150| 72| } 151| 1.74k| } 152| 1.11k| } 153| | 154| 1.36k| if (ignore_garbage && equalsigns == 0) { ------------------ | Branch (154:7): [Folded, False: 1.36k] | Branch (154:25): [True: 0, False: 0] ------------------ 155| 0| const chartype *equal = src; 156| 0| while (equal < srcend && *equal != '=') { ------------------ | Branch (156:12): [True: 0, False: 0] | Branch (156:30): [True: 0, False: 0] ------------------ 157| 0| equal++; 158| 0| } 159| 0| if (equal < srcend) { ------------------ | Branch (159:9): [True: 0, False: 0] ------------------ 160| 0| equallocation = size_t(equal - srcinit); 161| 0| equalsigns = 1; 162| 0| srclen = equallocation; 163| 0| full_input_length = equallocation + 1; 164| 0| if (srclen == 0) { ------------------ | Branch (164:11): [True: 0, False: 0] ------------------ 165| 0| return {SUCCESS, full_input_length, 0}; 166| 0| } 167| 0| srcend = equal; 168| 0| } 169| 0| } 170| 1.36k| char *buffer_start = buffer; 171| | // Optimization note: if this is almost full, then it is worth our 172| | // time, otherwise, we should just decode directly. 173| 1.36k| int last_block = (int)((bufferptr - buffer_start) % 64); 174| 1.36k| if (last_block != 0 && srcend - src + last_block >= 64) { ------------------ | Branch (174:7): [True: 248, False: 1.11k] | Branch (174:26): [True: 33, False: 215] ------------------ 175| | 176| 353| while ((bufferptr - buffer_start) % 64 != 0 && src < srcend) { ------------------ | Branch (176:12): [True: 334, False: 19] | Branch (176:52): [True: 323, False: 11] ------------------ 177| 323| uint8_t val = to_base64[uint8_t(*src)]; 178| 323| *bufferptr = char(val); 179| 323| if (!ignore_garbage && ------------------ | Branch (179:11): [True: 323, Folded] ------------------ 180| 323| (!scalar::base64::is_eight_byte(*src) || val > 64)) { ------------------ | Branch (180:12): [True: 0, False: 323] | Branch (180:52): [True: 3, False: 320] ------------------ 181| 3| return {error_code::INVALID_BASE64_CHARACTER, size_t(src - srcinit), 182| 3| size_t(dst - dstinit)}; 183| 3| } 184| 320| bufferptr += (val <= 63 && scalar::base64::is_eight_byte(*src)); ------------------ | Branch (184:21): [True: 136, False: 184] | Branch (184:34): [True: 136, False: 0] ------------------ 185| 320| src++; 186| 320| } 187| 33| } 188| | 189| 1.53k| for (; buffer_start + 64 <= bufferptr; buffer_start += 64) { ------------------ | Branch (189:10): [True: 173, False: 1.36k] ------------------ 190| | // 8 payload bytes still in the buffer become 6 output bytes, which cover 191| | // the 4-byte tail of a wide store. Fewer than that, and the tail can 192| | // stick out past the real end. 193| 173| const size_t after = size_t(bufferptr - (buffer_start + 64)); 194| 173| if (after >= 8) { ------------------ | Branch (194:9): [True: 147, False: 26] ------------------ 195| 147| base64_decode_block(dst, buffer_start); 196| 147| } else { 197| 26| base64_decode_block(dst, buffer_start); 198| 26| } 199| 173| dst += 48; 200| 173| } 201| 1.36k| if ((bufferptr - buffer_start) % 64 != 0) { ------------------ | Branch (201:7): [True: 226, False: 1.13k] ------------------ 202| 1.93k| while (buffer_start + 4 < bufferptr) { ------------------ | Branch (202:12): [True: 1.70k, False: 226] ------------------ 203| 1.70k| uint32_t triple = ((uint32_t(uint8_t(buffer_start[0])) << 3 * 6) + 204| 1.70k| (uint32_t(uint8_t(buffer_start[1])) << 2 * 6) + 205| 1.70k| (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + 206| 1.70k| (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) 207| 1.70k| << 8; 208| 1.70k|#if !SIMDUTF_IS_BIG_ENDIAN 209| 1.70k| triple = scalar::u32_swap_bytes(triple); 210| 1.70k|#endif 211| 1.70k| std::memcpy(dst, &triple, 3); 212| | 213| 1.70k| dst += 3; 214| 1.70k| buffer_start += 4; 215| 1.70k| } 216| 226| if (buffer_start + 4 <= bufferptr) { ------------------ | Branch (216:9): [True: 54, False: 172] ------------------ 217| 54| uint32_t triple = ((uint32_t(uint8_t(buffer_start[0])) << 3 * 6) + 218| 54| (uint32_t(uint8_t(buffer_start[1])) << 2 * 6) + 219| 54| (uint32_t(uint8_t(buffer_start[2])) << 1 * 6) + 220| 54| (uint32_t(uint8_t(buffer_start[3])) << 0 * 6)) 221| 54| << 8; 222| 54|#if !SIMDUTF_IS_BIG_ENDIAN 223| 54| triple = scalar::u32_swap_bytes(triple); 224| 54|#endif 225| 54| std::memcpy(dst, &triple, 3); 226| | 227| 54| dst += 3; 228| 54| buffer_start += 4; 229| 54| } 230| | // we may have 1, 2 or 3 bytes left and we need to decode them so let us 231| | // backtrack 232| 226| int leftover = int(bufferptr - buffer_start); 233| 577| while (leftover > 0) { ------------------ | Branch (233:12): [True: 351, False: 226] ------------------ 234| 351| if (!ignore_garbage) { ------------------ | Branch (234:11): [True: 351, Folded] ------------------ 235| 3.43k| while (to_base64[uint8_t(*(src - 1))] == 64) { ------------------ | Branch (235:16): [True: 3.08k, False: 351] ------------------ 236| 3.08k| src--; 237| 3.08k| } 238| 351| } else { 239| 0| while (!scalar::base64::is_eight_byte(*(src - 1)) || ------------------ | Branch (239:16): [True: 0, False: 0] ------------------ 240| 0| to_base64[uint8_t(*(src - 1))] >= 64) { ------------------ | Branch (240:16): [True: 0, False: 0] ------------------ 241| 0| src--; 242| 0| } 243| 0| } 244| 351| src--; 245| 351| leftover--; 246| 351| } 247| 226| } 248| 1.36k| if (src < srcend + equalsigns) { ------------------ | Branch (248:7): [True: 1.23k, False: 125] ------------------ 249| 1.23k| full_result r = scalar::base64::base64_tail_decode( 250| 1.23k| dst, src, srcend - src, equalsigns, options, last_chunk_options); 251| 1.23k| r = scalar::base64::patch_tail_result( 252| 1.23k| r, size_t(src - srcinit), size_t(dst - dstinit), equallocation, 253| 1.23k| full_input_length, last_chunk_options); 254| | // When is_partial(last_chunk_options) is true, we must either end with 255| | // the end of the stream (beyond whitespace) or right after a non-ignorable 256| | // character or at the very beginning of the stream. 257| | // See https://tc39.es/proposal-arraybuffer-base64/spec/#sec-frombase64 258| 1.23k| if (is_partial(last_chunk_options) && r.error == error_code::SUCCESS && ------------------ | Branch (258:9): [True: 0, False: 1.23k] | Branch (258:43): [True: 0, False: 0] ------------------ 259| 0| r.input_count < full_input_length) { ------------------ | Branch (259:9): [True: 0, False: 0] ------------------ 260| | // First check if we can extend the input to the end of the stream 261| 0| while (r.input_count < full_input_length && ------------------ | Branch (261:14): [True: 0, False: 0] ------------------ 262| 0| base64_ignorable(*(srcinit + r.input_count), options)) { ------------------ | Branch (262:14): [True: 0, False: 0] ------------------ 263| 0| r.input_count++; 264| 0| } 265| | // If we are still not at the end of the stream, then we must backtrack 266| | // to the last non-ignorable character. 267| 0| if (r.input_count < full_input_length) { ------------------ | Branch (267:11): [True: 0, False: 0] ------------------ 268| 0| while (r.input_count > 0 && ------------------ | Branch (268:16): [True: 0, False: 0] ------------------ 269| 0| base64_ignorable(*(srcinit + r.input_count - 1), options)) { ------------------ | Branch (269:16): [True: 0, False: 0] ------------------ 270| 0| r.input_count--; 271| 0| } 272| 0| } 273| 0| } 274| 1.23k| return r; 275| 1.23k| } 276| 125| if (!ignore_garbage && equalsigns > 0) { ------------------ | Branch (276:7): [True: 125, Folded] | Branch (276:26): [True: 0, False: 125] ------------------ 277| 0| if ((size_t(dst - dstinit) % 3 == 0) || ------------------ | Branch (277:9): [True: 0, False: 0] ------------------ 278| 0| ((size_t(dst - dstinit) % 3) + 1 + equalsigns != 4)) { ------------------ | Branch (278:9): [True: 0, False: 0] ------------------ 279| 0| return {INVALID_BASE64_CHARACTER, equallocation, size_t(dst - dstinit), 280| 0| true}; 281| 0| } 282| 0| } 283| 125| return {SUCCESS, srclen, size_t(dst - dstinit)}; 284| 125|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_116buf_block_readerILm64EEC2EPKhm: 81| 34.0k| : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, ------------------ | Branch (81:42): [True: 30.9k, False: 3.09k] ------------------ 82| 34.0k| idx{0} {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_116buf_block_readerILm64EE14has_full_blockEv: 90| 44.6k|simdutf_really_inline bool buf_block_reader::has_full_block() const { 91| 44.6k| return idx < lenminusstep; 92| 44.6k|} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_116buf_block_readerILm64EE10full_blockEv: 96| 10.8k|buf_block_reader::full_block() const { 97| 10.8k| return &buf[idx]; 98| 10.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_116buf_block_readerILm64EE7advanceEv: 114| 10.5k|simdutf_really_inline void buf_block_reader::advance() { 115| 10.5k| idx += STEP_SIZE; 116| 10.5k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_116buf_block_readerILm64EE11block_indexEv: 85| 33.8k|simdutf_really_inline size_t buf_block_reader::block_index() { 86| 33.8k| return idx; 87| 33.8k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_116buf_block_readerILm64EEC2EPKhm: 81| 34.0k| : buf{_buf}, len{_len}, lenminusstep{len < STEP_SIZE ? 0 : len - STEP_SIZE}, ------------------ | Branch (81:42): [True: 31.3k, False: 2.69k] ------------------ 82| 34.0k| idx{0} {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_116buf_block_readerILm64EE14has_full_blockEv: 90| 46.2k|simdutf_really_inline bool buf_block_reader::has_full_block() const { 91| 46.2k| return idx < lenminusstep; 92| 46.2k|} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_116buf_block_readerILm64EE10full_blockEv: 96| 12.4k|buf_block_reader::full_block() const { 97| 12.4k| return &buf[idx]; 98| 12.4k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_116buf_block_readerILm64EE7advanceEv: 114| 12.2k|simdutf_really_inline void buf_block_reader::advance() { 115| 12.2k| idx += STEP_SIZE; 116| 12.2k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_116buf_block_readerILm64EE11block_indexEv: 85| 33.7k|simdutf_really_inline size_t buf_block_reader::block_index() { 86| 33.7k| return idx; 87| 33.7k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf1631utf8_length_from_utf16_bytemaskILNS_10endiannessE0EEEmPKDsm: 10| 12.8k| size_t size) { 11| 12.8k| size_t pos = 0; 12| | 13| 12.8k| using vector_u16 = simd16; 14| 12.8k| constexpr size_t N = vector_u16::ELEMENTS; 15| | 16| 12.8k| const auto one = vector_u16::splat(1); 17| | 18| 12.8k| auto v_count = vector_u16::zero(); 19| | 20| | // each char16 yields at least one byte 21| 12.8k| size_t count = size / N * N; 22| | 23| | // in a single iteration the increment is 0, 1 or 2, despite we have 24| | // three additions 25| 12.8k| constexpr size_t max_iterations = 65535 / 2; 26| 12.8k| size_t iteration = max_iterations; 27| | 28| 18.5k| for (; pos < size / N * N; pos += N) { ------------------ | Branch (28:10): [True: 5.70k, False: 12.8k] ------------------ 29| 5.70k| auto input = vector_u16::load(reinterpret_cast(in + pos)); 30| | if constexpr (!match_system(big_endian)) { 31| | input = input.swap_bytes(); 32| | } 33| | // 0xd800 .. 0xdbff - low surrogate 34| | // 0xdc00 .. 0xdfff - high surrogate 35| 5.70k| const auto is_surrogate = ((input & uint16_t(0xf800)) == uint16_t(0xd800)); 36| | 37| | // c0 - chars that yield 2- or 3-byte UTF-8 codes 38| 5.70k| const auto c0 = min(input & uint16_t(0xff80), one); 39| | 40| | // c1 - chars that yield 3-byte UTF-8 codes (including surrogates) 41| 5.70k| const auto c1 = min(input & uint16_t(0xf800), one); 42| | 43| | /* 44| | Explanation how the counting works. 45| | 46| | In the case of a non-surrogate character we count: 47| | * always 1 -- see how `count` is initialized above; 48| | * c0 = 1 if the current char yields 2 or 3 bytes; 49| | * c1 = 1 if the current char yields 3 bytes. 50| | 51| | Thus, we always have correct count for the current char: 52| | from 1, 2 or 3 bytes. 53| | 54| | A trickier part is how we count surrogate pairs. Whether 55| | we encounter a surrogate (low or high), we count it as 56| | 3 chars and then minus 1 (`is_surrogate` is -1 or 0). 57| | Each surrogate char yields 2. A surrogate pair, that 58| | is a low surrogate followed by a high one, yields 59| | the expected 4 bytes. 60| | 61| | It also correctly handles cases when low surrogate is 62| | processed by the this loop, but high surrogate is counted 63| | by the scalar procedure. The scalar procedure uses exactly 64| | the described approach, thanks to that for valid UTF-16 65| | strings it always count correctly. 66| | */ 67| 5.70k| v_count += c0; 68| 5.70k| v_count += c1; 69| 5.70k| v_count += vector_u16(is_surrogate); 70| | 71| 5.70k| iteration -= 1; 72| 5.70k| if (iteration == 0) { ------------------ | Branch (72:9): [True: 0, False: 5.70k] ------------------ 73| 0| count += v_count.sum(); 74| 0| v_count = vector_u16::zero(); 75| 0| iteration = max_iterations; 76| 0| } 77| 5.70k| } 78| | 79| 12.8k| if (iteration > 0) { ------------------ | Branch (79:7): [True: 12.8k, False: 0] ------------------ 80| 12.8k| count += v_count.sum(); 81| 12.8k| } 82| | 83| 12.8k| return count + scalar::utf16::utf8_length_from_utf16(in + pos, 84| 12.8k| size - pos); 85| 12.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf1631utf8_length_from_utf16_bytemaskILNS_10endiannessE1EEEmPKDsm: 10| 12.8k| size_t size) { 11| 12.8k| size_t pos = 0; 12| | 13| 12.8k| using vector_u16 = simd16; 14| 12.8k| constexpr size_t N = vector_u16::ELEMENTS; 15| | 16| 12.8k| const auto one = vector_u16::splat(1); 17| | 18| 12.8k| auto v_count = vector_u16::zero(); 19| | 20| | // each char16 yields at least one byte 21| 12.8k| size_t count = size / N * N; 22| | 23| | // in a single iteration the increment is 0, 1 or 2, despite we have 24| | // three additions 25| 12.8k| constexpr size_t max_iterations = 65535 / 2; 26| 12.8k| size_t iteration = max_iterations; 27| | 28| 18.2k| for (; pos < size / N * N; pos += N) { ------------------ | Branch (28:10): [True: 5.42k, False: 12.8k] ------------------ 29| 5.42k| auto input = vector_u16::load(reinterpret_cast(in + pos)); 30| 5.42k| if constexpr (!match_system(big_endian)) { 31| 5.42k| input = input.swap_bytes(); 32| 5.42k| } 33| | // 0xd800 .. 0xdbff - low surrogate 34| | // 0xdc00 .. 0xdfff - high surrogate 35| 5.42k| const auto is_surrogate = ((input & uint16_t(0xf800)) == uint16_t(0xd800)); 36| | 37| | // c0 - chars that yield 2- or 3-byte UTF-8 codes 38| 5.42k| const auto c0 = min(input & uint16_t(0xff80), one); 39| | 40| | // c1 - chars that yield 3-byte UTF-8 codes (including surrogates) 41| 5.42k| const auto c1 = min(input & uint16_t(0xf800), one); 42| | 43| | /* 44| | Explanation how the counting works. 45| | 46| | In the case of a non-surrogate character we count: 47| | * always 1 -- see how `count` is initialized above; 48| | * c0 = 1 if the current char yields 2 or 3 bytes; 49| | * c1 = 1 if the current char yields 3 bytes. 50| | 51| | Thus, we always have correct count for the current char: 52| | from 1, 2 or 3 bytes. 53| | 54| | A trickier part is how we count surrogate pairs. Whether 55| | we encounter a surrogate (low or high), we count it as 56| | 3 chars and then minus 1 (`is_surrogate` is -1 or 0). 57| | Each surrogate char yields 2. A surrogate pair, that 58| | is a low surrogate followed by a high one, yields 59| | the expected 4 bytes. 60| | 61| | It also correctly handles cases when low surrogate is 62| | processed by the this loop, but high surrogate is counted 63| | by the scalar procedure. The scalar procedure uses exactly 64| | the described approach, thanks to that for valid UTF-16 65| | strings it always count correctly. 66| | */ 67| 5.42k| v_count += c0; 68| 5.42k| v_count += c1; 69| 5.42k| v_count += vector_u16(is_surrogate); 70| | 71| 5.42k| iteration -= 1; 72| 5.42k| if (iteration == 0) { ------------------ | Branch (72:9): [True: 0, False: 5.42k] ------------------ 73| 0| count += v_count.sum(); 74| 0| v_count = vector_u16::zero(); 75| 0| iteration = max_iterations; 76| 0| } 77| 5.42k| } 78| | 79| 12.8k| if (iteration > 0) { ------------------ | Branch (79:7): [True: 12.8k, False: 0] ------------------ 80| 12.8k| count += v_count.sum(); 81| 12.8k| } 82| | 83| 12.8k| return count + scalar::utf16::utf8_length_from_utf16(in + pos, 84| 12.8k| size - pos); 85| 12.8k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf1631utf8_length_from_utf16_bytemaskILNS_10endiannessE0EEEmPKDsm: 10| 13.1k| size_t size) { 11| 13.1k| size_t pos = 0; 12| | 13| 13.1k| using vector_u16 = simd16; 14| 13.1k| constexpr size_t N = vector_u16::ELEMENTS; 15| | 16| 13.1k| const auto one = vector_u16::splat(1); 17| | 18| 13.1k| auto v_count = vector_u16::zero(); 19| | 20| | // each char16 yields at least one byte 21| 13.1k| size_t count = size / N * N; 22| | 23| | // in a single iteration the increment is 0, 1 or 2, despite we have 24| | // three additions 25| 13.1k| constexpr size_t max_iterations = 65535 / 2; 26| 13.1k| size_t iteration = max_iterations; 27| | 28| 28.3k| for (; pos < size / N * N; pos += N) { ------------------ | Branch (28:10): [True: 15.2k, False: 13.1k] ------------------ 29| 15.2k| auto input = vector_u16::load(reinterpret_cast(in + pos)); 30| | if constexpr (!match_system(big_endian)) { 31| | input = input.swap_bytes(); 32| | } 33| | // 0xd800 .. 0xdbff - low surrogate 34| | // 0xdc00 .. 0xdfff - high surrogate 35| 15.2k| const auto is_surrogate = ((input & uint16_t(0xf800)) == uint16_t(0xd800)); 36| | 37| | // c0 - chars that yield 2- or 3-byte UTF-8 codes 38| 15.2k| const auto c0 = min(input & uint16_t(0xff80), one); 39| | 40| | // c1 - chars that yield 3-byte UTF-8 codes (including surrogates) 41| 15.2k| const auto c1 = min(input & uint16_t(0xf800), one); 42| | 43| | /* 44| | Explanation how the counting works. 45| | 46| | In the case of a non-surrogate character we count: 47| | * always 1 -- see how `count` is initialized above; 48| | * c0 = 1 if the current char yields 2 or 3 bytes; 49| | * c1 = 1 if the current char yields 3 bytes. 50| | 51| | Thus, we always have correct count for the current char: 52| | from 1, 2 or 3 bytes. 53| | 54| | A trickier part is how we count surrogate pairs. Whether 55| | we encounter a surrogate (low or high), we count it as 56| | 3 chars and then minus 1 (`is_surrogate` is -1 or 0). 57| | Each surrogate char yields 2. A surrogate pair, that 58| | is a low surrogate followed by a high one, yields 59| | the expected 4 bytes. 60| | 61| | It also correctly handles cases when low surrogate is 62| | processed by the this loop, but high surrogate is counted 63| | by the scalar procedure. The scalar procedure uses exactly 64| | the described approach, thanks to that for valid UTF-16 65| | strings it always count correctly. 66| | */ 67| 15.2k| v_count += c0; 68| 15.2k| v_count += c1; 69| 15.2k| v_count += vector_u16(is_surrogate); 70| | 71| 15.2k| iteration -= 1; 72| 15.2k| if (iteration == 0) { ------------------ | Branch (72:9): [True: 0, False: 15.2k] ------------------ 73| 0| count += v_count.sum(); 74| 0| v_count = vector_u16::zero(); 75| 0| iteration = max_iterations; 76| 0| } 77| 15.2k| } 78| | 79| 13.1k| if (iteration > 0) { ------------------ | Branch (79:7): [True: 13.1k, False: 0] ------------------ 80| 13.1k| count += v_count.sum(); 81| 13.1k| } 82| | 83| 13.1k| return count + scalar::utf16::utf8_length_from_utf16(in + pos, 84| 13.1k| size - pos); 85| 13.1k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf1631utf8_length_from_utf16_bytemaskILNS_10endiannessE1EEEmPKDsm: 10| 13.1k| size_t size) { 11| 13.1k| size_t pos = 0; 12| | 13| 13.1k| using vector_u16 = simd16; 14| 13.1k| constexpr size_t N = vector_u16::ELEMENTS; 15| | 16| 13.1k| const auto one = vector_u16::splat(1); 17| | 18| 13.1k| auto v_count = vector_u16::zero(); 19| | 20| | // each char16 yields at least one byte 21| 13.1k| size_t count = size / N * N; 22| | 23| | // in a single iteration the increment is 0, 1 or 2, despite we have 24| | // three additions 25| 13.1k| constexpr size_t max_iterations = 65535 / 2; 26| 13.1k| size_t iteration = max_iterations; 27| | 28| 28.1k| for (; pos < size / N * N; pos += N) { ------------------ | Branch (28:10): [True: 15.0k, False: 13.1k] ------------------ 29| 15.0k| auto input = vector_u16::load(reinterpret_cast(in + pos)); 30| 15.0k| if constexpr (!match_system(big_endian)) { 31| 15.0k| input = input.swap_bytes(); 32| 15.0k| } 33| | // 0xd800 .. 0xdbff - low surrogate 34| | // 0xdc00 .. 0xdfff - high surrogate 35| 15.0k| const auto is_surrogate = ((input & uint16_t(0xf800)) == uint16_t(0xd800)); 36| | 37| | // c0 - chars that yield 2- or 3-byte UTF-8 codes 38| 15.0k| const auto c0 = min(input & uint16_t(0xff80), one); 39| | 40| | // c1 - chars that yield 3-byte UTF-8 codes (including surrogates) 41| 15.0k| const auto c1 = min(input & uint16_t(0xf800), one); 42| | 43| | /* 44| | Explanation how the counting works. 45| | 46| | In the case of a non-surrogate character we count: 47| | * always 1 -- see how `count` is initialized above; 48| | * c0 = 1 if the current char yields 2 or 3 bytes; 49| | * c1 = 1 if the current char yields 3 bytes. 50| | 51| | Thus, we always have correct count for the current char: 52| | from 1, 2 or 3 bytes. 53| | 54| | A trickier part is how we count surrogate pairs. Whether 55| | we encounter a surrogate (low or high), we count it as 56| | 3 chars and then minus 1 (`is_surrogate` is -1 or 0). 57| | Each surrogate char yields 2. A surrogate pair, that 58| | is a low surrogate followed by a high one, yields 59| | the expected 4 bytes. 60| | 61| | It also correctly handles cases when low surrogate is 62| | processed by the this loop, but high surrogate is counted 63| | by the scalar procedure. The scalar procedure uses exactly 64| | the described approach, thanks to that for valid UTF-16 65| | strings it always count correctly. 66| | */ 67| 15.0k| v_count += c0; 68| 15.0k| v_count += c1; 69| 15.0k| v_count += vector_u16(is_surrogate); 70| | 71| 15.0k| iteration -= 1; 72| 15.0k| if (iteration == 0) { ------------------ | Branch (72:9): [True: 0, False: 15.0k] ------------------ 73| 0| count += v_count.sum(); 74| 0| v_count = vector_u16::zero(); 75| 0| iteration = max_iterations; 76| 0| } 77| 15.0k| } 78| | 79| 13.1k| if (iteration > 0) { ------------------ | Branch (79:7): [True: 13.1k, False: 0] ------------------ 80| 13.1k| count += v_count.sum(); 81| 13.1k| } 82| | 83| 13.1k| return count + scalar::utf16::utf8_length_from_utf16(in + pos, 84| 13.1k| size - pos); 85| 13.1k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf3222utf8_length_from_utf32EPKDim: 11| 6.02k| size_t length) { 12| 6.02k| using vector_u32 = simd32; 13| | 14| 6.02k| const char32_t *start = input; 15| | 16| | // we add up to three ones in a single iteration (see the vectorized loop in 17| | // section #2 below) 18| 6.02k| const size_t max_increment = 3; 19| | 20| 6.02k| const size_t N = vector_u32::ELEMENTS; 21| | 22| |#if SIMDUTF_SIMD_HAS_UNSIGNED_CMP 23| | const auto v_0000007f = vector_u32::splat(0x0000007f); 24| | const auto v_000007ff = vector_u32::splat(0x000007ff); 25| | const auto v_0000ffff = vector_u32::splat(0x0000ffff); 26| |#else 27| 6.02k| const auto v_ffffff80 = vector_u32::splat(0xffffff80); 28| 6.02k| const auto v_fffff800 = vector_u32::splat(0xfffff800); 29| 6.02k| const auto v_ffff0000 = vector_u32::splat(0xffff0000); 30| 6.02k| const auto one = vector_u32::splat(1); 31| 6.02k|#endif // SIMDUTF_SIMD_HAS_UNSIGNED_CMP 32| | 33| 6.02k| size_t counter = 0; 34| | 35| | // 1. vectorized loop unrolled 4 times 36| 6.02k| { 37| | // we use vector of uint32 counters, this is why this limit is used 38| 6.02k| const size_t max_iterations = 39| 6.02k| std::numeric_limits::max() / (max_increment * 4); 40| 6.02k| size_t blocks = length / (N * 4); 41| 6.02k| length -= blocks * (N * 4); 42| 6.28k| while (blocks != 0) { ------------------ | Branch (42:12): [True: 265, False: 6.02k] ------------------ 43| 265| const size_t iterations = min(blocks, max_iterations); 44| 265| blocks -= iterations; 45| | 46| 265| simd32 acc = vector_u32::zero(); 47| 1.88k| for (size_t i = 0; i < iterations; i++) { ------------------ | Branch (47:26): [True: 1.62k, False: 265] ------------------ 48| 1.62k| const auto in0 = vector_u32(input + 0 * N); 49| 1.62k| const auto in1 = vector_u32(input + 1 * N); 50| 1.62k| const auto in2 = vector_u32(input + 2 * N); 51| 1.62k| const auto in3 = vector_u32(input + 3 * N); 52| | 53| |#if SIMDUTF_SIMD_HAS_UNSIGNED_CMP 54| | acc -= as_vector_u32(in0 > v_0000007f); 55| | acc -= as_vector_u32(in1 > v_0000007f); 56| | acc -= as_vector_u32(in2 > v_0000007f); 57| | acc -= as_vector_u32(in3 > v_0000007f); 58| | 59| | acc -= as_vector_u32(in0 > v_000007ff); 60| | acc -= as_vector_u32(in1 > v_000007ff); 61| | acc -= as_vector_u32(in2 > v_000007ff); 62| | acc -= as_vector_u32(in3 > v_000007ff); 63| | 64| | acc -= as_vector_u32(in0 > v_0000ffff); 65| | acc -= as_vector_u32(in1 > v_0000ffff); 66| | acc -= as_vector_u32(in2 > v_0000ffff); 67| | acc -= as_vector_u32(in3 > v_0000ffff); 68| |#else 69| 1.62k| acc += min(one, in0 & v_ffffff80); 70| 1.62k| acc += min(one, in1 & v_ffffff80); 71| 1.62k| acc += min(one, in2 & v_ffffff80); 72| 1.62k| acc += min(one, in3 & v_ffffff80); 73| | 74| 1.62k| acc += min(one, in0 & v_fffff800); 75| 1.62k| acc += min(one, in1 & v_fffff800); 76| 1.62k| acc += min(one, in2 & v_fffff800); 77| 1.62k| acc += min(one, in3 & v_fffff800); 78| | 79| 1.62k| acc += min(one, in0 & v_ffff0000); 80| 1.62k| acc += min(one, in1 & v_ffff0000); 81| 1.62k| acc += min(one, in2 & v_ffff0000); 82| 1.62k| acc += min(one, in3 & v_ffff0000); 83| 1.62k|#endif // SIMDUTF_SIMD_HAS_UNSIGNED_CMP 84| | 85| 1.62k| input += 4 * N; 86| 1.62k| } 87| | 88| 265| counter += acc.sum(); 89| 265| } 90| 6.02k| } 91| | 92| | // 2. vectorized loop for tail 93| 6.02k| { 94| 6.02k| const size_t max_iterations = 95| 6.02k| std::numeric_limits::max() / max_increment; 96| 6.02k| size_t blocks = length / N; 97| 6.02k| length -= blocks * N; 98| 6.50k| while (blocks != 0) { ------------------ | Branch (98:12): [True: 484, False: 6.02k] ------------------ 99| 484| const size_t iterations = min(blocks, max_iterations); 100| 484| blocks -= iterations; 101| | 102| 484| auto acc = vector_u32::zero(); 103| 1.60k| for (size_t i = 0; i < iterations; i++) { ------------------ | Branch (103:26): [True: 1.12k, False: 484] ------------------ 104| 1.12k| const auto in = vector_u32(input); 105| | 106| |#if SIMDUTF_SIMD_HAS_UNSIGNED_CMP 107| | acc -= as_vector_u32(in > v_0000007f); 108| | acc -= as_vector_u32(in > v_000007ff); 109| | acc -= as_vector_u32(in > v_0000ffff); 110| |#else 111| 1.12k| acc += min(one, in & v_ffffff80); 112| 1.12k| acc += min(one, in & v_fffff800); 113| 1.12k| acc += min(one, in & v_ffff0000); 114| 1.12k|#endif // SIMDUTF_SIMD_HAS_UNSIGNED_CMP 115| | 116| 1.12k| input += N; 117| 1.12k| } 118| | 119| 484| counter += acc.sum(); 120| 484| } 121| 6.02k| } 122| | 123| 6.02k| const size_t consumed = input - start; 124| 6.02k| if (consumed != 0) { ------------------ | Branch (124:7): [True: 556, False: 5.46k] ------------------ 125| | // We don't count 0th bytes in the vectorized loops above, this 126| | // is why we need to count them in the end. 127| 556| counter += consumed; 128| 556| } 129| | 130| 6.02k| return counter + scalar::utf32::utf8_length_from_utf32(input, length); 131| 6.02k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf323minImEET_S4_S4_: 8| 749|template T min(T a, T b) { return a <= b ? a : b; } ------------------ | Branch (8:48): [True: 749, False: 0] ------------------ simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf3222utf8_length_from_utf32EPKDim: 11| 6.26k| size_t length) { 12| 6.26k| using vector_u32 = simd32; 13| | 14| 6.26k| const char32_t *start = input; 15| | 16| | // we add up to three ones in a single iteration (see the vectorized loop in 17| | // section #2 below) 18| 6.26k| const size_t max_increment = 3; 19| | 20| 6.26k| const size_t N = vector_u32::ELEMENTS; 21| | 22| |#if SIMDUTF_SIMD_HAS_UNSIGNED_CMP 23| | const auto v_0000007f = vector_u32::splat(0x0000007f); 24| | const auto v_000007ff = vector_u32::splat(0x000007ff); 25| | const auto v_0000ffff = vector_u32::splat(0x0000ffff); 26| |#else 27| 6.26k| const auto v_ffffff80 = vector_u32::splat(0xffffff80); 28| 6.26k| const auto v_fffff800 = vector_u32::splat(0xfffff800); 29| 6.26k| const auto v_ffff0000 = vector_u32::splat(0xffff0000); 30| 6.26k| const auto one = vector_u32::splat(1); 31| 6.26k|#endif // SIMDUTF_SIMD_HAS_UNSIGNED_CMP 32| | 33| 6.26k| size_t counter = 0; 34| | 35| | // 1. vectorized loop unrolled 4 times 36| 6.26k| { 37| | // we use vector of uint32 counters, this is why this limit is used 38| 6.26k| const size_t max_iterations = 39| 6.26k| std::numeric_limits::max() / (max_increment * 4); 40| 6.26k| size_t blocks = length / (N * 4); 41| 6.26k| length -= blocks * (N * 4); 42| 6.77k| while (blocks != 0) { ------------------ | Branch (42:12): [True: 508, False: 6.26k] ------------------ 43| 508| const size_t iterations = min(blocks, max_iterations); 44| 508| blocks -= iterations; 45| | 46| 508| simd32 acc = vector_u32::zero(); 47| 5.09k| for (size_t i = 0; i < iterations; i++) { ------------------ | Branch (47:26): [True: 4.58k, False: 508] ------------------ 48| 4.58k| const auto in0 = vector_u32(input + 0 * N); 49| 4.58k| const auto in1 = vector_u32(input + 1 * N); 50| 4.58k| const auto in2 = vector_u32(input + 2 * N); 51| 4.58k| const auto in3 = vector_u32(input + 3 * N); 52| | 53| |#if SIMDUTF_SIMD_HAS_UNSIGNED_CMP 54| | acc -= as_vector_u32(in0 > v_0000007f); 55| | acc -= as_vector_u32(in1 > v_0000007f); 56| | acc -= as_vector_u32(in2 > v_0000007f); 57| | acc -= as_vector_u32(in3 > v_0000007f); 58| | 59| | acc -= as_vector_u32(in0 > v_000007ff); 60| | acc -= as_vector_u32(in1 > v_000007ff); 61| | acc -= as_vector_u32(in2 > v_000007ff); 62| | acc -= as_vector_u32(in3 > v_000007ff); 63| | 64| | acc -= as_vector_u32(in0 > v_0000ffff); 65| | acc -= as_vector_u32(in1 > v_0000ffff); 66| | acc -= as_vector_u32(in2 > v_0000ffff); 67| | acc -= as_vector_u32(in3 > v_0000ffff); 68| |#else 69| 4.58k| acc += min(one, in0 & v_ffffff80); 70| 4.58k| acc += min(one, in1 & v_ffffff80); 71| 4.58k| acc += min(one, in2 & v_ffffff80); 72| 4.58k| acc += min(one, in3 & v_ffffff80); 73| | 74| 4.58k| acc += min(one, in0 & v_fffff800); 75| 4.58k| acc += min(one, in1 & v_fffff800); 76| 4.58k| acc += min(one, in2 & v_fffff800); 77| 4.58k| acc += min(one, in3 & v_fffff800); 78| | 79| 4.58k| acc += min(one, in0 & v_ffff0000); 80| 4.58k| acc += min(one, in1 & v_ffff0000); 81| 4.58k| acc += min(one, in2 & v_ffff0000); 82| 4.58k| acc += min(one, in3 & v_ffff0000); 83| 4.58k|#endif // SIMDUTF_SIMD_HAS_UNSIGNED_CMP 84| | 85| 4.58k| input += 4 * N; 86| 4.58k| } 87| | 88| 508| counter += acc.sum(); 89| 508| } 90| 6.26k| } 91| | 92| | // 2. vectorized loop for tail 93| 6.26k| { 94| 6.26k| const size_t max_iterations = 95| 6.26k| std::numeric_limits::max() / max_increment; 96| 6.26k| size_t blocks = length / N; 97| 6.26k| length -= blocks * N; 98| 6.71k| while (blocks != 0) { ------------------ | Branch (98:12): [True: 449, False: 6.26k] ------------------ 99| 449| const size_t iterations = min(blocks, max_iterations); 100| 449| blocks -= iterations; 101| | 102| 449| auto acc = vector_u32::zero(); 103| 1.46k| for (size_t i = 0; i < iterations; i++) { ------------------ | Branch (103:26): [True: 1.01k, False: 449] ------------------ 104| 1.01k| const auto in = vector_u32(input); 105| | 106| |#if SIMDUTF_SIMD_HAS_UNSIGNED_CMP 107| | acc -= as_vector_u32(in > v_0000007f); 108| | acc -= as_vector_u32(in > v_000007ff); 109| | acc -= as_vector_u32(in > v_0000ffff); 110| |#else 111| 1.01k| acc += min(one, in & v_ffffff80); 112| 1.01k| acc += min(one, in & v_fffff800); 113| 1.01k| acc += min(one, in & v_ffff0000); 114| 1.01k|#endif // SIMDUTF_SIMD_HAS_UNSIGNED_CMP 115| | 116| 1.01k| input += N; 117| 1.01k| } 118| | 119| 449| counter += acc.sum(); 120| 449| } 121| 6.26k| } 122| | 123| 6.26k| const size_t consumed = input - start; 124| 6.26k| if (consumed != 0) { ------------------ | Branch (124:7): [True: 560, False: 5.70k] ------------------ 125| | // We don't count 0th bytes in the vectorized loops above, this 126| | // is why we need to count them in the end. 127| 560| counter += consumed; 128| 560| } 129| | 130| 6.26k| return counter + scalar::utf32::utf8_length_from_utf32(input, length); 131| 6.26k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf323minImEET_S4_S4_: 8| 957|template T min(T a, T b) { return a <= b ? a : b; } ------------------ | Branch (8:48): [True: 957, False: 0] ------------------ simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14utf826count_code_points_bytemaskEPKcm: 21| 13.7k|count_code_points_bytemask(const char *in, size_t size) { 22| 13.7k| using vector_i8 = simd8; 23| 13.7k| using vector_u8 = simd8; 24| 13.7k| using vector_u64 = simd64; 25| | 26| 13.7k| constexpr size_t N = vector_i8::SIZE; 27| 13.7k| constexpr size_t max_iterations = 255 / 4; 28| | 29| 13.7k| size_t pos = 0; 30| 13.7k| size_t count = 0; 31| | 32| 13.7k| auto counters = vector_u64::zero(); 33| 13.7k| auto local = vector_u8::zero(); 34| 13.7k| size_t iterations = 0; 35| 17.0k| for (; pos + 4 * N <= size; pos += 4 * N) { ------------------ | Branch (35:10): [True: 3.34k, False: 13.7k] ------------------ 36| 3.34k| const auto input0 = 37| 3.34k| simd8::load(reinterpret_cast(in + pos + 0 * N)); 38| 3.34k| const auto input1 = 39| 3.34k| simd8::load(reinterpret_cast(in + pos + 1 * N)); 40| 3.34k| const auto input2 = 41| 3.34k| simd8::load(reinterpret_cast(in + pos + 2 * N)); 42| 3.34k| const auto input3 = 43| 3.34k| simd8::load(reinterpret_cast(in + pos + 3 * N)); 44| 3.34k| const auto mask0 = input0 > int8_t(-65); 45| 3.34k| const auto mask1 = input1 > int8_t(-65); 46| 3.34k| const auto mask2 = input2 > int8_t(-65); 47| 3.34k| const auto mask3 = input3 > int8_t(-65); 48| | 49| 3.34k| local -= vector_u8(mask0); 50| 3.34k| local -= vector_u8(mask1); 51| 3.34k| local -= vector_u8(mask2); 52| 3.34k| local -= vector_u8(mask3); 53| | 54| 3.34k| iterations += 1; 55| 3.34k| if (iterations == max_iterations) { ------------------ | Branch (55:9): [True: 0, False: 3.34k] ------------------ 56| 0| counters += sum_8bytes(local); 57| 0| local = vector_u8::zero(); 58| 0| iterations = 0; 59| 0| } 60| 3.34k| } 61| | 62| 13.7k| if (iterations > 0) { ------------------ | Branch (62:7): [True: 880, False: 12.8k] ------------------ 63| 880| count += local.sum_bytes(); 64| 880| } 65| | 66| 13.7k| count += counters.sum(); 67| | 68| 13.7k| return count + scalar::utf8::count_code_points(in + pos, size - pos); 69| 13.7k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14utf817count_code_pointsEPKcm: 8| 6.85k|simdutf_really_inline size_t count_code_points(const char *in, size_t size) { 9| 6.85k| size_t pos = 0; 10| 6.85k| size_t count = 0; 11| 9.08k| for (; pos + 64 <= size; pos += 64) { ------------------ | Branch (11:10): [True: 2.22k, False: 6.85k] ------------------ 12| 2.22k| simd8x64 input(reinterpret_cast(in + pos)); 13| 2.22k| uint64_t utf8_continuation_mask = input.gt(-65); 14| 2.22k| count += count_ones(utf8_continuation_mask); 15| 2.22k| } 16| 6.85k| return count + scalar::utf8::count_code_points(in + pos, size - pos); 17| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14utf826count_code_points_bytemaskEPKcm: 21| 13.7k|count_code_points_bytemask(const char *in, size_t size) { 22| 13.7k| using vector_i8 = simd8; 23| 13.7k| using vector_u8 = simd8; 24| 13.7k| using vector_u64 = simd64; 25| | 26| 13.7k| constexpr size_t N = vector_i8::SIZE; 27| 13.7k| constexpr size_t max_iterations = 255 / 4; 28| | 29| 13.7k| size_t pos = 0; 30| 13.7k| size_t count = 0; 31| | 32| 13.7k| auto counters = vector_u64::zero(); 33| 13.7k| auto local = vector_u8::zero(); 34| 13.7k| size_t iterations = 0; 35| 22.6k| for (; pos + 4 * N <= size; pos += 4 * N) { ------------------ | Branch (35:10): [True: 8.91k, False: 13.7k] ------------------ 36| 8.91k| const auto input0 = 37| 8.91k| simd8::load(reinterpret_cast(in + pos + 0 * N)); 38| 8.91k| const auto input1 = 39| 8.91k| simd8::load(reinterpret_cast(in + pos + 1 * N)); 40| 8.91k| const auto input2 = 41| 8.91k| simd8::load(reinterpret_cast(in + pos + 2 * N)); 42| 8.91k| const auto input3 = 43| 8.91k| simd8::load(reinterpret_cast(in + pos + 3 * N)); 44| 8.91k| const auto mask0 = input0 > int8_t(-65); 45| 8.91k| const auto mask1 = input1 > int8_t(-65); 46| 8.91k| const auto mask2 = input2 > int8_t(-65); 47| 8.91k| const auto mask3 = input3 > int8_t(-65); 48| | 49| 8.91k| local -= vector_u8(mask0); 50| 8.91k| local -= vector_u8(mask1); 51| 8.91k| local -= vector_u8(mask2); 52| 8.91k| local -= vector_u8(mask3); 53| | 54| 8.91k| iterations += 1; 55| 8.91k| if (iterations == max_iterations) { ------------------ | Branch (55:9): [True: 0, False: 8.91k] ------------------ 56| 0| counters += sum_8bytes(local); 57| 0| local = vector_u8::zero(); 58| 0| iterations = 0; 59| 0| } 60| 8.91k| } 61| | 62| 13.7k| if (iterations > 0) { ------------------ | Branch (62:7): [True: 1.67k, False: 12.0k] ------------------ 63| 1.67k| count += local.sum_bytes(); 64| 1.67k| } 65| | 66| 13.7k| count += counters.sum(); 67| | 68| 13.7k| return count + scalar::utf8::count_code_points(in + pos, size - pos); 69| 13.7k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14utf817count_code_pointsEPKcm: 8| 6.85k|simdutf_really_inline size_t count_code_points(const char *in, size_t size) { 9| 6.85k| size_t pos = 0; 10| 6.85k| size_t count = 0; 11| 9.62k| for (; pos + 64 <= size; pos += 64) { ------------------ | Branch (11:10): [True: 2.77k, False: 6.85k] ------------------ 12| 2.77k| simd8x64 input(reinterpret_cast(in + pos)); 13| 2.77k| uint64_t utf8_continuation_mask = input.gt(-65); 14| 2.77k| count += count_ones(utf8_continuation_mask); 15| 2.77k| } 16| 6.85k| return count + scalar::utf8::count_code_points(in + pos, size - pos); 17| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14utf831utf16_length_from_utf8_bytemaskEPKcm: 9| 27.2k| size_t size) { 10| 27.2k| using vector_i8 = simd8; 11| 27.2k| using vector_u8 = simd8; 12| 27.2k| using vector_u64 = simd64; 13| | 14| 27.2k| constexpr size_t N = vector_i8::SIZE; 15| 27.2k| constexpr size_t max_iterations = 255 / 2; 16| | 17| 27.2k| auto counters = vector_u64::zero(); 18| 27.2k| auto local = vector_u8::zero(); 19| | 20| 27.2k| size_t iterations = 0; 21| 27.2k| size_t pos = 0; 22| 27.2k| size_t count = 0; 23| 40.8k| for (; pos + N <= size; pos += N) { ------------------ | Branch (23:10): [True: 13.6k, False: 27.2k] ------------------ 24| 13.6k| const auto input = 25| 13.6k| vector_i8::load(reinterpret_cast(in + pos)); 26| | 27| 13.6k| const auto continuation = input > int8_t(-65); 28| 13.6k| const auto utf_4bytes = vector_u8(input.value) >= uint8_t(240); 29| | 30| 13.6k| local -= vector_u8(continuation); 31| 13.6k| local -= vector_u8(utf_4bytes); 32| | 33| 13.6k| iterations += 1; 34| 13.6k| if (iterations == max_iterations) { ------------------ | Branch (34:9): [True: 0, False: 13.6k] ------------------ 35| 0| counters += sum_8bytes(local); 36| 0| local = vector_u8::zero(); 37| 0| iterations = 0; 38| 0| } 39| 13.6k| } 40| | 41| 27.2k| if (iterations > 0) { ------------------ | Branch (41:7): [True: 2.33k, False: 24.9k] ------------------ 42| 2.33k| count += local.sum_bytes(); 43| 2.33k| } 44| | 45| 27.2k| count += counters.sum(); 46| | 47| 27.2k| return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); 48| 27.2k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14utf831utf16_length_from_utf8_bytemaskEPKcm: 9| 27.2k| size_t size) { 10| 27.2k| using vector_i8 = simd8; 11| 27.2k| using vector_u8 = simd8; 12| 27.2k| using vector_u64 = simd64; 13| | 14| 27.2k| constexpr size_t N = vector_i8::SIZE; 15| 27.2k| constexpr size_t max_iterations = 255 / 2; 16| | 17| 27.2k| auto counters = vector_u64::zero(); 18| 27.2k| auto local = vector_u8::zero(); 19| | 20| 27.2k| size_t iterations = 0; 21| 27.2k| size_t pos = 0; 22| 27.2k| size_t count = 0; 23| 61.6k| for (; pos + N <= size; pos += N) { ------------------ | Branch (23:10): [True: 34.4k, False: 27.2k] ------------------ 24| 34.4k| const auto input = 25| 34.4k| vector_i8::load(reinterpret_cast(in + pos)); 26| | 27| 34.4k| const auto continuation = input > int8_t(-65); 28| 34.4k| const auto utf_4bytes = vector_u8(input.value) >= uint8_t(240); 29| | 30| 34.4k| local -= vector_u8(continuation); 31| 34.4k| local -= vector_u8(utf_4bytes); 32| | 33| 34.4k| iterations += 1; 34| 34.4k| if (iterations == max_iterations) { ------------------ | Branch (34:9): [True: 0, False: 34.4k] ------------------ 35| 0| counters += sum_8bytes(local); 36| 0| local = vector_u8::zero(); 37| 0| iterations = 0; 38| 0| } 39| 34.4k| } 40| | 41| 27.2k| if (iterations > 0) { ------------------ | Branch (41:7): [True: 2.35k, False: 24.8k] ------------------ 42| 2.35k| count += local.sum_bytes(); 43| 2.35k| } 44| | 45| 27.2k| count += counters.sum(); 46| | 47| 27.2k| return count + scalar::utf8::utf16_length_from_utf8(in + pos, size - pos); 48| 27.2k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_114utf8_to_latin121validating_transcoderC2Ev: 101| 13.7k| validating_transcoder() : error(uint8_t(0)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_114utf8_to_latin121validating_transcoder7convertEPKcmPc: 147| 13.7k| char *latin1_output) { 148| 13.7k| if (size < 64) { // too short for the main loop ------------------ | Branch (148:9): [True: 11.7k, False: 1.95k] ------------------ 149| 11.7k| return scalar::utf8_to_latin1::convert(in, size, latin1_output); 150| 11.7k| } 151| 1.95k| size_t pos = 0; 152| 1.95k| char *start{latin1_output}; 153| | // In the worst case, we have the haswell kernel which can cause an overflow 154| | // of 8 bytes when calling convert_masked_utf8_to_latin1. If you skip the 155| | // last 16 bytes, and if the data is valid, then it is entirely safe because 156| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 157| | // generally assume that you have valid UTF-8 input, so we are going to go 158| | // back from the end counting 16 leading bytes, to give us a good margin. 159| 1.95k| size_t leading_byte = 0; 160| 1.95k| size_t margin = size; 161| 52.5k| for (; margin > 0 && leading_byte < 16; margin--) { ------------------ | Branch (161:12): [True: 52.5k, False: 21] | Branch (161:26): [True: 50.5k, False: 1.93k] ------------------ 162| 50.5k| leading_byte += (int8_t(in[margin - 1]) > 163| 50.5k| -65); // twos complement of -65 is 1011 1111 ... 164| 50.5k| } 165| | // If the input is long enough, then we have that margin-1 is the eight last 166| | // leading byte. 167| 1.95k| const size_t safety_margin = size - margin + 1; // to avoid overruns! 168| 9.16k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (168:12): [True: 7.21k, False: 1.94k] ------------------ 169| 7.21k| simd8x64 input(reinterpret_cast(in + pos)); 170| 7.21k| if (input.is_ascii()) { ------------------ | Branch (170:11): [True: 2.48k, False: 4.73k] ------------------ 171| 2.48k| input.store((int8_t *)latin1_output); 172| 2.48k| latin1_output += 64; 173| 2.48k| pos += 64; 174| 4.73k| } else { 175| | // you might think that a for-loop would work, but under Visual Studio, 176| | // it is not good enough. 177| 4.73k| static_assert( 178| 4.73k| (simd8x64::NUM_CHUNKS == 2) || 179| 4.73k| (simd8x64::NUM_CHUNKS == 4), 180| 4.73k| "We support either two or four chunks per 64-byte block."); 181| 4.73k| auto zero = simd8{uint8_t(0)}; 182| 4.73k| if constexpr (simd8x64::NUM_CHUNKS == 2) { 183| 4.73k| this->check_utf8_bytes(input.chunks[0], zero); 184| 4.73k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 185| | } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 186| | this->check_utf8_bytes(input.chunks[0], zero); 187| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 188| | this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 189| | this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 190| | } 191| 4.73k| uint64_t utf8_continuation_mask = 192| 4.73k| input.lt(-65 + 1); // -64 is 1100 0000 in twos complement. Note: in 193| | // this case, we also have ASCII to account for. 194| 4.73k| if (utf8_continuation_mask & 1) { ------------------ | Branch (194:13): [True: 8, False: 4.72k] ------------------ 195| 8| return 0; // error 196| 8| } 197| 4.72k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 198| 4.72k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 199| | // We process in blocks of up to 12 bytes except possibly 200| | // for fast paths which may process up to 16 bytes. For the 201| | // slow path to work, we should have at least 12 input bytes left. 202| 4.72k| size_t max_starting_point = (pos + 64) - 12; 203| | // Next loop is going to run at least five times. 204| 38.2k| while (pos < max_starting_point) { ------------------ | Branch (204:16): [True: 33.5k, False: 4.72k] ------------------ 205| | // Performance note: our ability to compute 'consumed' and 206| | // then shift and recompute is critical. If there is a 207| | // latency of, say, 4 cycles on getting 'consumed', then 208| | // the inner loop might have a total latency of about 6 cycles. 209| | // Yet we process between 6 to 12 inputs bytes, thus we get 210| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 211| | // for this section of the code. Hence, there is a limit 212| | // to how much we can further increase this latency before 213| | // it seriously harms performance. 214| 33.5k| size_t consumed = convert_masked_utf8_to_latin1( 215| 33.5k| in + pos, utf8_end_of_code_point_mask, latin1_output); 216| 33.5k| pos += consumed; 217| 33.5k| utf8_end_of_code_point_mask >>= consumed; 218| 33.5k| } 219| | // At this point there may remain between 0 and 12 bytes in the 220| | // 64-byte block. These bytes will be processed again. So we have an 221| | // 80% efficiency (in the worst case). In practice we expect an 222| | // 85% to 90% efficiency. 223| 4.72k| } 224| 7.21k| } 225| 1.94k| if (errors()) { ------------------ | Branch (225:9): [True: 244, False: 1.70k] ------------------ 226| 244| return 0; 227| 244| } 228| 1.70k| const size_t ascii = convert_ascii_tail(in, pos, size, latin1_output); 229| 1.70k| pos += ascii; 230| 1.70k| latin1_output += ascii; 231| 1.70k| if (pos == size) { ------------------ | Branch (231:9): [True: 401, False: 1.30k] ------------------ 232| 401| return latin1_output - start; 233| 401| } 234| 1.30k| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 1.30k|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 235| 1.30k| size_t howmany = 236| 1.30k| scalar::utf8_to_latin1::convert(in + pos, size - pos, latin1_output); 237| 1.30k| if (howmany == 0) { ------------------ | Branch (237:9): [True: 388, False: 915] ------------------ 238| 388| return 0; 239| 388| } 240| 915| return latin1_output + howmany - start; 241| 1.30k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_114utf8_to_latin121validating_transcoder16check_utf8_bytesENS1_4simd5simd8IhEES6_: 106| 9.46k| const simd8 prev_input) { 107| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 108| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 109| | // small negative numbers) 110| 9.46k| simd8 prev1 = input.prev<1>(prev_input); 111| 9.46k| this->error |= check_special_cases(input, prev1); 112| 9.46k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_114utf8_to_latin119check_special_casesENS1_4simd5simd8IhEES5_: 8| 9.46k|check_special_cases(const simd8 input, const simd8 prev1) { 9| | // For UTF-8 to Latin 1, we can allow any ASCII character, and any 10| | // continuation byte, but the non-ASCII leading bytes must be 0b11000011 or 11| | // 0b11000010 and nothing else. 12| | // 13| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 14| | // Bit 1 = Too Long (ASCII followed by continuation) 15| | // Bit 2 = Overlong 3-byte 16| | // Bit 4 = Surrogate 17| | // Bit 5 = Overlong 2-byte 18| | // Bit 7 = Two Continuations 19| 9.46k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 20| | // 11______ 11______ 21| 9.46k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 22| 9.46k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 23| 9.46k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 24| 9.46k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 25| 9.46k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 26| 9.46k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 27| | // 11110100 101_____ 28| | // 11110101 1001____ 29| | // 11110101 101_____ 30| | // 1111011_ 1001____ 31| | // 1111011_ 101_____ 32| | // 11111___ 1001____ 33| | // 11111___ 101_____ 34| 9.46k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 35| | // 11110101 1000____ 36| | // 1111011_ 1000____ 37| | // 11111___ 1000____ 38| 9.46k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 39| 9.46k| constexpr const uint8_t FORBIDDEN = 0xff; 40| | 41| 9.46k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 42| | // 0_______ ________ 43| 9.46k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 44| 9.46k| TOO_LONG, 45| | // 10______ ________ 46| 9.46k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 47| | // 1100____ ________ 48| 9.46k| TOO_SHORT | OVERLONG_2, 49| | // 1101____ ________ 50| 9.46k| FORBIDDEN, 51| | // 1110____ ________ 52| 9.46k| FORBIDDEN, 53| | // 1111____ ________ 54| 9.46k| FORBIDDEN); 55| 9.46k| constexpr const uint8_t CARRY = 56| 9.46k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 57| 9.46k| const simd8 byte_1_low = 58| 9.46k| (prev1 & 0x0F) 59| 9.46k| .lookup_16( 60| | // ____0000 ________ 61| 9.46k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 62| | // ____0001 ________ 63| 9.46k| CARRY | OVERLONG_2, 64| | // ____001_ ________ 65| 9.46k| CARRY, CARRY, 66| | 67| | // ____0100 ________ 68| 9.46k| FORBIDDEN, 69| | // ____0101 ________ 70| 9.46k| FORBIDDEN, 71| | // ____011_ ________ 72| 9.46k| FORBIDDEN, FORBIDDEN, 73| | 74| | // ____1___ ________ 75| 9.46k| FORBIDDEN, FORBIDDEN, FORBIDDEN, FORBIDDEN, FORBIDDEN, 76| | // ____1101 ________ 77| 9.46k| FORBIDDEN, FORBIDDEN, FORBIDDEN); 78| 9.46k| const simd8 byte_2_high = input.shr<4>().lookup_16( 79| | // ________ 0_______ 80| 9.46k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 81| 9.46k| TOO_SHORT, TOO_SHORT, 82| | 83| | // ________ 1000____ 84| 9.46k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 85| 9.46k| OVERLONG_4, 86| | // ________ 1001____ 87| 9.46k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 88| | // ________ 101_____ 89| 9.46k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 90| 9.46k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 91| | 92| | // ________ 11______ 93| 9.46k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 94| 9.46k| return (byte_1_high & byte_1_low & byte_2_high); 95| 9.46k|} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_114utf8_to_latin121validating_transcoder6errorsEv: 348| 1.94k| simdutf_really_inline bool errors() const { 349| 1.94k| return this->error.any_bits_set_anywhere(); 350| 1.94k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_114utf8_to_latin121validating_transcoder18convert_ascii_tailEPKcmmPc: 115| 1.70k| size_t size, char *output) { 116| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 117| | // tails. 118| 1.70k| if (size - pos >= 8) { ------------------ | Branch (118:9): [True: 1.70k, False: 0] ------------------ 119| 1.70k| uint64_t first_bytes; 120| 1.70k| std::memcpy(&first_bytes, in + pos, 8); 121| 1.70k| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (121:11): [True: 781, False: 923] ------------------ 122| 781| return 0; 123| 781| } 124| 1.70k| } 125| 923| const size_t start = pos; 126| 1.26k| while (pos + 64 <= size) { ------------------ | Branch (126:12): [True: 648, False: 621] ------------------ 127| 648| simd8x64 input(reinterpret_cast(in + pos)); 128| 648| if (!input.is_ascii()) { ------------------ | Branch (128:11): [True: 302, False: 346] ------------------ 129| 302| return pos - start; 130| 302| } 131| 346| input.store(reinterpret_cast(output + (pos - start))); 132| 346| pos += 64; 133| 346| } 134| 621| if (pos == size || size < 64) { ------------------ | Branch (134:9): [True: 165, False: 456] | Branch (134:24): [True: 0, False: 456] ------------------ 135| 165| return pos - start; 136| 165| } 137| 456| simd8x64 input(reinterpret_cast(in + size - 64)); 138| 456| if (!input.is_ascii()) { ------------------ | Branch (138:9): [True: 220, False: 236] ------------------ 139| 220| return pos - start; 140| 220| } 141| 236| input.store(reinterpret_cast(output + (pos - start) - 142| 236| (pos - (size - 64)))); 143| 236| return size - start; 144| 456| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_114utf8_to_latin121validating_transcoderC2Ev: 101| 13.7k| validating_transcoder() : error(uint8_t(0)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_114utf8_to_latin121validating_transcoder7convertEPKcmPc: 147| 13.7k| char *latin1_output) { 148| 13.7k| if (size < 64) { // too short for the main loop ------------------ | Branch (148:9): [True: 12.0k, False: 1.67k] ------------------ 149| 12.0k| return scalar::utf8_to_latin1::convert(in, size, latin1_output); 150| 12.0k| } 151| 1.67k| size_t pos = 0; 152| 1.67k| char *start{latin1_output}; 153| | // In the worst case, we have the haswell kernel which can cause an overflow 154| | // of 8 bytes when calling convert_masked_utf8_to_latin1. If you skip the 155| | // last 16 bytes, and if the data is valid, then it is entirely safe because 156| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 157| | // generally assume that you have valid UTF-8 input, so we are going to go 158| | // back from the end counting 16 leading bytes, to give us a good margin. 159| 1.67k| size_t leading_byte = 0; 160| 1.67k| size_t margin = size; 161| 40.5k| for (; margin > 0 && leading_byte < 16; margin--) { ------------------ | Branch (161:12): [True: 40.5k, False: 17] | Branch (161:26): [True: 38.8k, False: 1.65k] ------------------ 162| 38.8k| leading_byte += (int8_t(in[margin - 1]) > 163| 38.8k| -65); // twos complement of -65 is 1011 1111 ... 164| 38.8k| } 165| | // If the input is long enough, then we have that margin-1 is the eight last 166| | // leading byte. 167| 1.67k| const size_t safety_margin = size - margin + 1; // to avoid overruns! 168| 9.97k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (168:12): [True: 8.31k, False: 1.65k] ------------------ 169| 8.31k| simd8x64 input(reinterpret_cast(in + pos)); 170| 8.31k| if (input.is_ascii()) { ------------------ | Branch (170:11): [True: 2.71k, False: 5.60k] ------------------ 171| 2.71k| input.store((int8_t *)latin1_output); 172| 2.71k| latin1_output += 64; 173| 2.71k| pos += 64; 174| 5.60k| } else { 175| | // you might think that a for-loop would work, but under Visual Studio, 176| | // it is not good enough. 177| 5.60k| static_assert( 178| 5.60k| (simd8x64::NUM_CHUNKS == 2) || 179| 5.60k| (simd8x64::NUM_CHUNKS == 4), 180| 5.60k| "We support either two or four chunks per 64-byte block."); 181| 5.60k| auto zero = simd8{uint8_t(0)}; 182| | if constexpr (simd8x64::NUM_CHUNKS == 2) { 183| | this->check_utf8_bytes(input.chunks[0], zero); 184| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 185| 5.60k| } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 186| 5.60k| this->check_utf8_bytes(input.chunks[0], zero); 187| 5.60k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 188| 5.60k| this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 189| 5.60k| this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 190| 5.60k| } 191| 5.60k| uint64_t utf8_continuation_mask = 192| 5.60k| input.lt(-65 + 1); // -64 is 1100 0000 in twos complement. Note: in 193| | // this case, we also have ASCII to account for. 194| 5.60k| if (utf8_continuation_mask & 1) { ------------------ | Branch (194:13): [True: 14, False: 5.59k] ------------------ 195| 14| return 0; // error 196| 14| } 197| 5.59k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 198| 5.59k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 199| | // We process in blocks of up to 12 bytes except possibly 200| | // for fast paths which may process up to 16 bytes. For the 201| | // slow path to work, we should have at least 12 input bytes left. 202| 5.59k| size_t max_starting_point = (pos + 64) - 12; 203| | // Next loop is going to run at least five times. 204| 44.7k| while (pos < max_starting_point) { ------------------ | Branch (204:16): [True: 39.1k, False: 5.59k] ------------------ 205| | // Performance note: our ability to compute 'consumed' and 206| | // then shift and recompute is critical. If there is a 207| | // latency of, say, 4 cycles on getting 'consumed', then 208| | // the inner loop might have a total latency of about 6 cycles. 209| | // Yet we process between 6 to 12 inputs bytes, thus we get 210| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 211| | // for this section of the code. Hence, there is a limit 212| | // to how much we can further increase this latency before 213| | // it seriously harms performance. 214| 39.1k| size_t consumed = convert_masked_utf8_to_latin1( 215| 39.1k| in + pos, utf8_end_of_code_point_mask, latin1_output); 216| 39.1k| pos += consumed; 217| 39.1k| utf8_end_of_code_point_mask >>= consumed; 218| 39.1k| } 219| | // At this point there may remain between 0 and 12 bytes in the 220| | // 64-byte block. These bytes will be processed again. So we have an 221| | // 80% efficiency (in the worst case). In practice we expect an 222| | // 85% to 90% efficiency. 223| 5.59k| } 224| 8.31k| } 225| 1.65k| if (errors()) { ------------------ | Branch (225:9): [True: 240, False: 1.41k] ------------------ 226| 240| return 0; 227| 240| } 228| 1.41k| const size_t ascii = convert_ascii_tail(in, pos, size, latin1_output); 229| 1.41k| pos += ascii; 230| 1.41k| latin1_output += ascii; 231| 1.41k| if (pos == size) { ------------------ | Branch (231:9): [True: 428, False: 988] ------------------ 232| 428| return latin1_output - start; 233| 428| } 234| 988| SIMDUTF_VZEROUPPER(); ------------------ | | 16| 988|#define SIMDUTF_VZEROUPPER() (void)0 ------------------ 235| 988| size_t howmany = 236| 988| scalar::utf8_to_latin1::convert(in + pos, size - pos, latin1_output); 237| 988| if (howmany == 0) { ------------------ | Branch (237:9): [True: 289, False: 699] ------------------ 238| 289| return 0; 239| 289| } 240| 699| return latin1_output + howmany - start; 241| 988| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_114utf8_to_latin121validating_transcoder16check_utf8_bytesENS1_4simd5simd8IhEES6_: 106| 22.4k| const simd8 prev_input) { 107| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 108| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 109| | // small negative numbers) 110| 22.4k| simd8 prev1 = input.prev<1>(prev_input); 111| 22.4k| this->error |= check_special_cases(input, prev1); 112| 22.4k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_114utf8_to_latin119check_special_casesENS1_4simd5simd8IhEES5_: 8| 22.4k|check_special_cases(const simd8 input, const simd8 prev1) { 9| | // For UTF-8 to Latin 1, we can allow any ASCII character, and any 10| | // continuation byte, but the non-ASCII leading bytes must be 0b11000011 or 11| | // 0b11000010 and nothing else. 12| | // 13| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 14| | // Bit 1 = Too Long (ASCII followed by continuation) 15| | // Bit 2 = Overlong 3-byte 16| | // Bit 4 = Surrogate 17| | // Bit 5 = Overlong 2-byte 18| | // Bit 7 = Two Continuations 19| 22.4k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 20| | // 11______ 11______ 21| 22.4k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 22| 22.4k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 23| 22.4k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 24| 22.4k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 25| 22.4k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 26| 22.4k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 27| | // 11110100 101_____ 28| | // 11110101 1001____ 29| | // 11110101 101_____ 30| | // 1111011_ 1001____ 31| | // 1111011_ 101_____ 32| | // 11111___ 1001____ 33| | // 11111___ 101_____ 34| 22.4k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 35| | // 11110101 1000____ 36| | // 1111011_ 1000____ 37| | // 11111___ 1000____ 38| 22.4k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 39| 22.4k| constexpr const uint8_t FORBIDDEN = 0xff; 40| | 41| 22.4k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 42| | // 0_______ ________ 43| 22.4k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 44| 22.4k| TOO_LONG, 45| | // 10______ ________ 46| 22.4k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 47| | // 1100____ ________ 48| 22.4k| TOO_SHORT | OVERLONG_2, 49| | // 1101____ ________ 50| 22.4k| FORBIDDEN, 51| | // 1110____ ________ 52| 22.4k| FORBIDDEN, 53| | // 1111____ ________ 54| 22.4k| FORBIDDEN); 55| 22.4k| constexpr const uint8_t CARRY = 56| 22.4k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 57| 22.4k| const simd8 byte_1_low = 58| 22.4k| (prev1 & 0x0F) 59| 22.4k| .lookup_16( 60| | // ____0000 ________ 61| 22.4k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 62| | // ____0001 ________ 63| 22.4k| CARRY | OVERLONG_2, 64| | // ____001_ ________ 65| 22.4k| CARRY, CARRY, 66| | 67| | // ____0100 ________ 68| 22.4k| FORBIDDEN, 69| | // ____0101 ________ 70| 22.4k| FORBIDDEN, 71| | // ____011_ ________ 72| 22.4k| FORBIDDEN, FORBIDDEN, 73| | 74| | // ____1___ ________ 75| 22.4k| FORBIDDEN, FORBIDDEN, FORBIDDEN, FORBIDDEN, FORBIDDEN, 76| | // ____1101 ________ 77| 22.4k| FORBIDDEN, FORBIDDEN, FORBIDDEN); 78| 22.4k| const simd8 byte_2_high = input.shr<4>().lookup_16( 79| | // ________ 0_______ 80| 22.4k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 81| 22.4k| TOO_SHORT, TOO_SHORT, 82| | 83| | // ________ 1000____ 84| 22.4k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 85| 22.4k| OVERLONG_4, 86| | // ________ 1001____ 87| 22.4k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 88| | // ________ 101_____ 89| 22.4k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 90| 22.4k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 91| | 92| | // ________ 11______ 93| 22.4k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 94| 22.4k| return (byte_1_high & byte_1_low & byte_2_high); 95| 22.4k|} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_114utf8_to_latin121validating_transcoder6errorsEv: 348| 1.65k| simdutf_really_inline bool errors() const { 349| 1.65k| return this->error.any_bits_set_anywhere(); 350| 1.65k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_114utf8_to_latin121validating_transcoder18convert_ascii_tailEPKcmmPc: 115| 1.41k| size_t size, char *output) { 116| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 117| | // tails. 118| 1.41k| if (size - pos >= 8) { ------------------ | Branch (118:9): [True: 1.41k, False: 0] ------------------ 119| 1.41k| uint64_t first_bytes; 120| 1.41k| std::memcpy(&first_bytes, in + pos, 8); 121| 1.41k| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (121:11): [True: 671, False: 745] ------------------ 122| 671| return 0; 123| 671| } 124| 1.41k| } 125| 745| const size_t start = pos; 126| 1.11k| while (pos + 64 <= size) { ------------------ | Branch (126:12): [True: 566, False: 550] ------------------ 127| 566| simd8x64 input(reinterpret_cast(in + pos)); 128| 566| if (!input.is_ascii()) { ------------------ | Branch (128:11): [True: 195, False: 371] ------------------ 129| 195| return pos - start; 130| 195| } 131| 371| input.store(reinterpret_cast(output + (pos - start))); 132| 371| pos += 64; 133| 371| } 134| 550| if (pos == size || size < 64) { ------------------ | Branch (134:9): [True: 202, False: 348] | Branch (134:24): [True: 0, False: 348] ------------------ 135| 202| return pos - start; 136| 202| } 137| 348| simd8x64 input(reinterpret_cast(in + size - 64)); 138| 348| if (!input.is_ascii()) { ------------------ | Branch (138:9): [True: 122, False: 226] ------------------ 139| 122| return pos - start; 140| 122| } 141| 226| input.store(reinterpret_cast(output + (pos - start) - 142| 226| (pos - (size - 64)))); 143| 226| return size - start; 144| 348| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoderC2Ev: 114| 27.2k| validating_transcoder() : error(uint8_t(0)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoderC2Ev: 114| 27.2k| validating_transcoder() : error(uint8_t(0)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoder7convertILNS_10endiannessE0EEEmPKcmPDs: 164| 13.6k| char16_t *utf16_output) { 165| 13.6k| if (size < 64) { // too short for the main loop ------------------ | Branch (165:9): [True: 12.6k, False: 957] ------------------ 166| 12.6k| return scalar::utf8_to_utf16::convert(in, size, utf16_output); 167| 12.6k| } 168| 957| size_t pos = 0; 169| 957| char16_t *start{utf16_output}; 170| | // In the worst case, we have the haswell kernel which can cause an overflow 171| | // of 8 bytes when calling convert_masked_utf8_to_utf16. If you skip the 172| | // last 16 bytes, and if the data is valid, then it is entirely safe because 173| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 174| | // generally assume that you have valid UTF-8 input, so we are going to go 175| | // back from the end counting 8 leading bytes, to give us a good margin. 176| 957| size_t leading_byte = 0; 177| 957| size_t margin = size; 178| 20.7k| for (; margin > 0 && leading_byte < 8; margin--) { ------------------ | Branch (178:12): [True: 20.7k, False: 14] | Branch (178:26): [True: 19.7k, False: 943] ------------------ 179| 19.7k| leading_byte += (int8_t(in[margin - 1]) > -65); 180| 19.7k| } 181| | // If the input is long enough, then we have that margin-1 is the eight last 182| | // leading byte. 183| 957| const size_t safety_margin = size - margin + 1; // to avoid overruns! 184| 3.92k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (184:12): [True: 2.97k, False: 946] ------------------ 185| 2.97k| simd8x64 input(reinterpret_cast(in + pos)); 186| 2.97k| if (input.is_ascii()) { ------------------ | Branch (186:11): [True: 685, False: 2.29k] ------------------ 187| 685| input.store_ascii_as_utf16(utf16_output); 188| 685| utf16_output += 64; 189| 685| pos += 64; 190| 2.29k| } else { 191| | // you might think that a for-loop would work, but under Visual Studio, 192| | // it is not good enough. 193| 2.29k| static_assert( 194| 2.29k| (simd8x64::NUM_CHUNKS == 2) || 195| 2.29k| (simd8x64::NUM_CHUNKS == 4), 196| 2.29k| "We support either two or four chunks per 64-byte block."); 197| 2.29k| auto zero = simd8{uint8_t(0)}; 198| 2.29k| if constexpr (simd8x64::NUM_CHUNKS == 2) { 199| 2.29k| this->check_utf8_bytes(input.chunks[0], zero); 200| 2.29k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 201| | } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 202| | this->check_utf8_bytes(input.chunks[0], zero); 203| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 204| | this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 205| | this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 206| | } 207| 2.29k| uint64_t utf8_continuation_mask = input.lt(-65 + 1); 208| 2.29k| if (utf8_continuation_mask & 1) { ------------------ | Branch (208:13): [True: 11, False: 2.28k] ------------------ 209| 11| return 0; // error 210| 11| } 211| 2.28k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 212| 2.28k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 213| | // We process in blocks of up to 12 bytes except possibly 214| | // for fast paths which may process up to 16 bytes. For the 215| | // slow path to work, we should have at least 12 input bytes left. 216| 2.28k| size_t max_starting_point = (pos + 64) - 12; 217| | // Next loop is going to run at least five times. 218| 19.0k| while (pos < max_starting_point) { ------------------ | Branch (218:16): [True: 16.8k, False: 2.28k] ------------------ 219| | // Performance note: our ability to compute 'consumed' and 220| | // then shift and recompute is critical. If there is a 221| | // latency of, say, 4 cycles on getting 'consumed', then 222| | // the inner loop might have a total latency of about 6 cycles. 223| | // Yet we process between 6 to 12 inputs bytes, thus we get 224| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 225| | // for this section of the code. Hence, there is a limit 226| | // to how much we can further increase this latency before 227| | // it seriously harms performance. 228| 16.8k| size_t consumed = convert_masked_utf8_to_utf16( 229| 16.8k| in + pos, utf8_end_of_code_point_mask, utf16_output); 230| 16.8k| pos += consumed; 231| 16.8k| utf8_end_of_code_point_mask >>= consumed; 232| 16.8k| } 233| | // At this point there may remain between 0 and 12 bytes in the 234| | // 64-byte block. These bytes will be processed again. So we have an 235| | // 80% efficiency (in the worst case). In practice we expect an 236| | // 85% to 90% efficiency. 237| 2.28k| } 238| 2.97k| } 239| 946| if (errors()) { ------------------ | Branch (239:9): [True: 209, False: 737] ------------------ 240| 209| return 0; 241| 209| } 242| 737| const size_t ascii = 243| 737| convert_ascii_tail(in, pos, size, utf16_output); 244| 737| pos += ascii; 245| 737| utf16_output += ascii; 246| 737| if (pos == size) { ------------------ | Branch (246:9): [True: 97, False: 640] ------------------ 247| 97| return utf16_output - start; 248| 97| } 249| 640| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 640|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 250| 640| size_t howmany = scalar::utf8_to_utf16::convert( 251| 640| in + pos, size - pos, utf16_output); 252| 640| if (howmany == 0) { ------------------ | Branch (252:9): [True: 281, False: 359] ------------------ 253| 281| return 0; 254| 281| } 255| 359| return utf16_output + howmany - start; 256| 640| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoder16check_utf8_bytesENS1_4simd5simd8IhEES6_: 119| 8.98k| const simd8 prev_input) { 120| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 121| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 122| | // small negative numbers) 123| 8.98k| simd8 prev1 = input.prev<1>(prev_input); 124| 8.98k| simd8 sc = check_special_cases(input, prev1); 125| 8.98k| this->error |= check_multibyte_lengths(input, prev_input, sc); 126| 8.98k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1619check_special_casesENS1_4simd5simd8IhEES5_: 8| 8.98k|check_special_cases(const simd8 input, const simd8 prev1) { 9| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 10| | // Bit 1 = Too Long (ASCII followed by continuation) 11| | // Bit 2 = Overlong 3-byte 12| | // Bit 4 = Surrogate 13| | // Bit 5 = Overlong 2-byte 14| | // Bit 7 = Two Continuations 15| 8.98k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 16| | // 11______ 11______ 17| 8.98k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 18| 8.98k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 19| 8.98k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 20| 8.98k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 21| 8.98k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 22| 8.98k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 23| | // 11110100 101_____ 24| | // 11110101 1001____ 25| | // 11110101 101_____ 26| | // 1111011_ 1001____ 27| | // 1111011_ 101_____ 28| | // 11111___ 1001____ 29| | // 11111___ 101_____ 30| 8.98k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 31| | // 11110101 1000____ 32| | // 1111011_ 1000____ 33| | // 11111___ 1000____ 34| 8.98k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 35| | 36| 8.98k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 37| | // 0_______ ________ 38| 8.98k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 39| 8.98k| TOO_LONG, 40| | // 10______ ________ 41| 8.98k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 42| | // 1100____ ________ 43| 8.98k| TOO_SHORT | OVERLONG_2, 44| | // 1101____ ________ 45| 8.98k| TOO_SHORT, 46| | // 1110____ ________ 47| 8.98k| TOO_SHORT | OVERLONG_3 | SURROGATE, 48| | // 1111____ ________ 49| 8.98k| TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); 50| 8.98k| constexpr const uint8_t CARRY = 51| 8.98k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 52| 8.98k| const simd8 byte_1_low = 53| 8.98k| (prev1 & 0x0F) 54| 8.98k| .lookup_16( 55| | // ____0000 ________ 56| 8.98k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 57| | // ____0001 ________ 58| 8.98k| CARRY | OVERLONG_2, 59| | // ____001_ ________ 60| 8.98k| CARRY, CARRY, 61| | 62| | // ____0100 ________ 63| 8.98k| CARRY | TOO_LARGE, 64| | // ____0101 ________ 65| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 66| | // ____011_ ________ 67| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 68| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 69| | 70| | // ____1___ ________ 71| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 72| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 73| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 74| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 75| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 76| | // ____1101 ________ 77| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, 78| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000, 79| 8.98k| CARRY | TOO_LARGE | TOO_LARGE_1000); 80| 8.98k| const simd8 byte_2_high = input.shr<4>().lookup_16( 81| | // ________ 0_______ 82| 8.98k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 83| 8.98k| TOO_SHORT, TOO_SHORT, 84| | 85| | // ________ 1000____ 86| 8.98k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 87| 8.98k| OVERLONG_4, 88| | // ________ 1001____ 89| 8.98k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 90| | // ________ 101_____ 91| 8.98k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 92| 8.98k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 93| | 94| | // ________ 11______ 95| 8.98k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 96| 8.98k| return (byte_1_high & byte_1_low & byte_2_high); 97| 8.98k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1623check_multibyte_lengthsENS1_4simd5simd8IhEES5_S5_: 101| 8.98k| const simd8 sc) { 102| 8.98k| simd8 prev2 = input.prev<2>(prev_input); 103| 8.98k| simd8 prev3 = input.prev<3>(prev_input); 104| 8.98k| simd8 must23 = 105| 8.98k| simd8(must_be_2_3_continuation(prev2, prev3)); 106| 8.98k| simd8 must23_80 = must23 & uint8_t(0x80); 107| 8.98k| return must23_80 ^ sc; 108| 8.98k|} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoder6errorsEv: 366| 1.86k| simdutf_really_inline bool errors() const { 367| 1.86k| return this->error.any_bits_set_anywhere(); 368| 1.86k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoder18convert_ascii_tailILNS_10endiannessE0EEEmPKcmmPDs: 131| 737| char16_t *output) { 132| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 133| | // tails. 134| 737| if (size - pos >= 8) { ------------------ | Branch (134:9): [True: 737, False: 0] ------------------ 135| 737| uint64_t first_bytes; 136| 737| std::memcpy(&first_bytes, in + pos, 8); 137| 737| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (137:11): [True: 356, False: 381] ------------------ 138| 356| return 0; 139| 356| } 140| 737| } 141| 381| const size_t start = pos; 142| 429| while (pos + 64 <= size) { ------------------ | Branch (142:12): [True: 186, False: 243] ------------------ 143| 186| simd8x64 input(reinterpret_cast(in + pos)); 144| 186| if (!input.is_ascii()) { ------------------ | Branch (144:11): [True: 138, False: 48] ------------------ 145| 138| return pos - start; 146| 138| } 147| 48| input.store_ascii_as_utf16(output + (pos - start)); 148| 48| pos += 64; 149| 48| } 150| 243| if (pos == size || size < 64) { ------------------ | Branch (150:9): [True: 18, False: 225] | Branch (150:24): [True: 0, False: 225] ------------------ 151| 18| return pos - start; 152| 18| } 153| 225| simd8x64 input(reinterpret_cast(in + size - 64)); 154| 225| if (!input.is_ascii()) { ------------------ | Branch (154:9): [True: 146, False: 79] ------------------ 155| 146| return pos - start; 156| 146| } 157| 79| input.store_ascii_as_utf16(output + (pos - start) - 158| 79| (pos - (size - 64))); 159| 79| return size - start; 160| 225| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoder7convertILNS_10endiannessE1EEEmPKcmPDs: 164| 13.6k| char16_t *utf16_output) { 165| 13.6k| if (size < 64) { // too short for the main loop ------------------ | Branch (165:9): [True: 12.7k, False: 927] ------------------ 166| 12.7k| return scalar::utf8_to_utf16::convert(in, size, utf16_output); 167| 12.7k| } 168| 927| size_t pos = 0; 169| 927| char16_t *start{utf16_output}; 170| | // In the worst case, we have the haswell kernel which can cause an overflow 171| | // of 8 bytes when calling convert_masked_utf8_to_utf16. If you skip the 172| | // last 16 bytes, and if the data is valid, then it is entirely safe because 173| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 174| | // generally assume that you have valid UTF-8 input, so we are going to go 175| | // back from the end counting 8 leading bytes, to give us a good margin. 176| 927| size_t leading_byte = 0; 177| 927| size_t margin = size; 178| 20.0k| for (; margin > 0 && leading_byte < 8; margin--) { ------------------ | Branch (178:12): [True: 19.9k, False: 14] | Branch (178:26): [True: 19.0k, False: 913] ------------------ 179| 19.0k| leading_byte += (int8_t(in[margin - 1]) > -65); 180| 19.0k| } 181| | // If the input is long enough, then we have that margin-1 is the eight last 182| | // leading byte. 183| 927| const size_t safety_margin = size - margin + 1; // to avoid overruns! 184| 3.81k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (184:12): [True: 2.90k, False: 916] ------------------ 185| 2.90k| simd8x64 input(reinterpret_cast(in + pos)); 186| 2.90k| if (input.is_ascii()) { ------------------ | Branch (186:11): [True: 699, False: 2.20k] ------------------ 187| 699| input.store_ascii_as_utf16(utf16_output); 188| 699| utf16_output += 64; 189| 699| pos += 64; 190| 2.20k| } else { 191| | // you might think that a for-loop would work, but under Visual Studio, 192| | // it is not good enough. 193| 2.20k| static_assert( 194| 2.20k| (simd8x64::NUM_CHUNKS == 2) || 195| 2.20k| (simd8x64::NUM_CHUNKS == 4), 196| 2.20k| "We support either two or four chunks per 64-byte block."); 197| 2.20k| auto zero = simd8{uint8_t(0)}; 198| 2.20k| if constexpr (simd8x64::NUM_CHUNKS == 2) { 199| 2.20k| this->check_utf8_bytes(input.chunks[0], zero); 200| 2.20k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 201| | } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 202| | this->check_utf8_bytes(input.chunks[0], zero); 203| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 204| | this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 205| | this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 206| | } 207| 2.20k| uint64_t utf8_continuation_mask = input.lt(-65 + 1); 208| 2.20k| if (utf8_continuation_mask & 1) { ------------------ | Branch (208:13): [True: 11, False: 2.19k] ------------------ 209| 11| return 0; // error 210| 11| } 211| 2.19k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 212| 2.19k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 213| | // We process in blocks of up to 12 bytes except possibly 214| | // for fast paths which may process up to 16 bytes. For the 215| | // slow path to work, we should have at least 12 input bytes left. 216| 2.19k| size_t max_starting_point = (pos + 64) - 12; 217| | // Next loop is going to run at least five times. 218| 18.5k| while (pos < max_starting_point) { ------------------ | Branch (218:16): [True: 16.3k, False: 2.19k] ------------------ 219| | // Performance note: our ability to compute 'consumed' and 220| | // then shift and recompute is critical. If there is a 221| | // latency of, say, 4 cycles on getting 'consumed', then 222| | // the inner loop might have a total latency of about 6 cycles. 223| | // Yet we process between 6 to 12 inputs bytes, thus we get 224| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 225| | // for this section of the code. Hence, there is a limit 226| | // to how much we can further increase this latency before 227| | // it seriously harms performance. 228| 16.3k| size_t consumed = convert_masked_utf8_to_utf16( 229| 16.3k| in + pos, utf8_end_of_code_point_mask, utf16_output); 230| 16.3k| pos += consumed; 231| 16.3k| utf8_end_of_code_point_mask >>= consumed; 232| 16.3k| } 233| | // At this point there may remain between 0 and 12 bytes in the 234| | // 64-byte block. These bytes will be processed again. So we have an 235| | // 80% efficiency (in the worst case). In practice we expect an 236| | // 85% to 90% efficiency. 237| 2.19k| } 238| 2.90k| } 239| 916| if (errors()) { ------------------ | Branch (239:9): [True: 209, False: 707] ------------------ 240| 209| return 0; 241| 209| } 242| 707| const size_t ascii = 243| 707| convert_ascii_tail(in, pos, size, utf16_output); 244| 707| pos += ascii; 245| 707| utf16_output += ascii; 246| 707| if (pos == size) { ------------------ | Branch (246:9): [True: 99, False: 608] ------------------ 247| 99| return utf16_output - start; 248| 99| } 249| 608| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 608|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 250| 608| size_t howmany = scalar::utf8_to_utf16::convert( 251| 608| in + pos, size - pos, utf16_output); 252| 608| if (howmany == 0) { ------------------ | Branch (252:9): [True: 281, False: 327] ------------------ 253| 281| return 0; 254| 281| } 255| 327| return utf16_output + howmany - start; 256| 608| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf1621validating_transcoder18convert_ascii_tailILNS_10endiannessE1EEEmPKcmmPDs: 131| 707| char16_t *output) { 132| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 133| | // tails. 134| 707| if (size - pos >= 8) { ------------------ | Branch (134:9): [True: 707, False: 0] ------------------ 135| 707| uint64_t first_bytes; 136| 707| std::memcpy(&first_bytes, in + pos, 8); 137| 707| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (137:11): [True: 315, False: 392] ------------------ 138| 315| return 0; 139| 315| } 140| 707| } 141| 392| const size_t start = pos; 142| 441| while (pos + 64 <= size) { ------------------ | Branch (142:12): [True: 188, False: 253] ------------------ 143| 188| simd8x64 input(reinterpret_cast(in + pos)); 144| 188| if (!input.is_ascii()) { ------------------ | Branch (144:11): [True: 139, False: 49] ------------------ 145| 139| return pos - start; 146| 139| } 147| 49| input.store_ascii_as_utf16(output + (pos - start)); 148| 49| pos += 64; 149| 49| } 150| 253| if (pos == size || size < 64) { ------------------ | Branch (150:9): [True: 20, False: 233] | Branch (150:24): [True: 0, False: 233] ------------------ 151| 20| return pos - start; 152| 20| } 153| 233| simd8x64 input(reinterpret_cast(in + size - 64)); 154| 233| if (!input.is_ascii()) { ------------------ | Branch (154:9): [True: 154, False: 79] ------------------ 155| 154| return pos - start; 156| 154| } 157| 79| input.store_ascii_as_utf16(output + (pos - start) - 158| 79| (pos - (size - 64))); 159| 79| return size - start; 160| 233| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoder7convertILNS_10endiannessE0EEEmPKcmPDs: 164| 13.5k| char16_t *utf16_output) { 165| 13.5k| if (size < 64) { // too short for the main loop ------------------ | Branch (165:9): [True: 12.7k, False: 860] ------------------ 166| 12.7k| return scalar::utf8_to_utf16::convert(in, size, utf16_output); 167| 12.7k| } 168| 860| size_t pos = 0; 169| 860| char16_t *start{utf16_output}; 170| | // In the worst case, we have the haswell kernel which can cause an overflow 171| | // of 8 bytes when calling convert_masked_utf8_to_utf16. If you skip the 172| | // last 16 bytes, and if the data is valid, then it is entirely safe because 173| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 174| | // generally assume that you have valid UTF-8 input, so we are going to go 175| | // back from the end counting 8 leading bytes, to give us a good margin. 176| 860| size_t leading_byte = 0; 177| 860| size_t margin = size; 178| 15.5k| for (; margin > 0 && leading_byte < 8; margin--) { ------------------ | Branch (178:12): [True: 15.5k, False: 10] | Branch (178:26): [True: 14.7k, False: 850] ------------------ 179| 14.7k| leading_byte += (int8_t(in[margin - 1]) > -65); 180| 14.7k| } 181| | // If the input is long enough, then we have that margin-1 is the eight last 182| | // leading byte. 183| 860| const size_t safety_margin = size - margin + 1; // to avoid overruns! 184| 4.72k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (184:12): [True: 3.88k, False: 844] ------------------ 185| 3.88k| simd8x64 input(reinterpret_cast(in + pos)); 186| 3.88k| if (input.is_ascii()) { ------------------ | Branch (186:11): [True: 879, False: 3.00k] ------------------ 187| 879| input.store_ascii_as_utf16(utf16_output); 188| 879| utf16_output += 64; 189| 879| pos += 64; 190| 3.00k| } else { 191| | // you might think that a for-loop would work, but under Visual Studio, 192| | // it is not good enough. 193| 3.00k| static_assert( 194| 3.00k| (simd8x64::NUM_CHUNKS == 2) || 195| 3.00k| (simd8x64::NUM_CHUNKS == 4), 196| 3.00k| "We support either two or four chunks per 64-byte block."); 197| 3.00k| auto zero = simd8{uint8_t(0)}; 198| | if constexpr (simd8x64::NUM_CHUNKS == 2) { 199| | this->check_utf8_bytes(input.chunks[0], zero); 200| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 201| 3.00k| } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 202| 3.00k| this->check_utf8_bytes(input.chunks[0], zero); 203| 3.00k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 204| 3.00k| this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 205| 3.00k| this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 206| 3.00k| } 207| 3.00k| uint64_t utf8_continuation_mask = input.lt(-65 + 1); 208| 3.00k| if (utf8_continuation_mask & 1) { ------------------ | Branch (208:13): [True: 16, False: 2.98k] ------------------ 209| 16| return 0; // error 210| 16| } 211| 2.98k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 212| 2.98k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 213| | // We process in blocks of up to 12 bytes except possibly 214| | // for fast paths which may process up to 16 bytes. For the 215| | // slow path to work, we should have at least 12 input bytes left. 216| 2.98k| size_t max_starting_point = (pos + 64) - 12; 217| | // Next loop is going to run at least five times. 218| 24.1k| while (pos < max_starting_point) { ------------------ | Branch (218:16): [True: 21.1k, False: 2.98k] ------------------ 219| | // Performance note: our ability to compute 'consumed' and 220| | // then shift and recompute is critical. If there is a 221| | // latency of, say, 4 cycles on getting 'consumed', then 222| | // the inner loop might have a total latency of about 6 cycles. 223| | // Yet we process between 6 to 12 inputs bytes, thus we get 224| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 225| | // for this section of the code. Hence, there is a limit 226| | // to how much we can further increase this latency before 227| | // it seriously harms performance. 228| 21.1k| size_t consumed = convert_masked_utf8_to_utf16( 229| 21.1k| in + pos, utf8_end_of_code_point_mask, utf16_output); 230| 21.1k| pos += consumed; 231| 21.1k| utf8_end_of_code_point_mask >>= consumed; 232| 21.1k| } 233| | // At this point there may remain between 0 and 12 bytes in the 234| | // 64-byte block. These bytes will be processed again. So we have an 235| | // 80% efficiency (in the worst case). In practice we expect an 236| | // 85% to 90% efficiency. 237| 2.98k| } 238| 3.88k| } 239| 844| if (errors()) { ------------------ | Branch (239:9): [True: 233, False: 611] ------------------ 240| 233| return 0; 241| 233| } 242| 611| const size_t ascii = 243| 611| convert_ascii_tail(in, pos, size, utf16_output); 244| 611| pos += ascii; 245| 611| utf16_output += ascii; 246| 611| if (pos == size) { ------------------ | Branch (246:9): [True: 114, False: 497] ------------------ 247| 114| return utf16_output - start; 248| 114| } 249| 497| SIMDUTF_VZEROUPPER(); ------------------ | | 16| 497|#define SIMDUTF_VZEROUPPER() (void)0 ------------------ 250| 497| size_t howmany = scalar::utf8_to_utf16::convert( 251| 497| in + pos, size - pos, utf16_output); 252| 497| if (howmany == 0) { ------------------ | Branch (252:9): [True: 185, False: 312] ------------------ 253| 185| return 0; 254| 185| } 255| 312| return utf16_output + howmany - start; 256| 497| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoder16check_utf8_bytesENS1_4simd5simd8IhEES6_: 119| 23.9k| const simd8 prev_input) { 120| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 121| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 122| | // small negative numbers) 123| 23.9k| simd8 prev1 = input.prev<1>(prev_input); 124| 23.9k| simd8 sc = check_special_cases(input, prev1); 125| 23.9k| this->error |= check_multibyte_lengths(input, prev_input, sc); 126| 23.9k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1619check_special_casesENS1_4simd5simd8IhEES5_: 8| 23.9k|check_special_cases(const simd8 input, const simd8 prev1) { 9| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 10| | // Bit 1 = Too Long (ASCII followed by continuation) 11| | // Bit 2 = Overlong 3-byte 12| | // Bit 4 = Surrogate 13| | // Bit 5 = Overlong 2-byte 14| | // Bit 7 = Two Continuations 15| 23.9k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 16| | // 11______ 11______ 17| 23.9k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 18| 23.9k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 19| 23.9k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 20| 23.9k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 21| 23.9k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 22| 23.9k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 23| | // 11110100 101_____ 24| | // 11110101 1001____ 25| | // 11110101 101_____ 26| | // 1111011_ 1001____ 27| | // 1111011_ 101_____ 28| | // 11111___ 1001____ 29| | // 11111___ 101_____ 30| 23.9k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 31| | // 11110101 1000____ 32| | // 1111011_ 1000____ 33| | // 11111___ 1000____ 34| 23.9k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 35| | 36| 23.9k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 37| | // 0_______ ________ 38| 23.9k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 39| 23.9k| TOO_LONG, 40| | // 10______ ________ 41| 23.9k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 42| | // 1100____ ________ 43| 23.9k| TOO_SHORT | OVERLONG_2, 44| | // 1101____ ________ 45| 23.9k| TOO_SHORT, 46| | // 1110____ ________ 47| 23.9k| TOO_SHORT | OVERLONG_3 | SURROGATE, 48| | // 1111____ ________ 49| 23.9k| TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); 50| 23.9k| constexpr const uint8_t CARRY = 51| 23.9k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 52| 23.9k| const simd8 byte_1_low = 53| 23.9k| (prev1 & 0x0F) 54| 23.9k| .lookup_16( 55| | // ____0000 ________ 56| 23.9k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 57| | // ____0001 ________ 58| 23.9k| CARRY | OVERLONG_2, 59| | // ____001_ ________ 60| 23.9k| CARRY, CARRY, 61| | 62| | // ____0100 ________ 63| 23.9k| CARRY | TOO_LARGE, 64| | // ____0101 ________ 65| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 66| | // ____011_ ________ 67| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 68| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 69| | 70| | // ____1___ ________ 71| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 72| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 73| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 74| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 75| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 76| | // ____1101 ________ 77| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, 78| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 79| 23.9k| CARRY | TOO_LARGE | TOO_LARGE_1000); 80| 23.9k| const simd8 byte_2_high = input.shr<4>().lookup_16( 81| | // ________ 0_______ 82| 23.9k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 83| 23.9k| TOO_SHORT, TOO_SHORT, 84| | 85| | // ________ 1000____ 86| 23.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 87| 23.9k| OVERLONG_4, 88| | // ________ 1001____ 89| 23.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 90| | // ________ 101_____ 91| 23.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 92| 23.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 93| | 94| | // ________ 11______ 95| 23.9k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 96| 23.9k| return (byte_1_high & byte_1_low & byte_2_high); 97| 23.9k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1623check_multibyte_lengthsENS1_4simd5simd8IhEES5_S5_: 101| 23.9k| const simd8 sc) { 102| 23.9k| simd8 prev2 = input.prev<2>(prev_input); 103| 23.9k| simd8 prev3 = input.prev<3>(prev_input); 104| 23.9k| simd8 must23 = 105| 23.9k| simd8(must_be_2_3_continuation(prev2, prev3)); 106| 23.9k| simd8 must23_80 = must23 & uint8_t(0x80); 107| 23.9k| return must23_80 ^ sc; 108| 23.9k|} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoder6errorsEv: 366| 1.66k| simdutf_really_inline bool errors() const { 367| 1.66k| return this->error.any_bits_set_anywhere(); 368| 1.66k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoder18convert_ascii_tailILNS_10endiannessE0EEEmPKcmmPDs: 131| 611| char16_t *output) { 132| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 133| | // tails. 134| 611| if (size - pos >= 8) { ------------------ | Branch (134:9): [True: 611, False: 0] ------------------ 135| 611| uint64_t first_bytes; 136| 611| std::memcpy(&first_bytes, in + pos, 8); 137| 611| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (137:11): [True: 345, False: 266] ------------------ 138| 345| return 0; 139| 345| } 140| 611| } 141| 266| const size_t start = pos; 142| 328| while (pos + 64 <= size) { ------------------ | Branch (142:12): [True: 135, False: 193] ------------------ 143| 135| simd8x64 input(reinterpret_cast(in + pos)); 144| 135| if (!input.is_ascii()) { ------------------ | Branch (144:11): [True: 73, False: 62] ------------------ 145| 73| return pos - start; 146| 73| } 147| 62| input.store_ascii_as_utf16(output + (pos - start)); 148| 62| pos += 64; 149| 62| } 150| 193| if (pos == size || size < 64) { ------------------ | Branch (150:9): [True: 28, False: 165] | Branch (150:24): [True: 0, False: 165] ------------------ 151| 28| return pos - start; 152| 28| } 153| 165| simd8x64 input(reinterpret_cast(in + size - 64)); 154| 165| if (!input.is_ascii()) { ------------------ | Branch (154:9): [True: 79, False: 86] ------------------ 155| 79| return pos - start; 156| 79| } 157| 86| input.store_ascii_as_utf16(output + (pos - start) - 158| 86| (pos - (size - 64))); 159| 86| return size - start; 160| 165| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoder7convertILNS_10endiannessE1EEEmPKcmPDs: 164| 13.6k| char16_t *utf16_output) { 165| 13.6k| if (size < 64) { // too short for the main loop ------------------ | Branch (165:9): [True: 12.7k, False: 833] ------------------ 166| 12.7k| return scalar::utf8_to_utf16::convert(in, size, utf16_output); 167| 12.7k| } 168| 833| size_t pos = 0; 169| 833| char16_t *start{utf16_output}; 170| | // In the worst case, we have the haswell kernel which can cause an overflow 171| | // of 8 bytes when calling convert_masked_utf8_to_utf16. If you skip the 172| | // last 16 bytes, and if the data is valid, then it is entirely safe because 173| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 174| | // generally assume that you have valid UTF-8 input, so we are going to go 175| | // back from the end counting 8 leading bytes, to give us a good margin. 176| 833| size_t leading_byte = 0; 177| 833| size_t margin = size; 178| 15.0k| for (; margin > 0 && leading_byte < 8; margin--) { ------------------ | Branch (178:12): [True: 15.0k, False: 10] | Branch (178:26): [True: 14.2k, False: 823] ------------------ 179| 14.2k| leading_byte += (int8_t(in[margin - 1]) > -65); 180| 14.2k| } 181| | // If the input is long enough, then we have that margin-1 is the eight last 182| | // leading byte. 183| 833| const size_t safety_margin = size - margin + 1; // to avoid overruns! 184| 4.66k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (184:12): [True: 3.85k, False: 817] ------------------ 185| 3.85k| simd8x64 input(reinterpret_cast(in + pos)); 186| 3.85k| if (input.is_ascii()) { ------------------ | Branch (186:11): [True: 877, False: 2.97k] ------------------ 187| 877| input.store_ascii_as_utf16(utf16_output); 188| 877| utf16_output += 64; 189| 877| pos += 64; 190| 2.97k| } else { 191| | // you might think that a for-loop would work, but under Visual Studio, 192| | // it is not good enough. 193| 2.97k| static_assert( 194| 2.97k| (simd8x64::NUM_CHUNKS == 2) || 195| 2.97k| (simd8x64::NUM_CHUNKS == 4), 196| 2.97k| "We support either two or four chunks per 64-byte block."); 197| 2.97k| auto zero = simd8{uint8_t(0)}; 198| | if constexpr (simd8x64::NUM_CHUNKS == 2) { 199| | this->check_utf8_bytes(input.chunks[0], zero); 200| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 201| 2.97k| } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 202| 2.97k| this->check_utf8_bytes(input.chunks[0], zero); 203| 2.97k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 204| 2.97k| this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 205| 2.97k| this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 206| 2.97k| } 207| 2.97k| uint64_t utf8_continuation_mask = input.lt(-65 + 1); 208| 2.97k| if (utf8_continuation_mask & 1) { ------------------ | Branch (208:13): [True: 16, False: 2.95k] ------------------ 209| 16| return 0; // error 210| 16| } 211| 2.95k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 212| 2.95k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 213| | // We process in blocks of up to 12 bytes except possibly 214| | // for fast paths which may process up to 16 bytes. For the 215| | // slow path to work, we should have at least 12 input bytes left. 216| 2.95k| size_t max_starting_point = (pos + 64) - 12; 217| | // Next loop is going to run at least five times. 218| 24.1k| while (pos < max_starting_point) { ------------------ | Branch (218:16): [True: 21.2k, False: 2.95k] ------------------ 219| | // Performance note: our ability to compute 'consumed' and 220| | // then shift and recompute is critical. If there is a 221| | // latency of, say, 4 cycles on getting 'consumed', then 222| | // the inner loop might have a total latency of about 6 cycles. 223| | // Yet we process between 6 to 12 inputs bytes, thus we get 224| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 225| | // for this section of the code. Hence, there is a limit 226| | // to how much we can further increase this latency before 227| | // it seriously harms performance. 228| 21.2k| size_t consumed = convert_masked_utf8_to_utf16( 229| 21.2k| in + pos, utf8_end_of_code_point_mask, utf16_output); 230| 21.2k| pos += consumed; 231| 21.2k| utf8_end_of_code_point_mask >>= consumed; 232| 21.2k| } 233| | // At this point there may remain between 0 and 12 bytes in the 234| | // 64-byte block. These bytes will be processed again. So we have an 235| | // 80% efficiency (in the worst case). In practice we expect an 236| | // 85% to 90% efficiency. 237| 2.95k| } 238| 3.85k| } 239| 817| if (errors()) { ------------------ | Branch (239:9): [True: 233, False: 584] ------------------ 240| 233| return 0; 241| 233| } 242| 584| const size_t ascii = 243| 584| convert_ascii_tail(in, pos, size, utf16_output); 244| 584| pos += ascii; 245| 584| utf16_output += ascii; 246| 584| if (pos == size) { ------------------ | Branch (246:9): [True: 113, False: 471] ------------------ 247| 113| return utf16_output - start; 248| 113| } 249| 471| SIMDUTF_VZEROUPPER(); ------------------ | | 16| 471|#define SIMDUTF_VZEROUPPER() (void)0 ------------------ 250| 471| size_t howmany = scalar::utf8_to_utf16::convert( 251| 471| in + pos, size - pos, utf16_output); 252| 471| if (howmany == 0) { ------------------ | Branch (252:9): [True: 185, False: 286] ------------------ 253| 185| return 0; 254| 185| } 255| 286| return utf16_output + howmany - start; 256| 471| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf1621validating_transcoder18convert_ascii_tailILNS_10endiannessE1EEEmPKcmmPDs: 131| 584| char16_t *output) { 132| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 133| | // tails. 134| 584| if (size - pos >= 8) { ------------------ | Branch (134:9): [True: 584, False: 0] ------------------ 135| 584| uint64_t first_bytes; 136| 584| std::memcpy(&first_bytes, in + pos, 8); 137| 584| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (137:11): [True: 324, False: 260] ------------------ 138| 324| return 0; 139| 324| } 140| 584| } 141| 260| const size_t start = pos; 142| 322| while (pos + 64 <= size) { ------------------ | Branch (142:12): [True: 134, False: 188] ------------------ 143| 134| simd8x64 input(reinterpret_cast(in + pos)); 144| 134| if (!input.is_ascii()) { ------------------ | Branch (144:11): [True: 72, False: 62] ------------------ 145| 72| return pos - start; 146| 72| } 147| 62| input.store_ascii_as_utf16(output + (pos - start)); 148| 62| pos += 64; 149| 62| } 150| 188| if (pos == size || size < 64) { ------------------ | Branch (150:9): [True: 28, False: 160] | Branch (150:24): [True: 0, False: 160] ------------------ 151| 28| return pos - start; 152| 28| } 153| 160| simd8x64 input(reinterpret_cast(in + size - 64)); 154| 160| if (!input.is_ascii()) { ------------------ | Branch (154:9): [True: 75, False: 85] ------------------ 155| 75| return pos - start; 156| 75| } 157| 85| input.store_ascii_as_utf16(output + (pos - start) - 158| 85| (pos - (size - 64))); 159| 85| return size - start; 160| 160| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3221validating_transcoderC2Ev: 114| 6.85k| validating_transcoder() : error(uint8_t(0)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3221validating_transcoder7convertEPKcmPDi: 161| 6.85k| char32_t *utf32_output) { 162| 6.85k| if (size < 64) { // too short for the main loop ------------------ | Branch (162:9): [True: 6.10k, False: 745] ------------------ 163| 6.10k| return scalar::utf8_to_utf32::convert(in, size, utf32_output); 164| 6.10k| } 165| 745| size_t pos = 0; 166| 745| char32_t *start{utf32_output}; 167| | // In the worst case, we have the haswell kernel which can cause an overflow 168| | // of 8 words when calling convert_masked_utf8_to_utf32. If you skip the 169| | // last 16 bytes, and if the data is valid, then it is entirely safe because 170| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 171| | // generally assume that you have valid UTF-8 input, so we are going to go 172| | // back from the end counting 16 leading bytes, to give us a good margin. 173| 745| size_t leading_byte = 0; 174| 745| size_t margin = size; 175| 15.9k| for (; margin > 0 && leading_byte < 8; margin--) { ------------------ | Branch (175:12): [True: 15.9k, False: 14] | Branch (175:26): [True: 15.2k, False: 731] ------------------ 176| 15.2k| leading_byte += (int8_t(in[margin - 1]) > -65); 177| 15.2k| } 178| | // If the input is long enough, then we have that margin-1 is the fourth 179| | // last leading byte. 180| 745| const size_t safety_margin = size - margin + 1; // to avoid overruns! 181| 2.72k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (181:12): [True: 1.99k, False: 734] ------------------ 182| 1.99k| simd8x64 input(reinterpret_cast(in + pos)); 183| 1.99k| if (input.is_ascii()) { ------------------ | Branch (183:11): [True: 675, False: 1.31k] ------------------ 184| 675| input.store_ascii_as_utf32(utf32_output); 185| 675| utf32_output += 64; 186| 675| pos += 64; 187| 1.31k| } else { 188| | // you might think that a for-loop would work, but under Visual Studio, 189| | // it is not good enough. 190| 1.31k| static_assert( 191| 1.31k| (simd8x64::NUM_CHUNKS == 2) || 192| 1.31k| (simd8x64::NUM_CHUNKS == 4), 193| 1.31k| "We support either two or four chunks per 64-byte block."); 194| 1.31k| auto zero = simd8{uint8_t(0)}; 195| 1.31k| if constexpr (simd8x64::NUM_CHUNKS == 2) { 196| 1.31k| this->check_utf8_bytes(input.chunks[0], zero); 197| 1.31k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 198| | } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 199| | this->check_utf8_bytes(input.chunks[0], zero); 200| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 201| | this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 202| | this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 203| | } 204| 1.31k| uint64_t utf8_continuation_mask = input.lt(-65 + 1); 205| 1.31k| if (utf8_continuation_mask & 1) { ------------------ | Branch (205:13): [True: 11, False: 1.30k] ------------------ 206| 11| return 0; // we have an error 207| 11| } 208| 1.30k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 209| 1.30k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 210| | // We process in blocks of up to 12 bytes except possibly 211| | // for fast paths which may process up to 16 bytes. For the 212| | // slow path to work, we should have at least 12 input bytes left. 213| 1.30k| size_t max_starting_point = (pos + 64) - 12; 214| | // Next loop is going to run at least five times. 215| 13.0k| while (pos < max_starting_point) { ------------------ | Branch (215:16): [True: 11.7k, False: 1.30k] ------------------ 216| | // Performance note: our ability to compute 'consumed' and 217| | // then shift and recompute is critical. If there is a 218| | // latency of, say, 4 cycles on getting 'consumed', then 219| | // the inner loop might have a total latency of about 6 cycles. 220| | // Yet we process between 6 to 12 inputs bytes, thus we get 221| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 222| | // for this section of the code. Hence, there is a limit 223| | // to how much we can further increase this latency before 224| | // it seriously harms performance. 225| 11.7k| size_t consumed = convert_masked_utf8_to_utf32( 226| 11.7k| in + pos, utf8_end_of_code_point_mask, utf32_output); 227| 11.7k| pos += consumed; 228| 11.7k| utf8_end_of_code_point_mask >>= consumed; 229| 11.7k| } 230| | // At this point there may remain between 0 and 12 bytes in the 231| | // 64-byte block. These bytes will be processed again. So we have an 232| | // 80% efficiency (in the worst case). In practice we expect an 233| | // 85% to 90% efficiency. 234| 1.30k| } 235| 1.99k| } 236| 734| if (errors()) { ------------------ | Branch (236:9): [True: 209, False: 525] ------------------ 237| 209| return 0; 238| 209| } 239| 525| const size_t ascii = convert_ascii_tail(in, pos, size, utf32_output); 240| 525| pos += ascii; 241| 525| utf32_output += ascii; 242| 525| if (pos == size) { ------------------ | Branch (242:9): [True: 91, False: 434] ------------------ 243| 91| return utf32_output - start; 244| 91| } 245| 434| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 434|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 246| 434| size_t howmany = 247| 434| scalar::utf8_to_utf32::convert(in + pos, size - pos, utf32_output); 248| 434| if (howmany == 0) { ------------------ | Branch (248:9): [True: 281, False: 153] ------------------ 249| 281| return 0; 250| 281| } 251| 153| return utf32_output + howmany - start; 252| 434| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3221validating_transcoder16check_utf8_bytesENS1_4simd5simd8IhEES6_: 119| 2.63k| const simd8 prev_input) { 120| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 121| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 122| | // small negative numbers) 123| 2.63k| simd8 prev1 = input.prev<1>(prev_input); 124| 2.63k| simd8 sc = check_special_cases(input, prev1); 125| 2.63k| this->error |= check_multibyte_lengths(input, prev_input, sc); 126| 2.63k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3219check_special_casesENS1_4simd5simd8IhEES5_: 8| 2.63k|check_special_cases(const simd8 input, const simd8 prev1) { 9| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 10| | // Bit 1 = Too Long (ASCII followed by continuation) 11| | // Bit 2 = Overlong 3-byte 12| | // Bit 4 = Surrogate 13| | // Bit 5 = Overlong 2-byte 14| | // Bit 7 = Two Continuations 15| 2.63k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 16| | // 11______ 11______ 17| 2.63k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 18| 2.63k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 19| 2.63k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 20| 2.63k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 21| 2.63k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 22| 2.63k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 23| | // 11110100 101_____ 24| | // 11110101 1001____ 25| | // 11110101 101_____ 26| | // 1111011_ 1001____ 27| | // 1111011_ 101_____ 28| | // 11111___ 1001____ 29| | // 11111___ 101_____ 30| 2.63k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 31| | // 11110101 1000____ 32| | // 1111011_ 1000____ 33| | // 11111___ 1000____ 34| 2.63k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 35| | 36| 2.63k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 37| | // 0_______ ________ 38| 2.63k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 39| 2.63k| TOO_LONG, 40| | // 10______ ________ 41| 2.63k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 42| | // 1100____ ________ 43| 2.63k| TOO_SHORT | OVERLONG_2, 44| | // 1101____ ________ 45| 2.63k| TOO_SHORT, 46| | // 1110____ ________ 47| 2.63k| TOO_SHORT | OVERLONG_3 | SURROGATE, 48| | // 1111____ ________ 49| 2.63k| TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); 50| 2.63k| constexpr const uint8_t CARRY = 51| 2.63k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 52| 2.63k| const simd8 byte_1_low = 53| 2.63k| (prev1 & 0x0F) 54| 2.63k| .lookup_16( 55| | // ____0000 ________ 56| 2.63k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 57| | // ____0001 ________ 58| 2.63k| CARRY | OVERLONG_2, 59| | // ____001_ ________ 60| 2.63k| CARRY, CARRY, 61| | 62| | // ____0100 ________ 63| 2.63k| CARRY | TOO_LARGE, 64| | // ____0101 ________ 65| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 66| | // ____011_ ________ 67| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 68| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 69| | 70| | // ____1___ ________ 71| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 72| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 73| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 74| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 75| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 76| | // ____1101 ________ 77| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, 78| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000, 79| 2.63k| CARRY | TOO_LARGE | TOO_LARGE_1000); 80| 2.63k| const simd8 byte_2_high = input.shr<4>().lookup_16( 81| | // ________ 0_______ 82| 2.63k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 83| 2.63k| TOO_SHORT, TOO_SHORT, 84| | 85| | // ________ 1000____ 86| 2.63k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 87| 2.63k| OVERLONG_4, 88| | // ________ 1001____ 89| 2.63k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 90| | // ________ 101_____ 91| 2.63k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 92| 2.63k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 93| | 94| | // ________ 11______ 95| 2.63k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 96| 2.63k| return (byte_1_high & byte_1_low & byte_2_high); 97| 2.63k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3223check_multibyte_lengthsENS1_4simd5simd8IhEES5_S5_: 101| 2.63k| const simd8 sc) { 102| 2.63k| simd8 prev2 = input.prev<2>(prev_input); 103| 2.63k| simd8 prev3 = input.prev<3>(prev_input); 104| 2.63k| simd8 must23 = 105| 2.63k| simd8(must_be_2_3_continuation(prev2, prev3)); 106| 2.63k| simd8 must23_80 = must23 & uint8_t(0x80); 107| 2.63k| return must23_80 ^ sc; 108| 2.63k|} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3221validating_transcoder6errorsEv: 355| 734| simdutf_really_inline bool errors() const { 356| 734| return this->error.any_bits_set_anywhere(); 357| 734| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113utf8_to_utf3221validating_transcoder18convert_ascii_tailEPKcmmPDi: 130| 525| char32_t *output) { 131| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 132| | // tails. 133| 525| if (size - pos >= 8) { ------------------ | Branch (133:9): [True: 525, False: 0] ------------------ 134| 525| uint64_t first_bytes; 135| 525| std::memcpy(&first_bytes, in + pos, 8); 136| 525| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (136:11): [True: 159, False: 366] ------------------ 137| 159| return 0; 138| 159| } 139| 525| } 140| 366| const size_t start = pos; 141| 410| while (pos + 64 <= size) { ------------------ | Branch (141:12): [True: 177, False: 233] ------------------ 142| 177| simd8x64 input(reinterpret_cast(in + pos)); 143| 177| if (!input.is_ascii()) { ------------------ | Branch (143:11): [True: 133, False: 44] ------------------ 144| 133| return pos - start; 145| 133| } 146| 44| input.store_ascii_as_utf32(output + (pos - start)); 147| 44| pos += 64; 148| 44| } 149| 233| if (pos == size || size < 64) { ------------------ | Branch (149:9): [True: 16, False: 217] | Branch (149:24): [True: 0, False: 217] ------------------ 150| 16| return pos - start; 151| 16| } 152| 217| simd8x64 input(reinterpret_cast(in + size - 64)); 153| 217| if (!input.is_ascii()) { ------------------ | Branch (153:9): [True: 142, False: 75] ------------------ 154| 142| return pos - start; 155| 142| } 156| 75| input.store_ascii_as_utf32(output + (pos - start) - (pos - (size - 64))); 157| 75| return size - start; 158| 217| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3221validating_transcoderC2Ev: 114| 6.85k| validating_transcoder() : error(uint8_t(0)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3221validating_transcoder7convertEPKcmPDi: 161| 6.85k| char32_t *utf32_output) { 162| 6.85k| if (size < 64) { // too short for the main loop ------------------ | Branch (162:9): [True: 6.18k, False: 670] ------------------ 163| 6.18k| return scalar::utf8_to_utf32::convert(in, size, utf32_output); 164| 6.18k| } 165| 670| size_t pos = 0; 166| 670| char32_t *start{utf32_output}; 167| | // In the worst case, we have the haswell kernel which can cause an overflow 168| | // of 8 words when calling convert_masked_utf8_to_utf32. If you skip the 169| | // last 16 bytes, and if the data is valid, then it is entirely safe because 170| | // 16 UTF-8 bytes generate much more than 8 bytes. However, you cannot 171| | // generally assume that you have valid UTF-8 input, so we are going to go 172| | // back from the end counting 16 leading bytes, to give us a good margin. 173| 670| size_t leading_byte = 0; 174| 670| size_t margin = size; 175| 11.4k| for (; margin > 0 && leading_byte < 8; margin--) { ------------------ | Branch (175:12): [True: 11.4k, False: 10] | Branch (175:26): [True: 10.7k, False: 660] ------------------ 176| 10.7k| leading_byte += (int8_t(in[margin - 1]) > -65); 177| 10.7k| } 178| | // If the input is long enough, then we have that margin-1 is the fourth 179| | // last leading byte. 180| 670| const size_t safety_margin = size - margin + 1; // to avoid overruns! 181| 3.28k| while (pos + 64 + safety_margin <= size) { ------------------ | Branch (181:12): [True: 2.62k, False: 654] ------------------ 182| 2.62k| simd8x64 input(reinterpret_cast(in + pos)); 183| 2.62k| if (input.is_ascii()) { ------------------ | Branch (183:11): [True: 854, False: 1.77k] ------------------ 184| 854| input.store_ascii_as_utf32(utf32_output); 185| 854| utf32_output += 64; 186| 854| pos += 64; 187| 1.77k| } else { 188| | // you might think that a for-loop would work, but under Visual Studio, 189| | // it is not good enough. 190| 1.77k| static_assert( 191| 1.77k| (simd8x64::NUM_CHUNKS == 2) || 192| 1.77k| (simd8x64::NUM_CHUNKS == 4), 193| 1.77k| "We support either two or four chunks per 64-byte block."); 194| 1.77k| auto zero = simd8{uint8_t(0)}; 195| | if constexpr (simd8x64::NUM_CHUNKS == 2) { 196| | this->check_utf8_bytes(input.chunks[0], zero); 197| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 198| 1.77k| } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 199| 1.77k| this->check_utf8_bytes(input.chunks[0], zero); 200| 1.77k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 201| 1.77k| this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 202| 1.77k| this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 203| 1.77k| } 204| 1.77k| uint64_t utf8_continuation_mask = input.lt(-65 + 1); 205| 1.77k| if (utf8_continuation_mask & 1) { ------------------ | Branch (205:13): [True: 16, False: 1.75k] ------------------ 206| 16| return 0; // we have an error 207| 16| } 208| 1.75k| uint64_t utf8_leading_mask = ~utf8_continuation_mask; 209| 1.75k| uint64_t utf8_end_of_code_point_mask = utf8_leading_mask >> 1; 210| | // We process in blocks of up to 12 bytes except possibly 211| | // for fast paths which may process up to 16 bytes. For the 212| | // slow path to work, we should have at least 12 input bytes left. 213| 1.75k| size_t max_starting_point = (pos + 64) - 12; 214| | // Next loop is going to run at least five times. 215| 16.6k| while (pos < max_starting_point) { ------------------ | Branch (215:16): [True: 14.9k, False: 1.75k] ------------------ 216| | // Performance note: our ability to compute 'consumed' and 217| | // then shift and recompute is critical. If there is a 218| | // latency of, say, 4 cycles on getting 'consumed', then 219| | // the inner loop might have a total latency of about 6 cycles. 220| | // Yet we process between 6 to 12 inputs bytes, thus we get 221| | // a speed limit between 1 cycle/byte and 0.5 cycle/byte 222| | // for this section of the code. Hence, there is a limit 223| | // to how much we can further increase this latency before 224| | // it seriously harms performance. 225| 14.9k| size_t consumed = convert_masked_utf8_to_utf32( 226| 14.9k| in + pos, utf8_end_of_code_point_mask, utf32_output); 227| 14.9k| pos += consumed; 228| 14.9k| utf8_end_of_code_point_mask >>= consumed; 229| 14.9k| } 230| | // At this point there may remain between 0 and 12 bytes in the 231| | // 64-byte block. These bytes will be processed again. So we have an 232| | // 80% efficiency (in the worst case). In practice we expect an 233| | // 85% to 90% efficiency. 234| 1.75k| } 235| 2.62k| } 236| 654| if (errors()) { ------------------ | Branch (236:9): [True: 233, False: 421] ------------------ 237| 233| return 0; 238| 233| } 239| 421| const size_t ascii = convert_ascii_tail(in, pos, size, utf32_output); 240| 421| pos += ascii; 241| 421| utf32_output += ascii; 242| 421| if (pos == size) { ------------------ | Branch (242:9): [True: 106, False: 315] ------------------ 243| 106| return utf32_output - start; 244| 106| } 245| 315| SIMDUTF_VZEROUPPER(); ------------------ | | 16| 315|#define SIMDUTF_VZEROUPPER() (void)0 ------------------ 246| 315| size_t howmany = 247| 315| scalar::utf8_to_utf32::convert(in + pos, size - pos, utf32_output); 248| 315| if (howmany == 0) { ------------------ | Branch (248:9): [True: 185, False: 130] ------------------ 249| 185| return 0; 250| 185| } 251| 130| return utf32_output + howmany - start; 252| 315| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3221validating_transcoder16check_utf8_bytesENS1_4simd5simd8IhEES6_: 119| 7.09k| const simd8 prev_input) { 120| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 121| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 122| | // small negative numbers) 123| 7.09k| simd8 prev1 = input.prev<1>(prev_input); 124| 7.09k| simd8 sc = check_special_cases(input, prev1); 125| 7.09k| this->error |= check_multibyte_lengths(input, prev_input, sc); 126| 7.09k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3219check_special_casesENS1_4simd5simd8IhEES5_: 8| 7.09k|check_special_cases(const simd8 input, const simd8 prev1) { 9| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 10| | // Bit 1 = Too Long (ASCII followed by continuation) 11| | // Bit 2 = Overlong 3-byte 12| | // Bit 4 = Surrogate 13| | // Bit 5 = Overlong 2-byte 14| | // Bit 7 = Two Continuations 15| 7.09k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 16| | // 11______ 11______ 17| 7.09k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 18| 7.09k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 19| 7.09k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 20| 7.09k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 21| 7.09k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 22| 7.09k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 23| | // 11110100 101_____ 24| | // 11110101 1001____ 25| | // 11110101 101_____ 26| | // 1111011_ 1001____ 27| | // 1111011_ 101_____ 28| | // 11111___ 1001____ 29| | // 11111___ 101_____ 30| 7.09k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 31| | // 11110101 1000____ 32| | // 1111011_ 1000____ 33| | // 11111___ 1000____ 34| 7.09k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 35| | 36| 7.09k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 37| | // 0_______ ________ 38| 7.09k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 39| 7.09k| TOO_LONG, 40| | // 10______ ________ 41| 7.09k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 42| | // 1100____ ________ 43| 7.09k| TOO_SHORT | OVERLONG_2, 44| | // 1101____ ________ 45| 7.09k| TOO_SHORT, 46| | // 1110____ ________ 47| 7.09k| TOO_SHORT | OVERLONG_3 | SURROGATE, 48| | // 1111____ ________ 49| 7.09k| TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); 50| 7.09k| constexpr const uint8_t CARRY = 51| 7.09k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 52| 7.09k| const simd8 byte_1_low = 53| 7.09k| (prev1 & 0x0F) 54| 7.09k| .lookup_16( 55| | // ____0000 ________ 56| 7.09k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 57| | // ____0001 ________ 58| 7.09k| CARRY | OVERLONG_2, 59| | // ____001_ ________ 60| 7.09k| CARRY, CARRY, 61| | 62| | // ____0100 ________ 63| 7.09k| CARRY | TOO_LARGE, 64| | // ____0101 ________ 65| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 66| | // ____011_ ________ 67| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 68| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 69| | 70| | // ____1___ ________ 71| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 72| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 73| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 74| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 75| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 76| | // ____1101 ________ 77| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, 78| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000, 79| 7.09k| CARRY | TOO_LARGE | TOO_LARGE_1000); 80| 7.09k| const simd8 byte_2_high = input.shr<4>().lookup_16( 81| | // ________ 0_______ 82| 7.09k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 83| 7.09k| TOO_SHORT, TOO_SHORT, 84| | 85| | // ________ 1000____ 86| 7.09k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 87| 7.09k| OVERLONG_4, 88| | // ________ 1001____ 89| 7.09k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 90| | // ________ 101_____ 91| 7.09k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 92| 7.09k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 93| | 94| | // ________ 11______ 95| 7.09k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 96| 7.09k| return (byte_1_high & byte_1_low & byte_2_high); 97| 7.09k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3223check_multibyte_lengthsENS1_4simd5simd8IhEES5_S5_: 101| 7.09k| const simd8 sc) { 102| 7.09k| simd8 prev2 = input.prev<2>(prev_input); 103| 7.09k| simd8 prev3 = input.prev<3>(prev_input); 104| 7.09k| simd8 must23 = 105| 7.09k| simd8(must_be_2_3_continuation(prev2, prev3)); 106| 7.09k| simd8 must23_80 = must23 & uint8_t(0x80); 107| 7.09k| return must23_80 ^ sc; 108| 7.09k|} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3221validating_transcoder6errorsEv: 355| 654| simdutf_really_inline bool errors() const { 356| 654| return this->error.any_bits_set_anywhere(); 357| 654| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113utf8_to_utf3221validating_transcoder18convert_ascii_tailEPKcmmPDi: 130| 421| char32_t *output) { 131| | // A cheap test on the first bytes avoids any SIMD work for most non-ASCII 132| | // tails. 133| 421| if (size - pos >= 8) { ------------------ | Branch (133:9): [True: 421, False: 0] ------------------ 134| 421| uint64_t first_bytes; 135| 421| std::memcpy(&first_bytes, in + pos, 8); 136| 421| if (first_bytes & 0x8080808080808080) { ------------------ | Branch (136:11): [True: 175, False: 246] ------------------ 137| 175| return 0; 138| 175| } 139| 421| } 140| 246| const size_t start = pos; 141| 302| while (pos + 64 <= size) { ------------------ | Branch (141:12): [True: 126, False: 176] ------------------ 142| 126| simd8x64 input(reinterpret_cast(in + pos)); 143| 126| if (!input.is_ascii()) { ------------------ | Branch (143:11): [True: 70, False: 56] ------------------ 144| 70| return pos - start; 145| 70| } 146| 56| input.store_ascii_as_utf32(output + (pos - start)); 147| 56| pos += 64; 148| 56| } 149| 176| if (pos == size || size < 64) { ------------------ | Branch (149:9): [True: 24, False: 152] | Branch (149:24): [True: 0, False: 152] ------------------ 150| 24| return pos - start; 151| 24| } 152| 152| simd8x64 input(reinterpret_cast(in + size - 64)); 153| 152| if (!input.is_ascii()) { ------------------ | Branch (153:9): [True: 70, False: 82] ------------------ 154| 70| return pos - start; 155| 70| } 156| 82| input.store_ascii_as_utf32(output + (pos - start) - (pos - (size - 64))); 157| 82| return size - start; 158| 152| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation12utf8_checker16check_next_inputERKNS1_4simd8simd8x64IhEE: 215| 15.7k| simdutf_really_inline bool check_next_input(const simd8x64 &input) { 216| 15.7k| if (simdutf_likely(is_ascii(input))) { ------------------ | | 90| 15.7k| #define simdutf_likely(x) __builtin_expect(!!(x), 1) | | ------------------ | | | Branch (90:31): [True: 4.25k, False: 11.4k] | | ------------------ ------------------ 217| 4.25k| this->error |= this->prev_incomplete; 218| 4.25k| return true; 219| 11.4k| } else { 220| | // you might think that a for-loop would work, but under Visual Studio, it 221| | // is not good enough. 222| 11.4k| static_assert((simd8x64::NUM_CHUNKS == 2) || 223| 11.4k| (simd8x64::NUM_CHUNKS == 4), 224| 11.4k| "We support either two or four chunks per 64-byte block."); 225| 11.4k| if constexpr (simd8x64::NUM_CHUNKS == 2) { 226| 11.4k| this->check_utf8_bytes(input.chunks[0], this->prev_input_block); 227| 11.4k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 228| | } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 229| | this->check_utf8_bytes(input.chunks[0], this->prev_input_block); 230| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 231| | this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 232| | this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 233| | } 234| 11.4k| this->prev_incomplete = 235| 11.4k| is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); 236| 11.4k| this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; 237| 11.4k| return false; 238| 11.4k| } 239| 15.7k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation12utf8_checker16check_utf8_bytesENS1_4simd5simd8IhEES6_: 195| 22.9k| const simd8 prev_input) { 196| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 197| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 198| | // small negative numbers) 199| 22.9k| simd8 prev1 = input.prev<1>(prev_input); 200| 22.9k| simd8 sc = check_special_cases(input, prev1); 201| 22.9k| this->error |= check_multibyte_lengths(input, prev_input, sc); 202| 22.9k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation19check_special_casesENS1_4simd5simd8IhEES5_: 9| 22.9k|check_special_cases(const simd8 input, const simd8 prev1) { 10| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 11| | // Bit 1 = Too Long (ASCII followed by continuation) 12| | // Bit 2 = Overlong 3-byte 13| | // Bit 4 = Surrogate 14| | // Bit 5 = Overlong 2-byte 15| | // Bit 7 = Two Continuations 16| 22.9k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 17| | // 11______ 11______ 18| 22.9k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 19| 22.9k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 20| 22.9k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 21| 22.9k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 22| 22.9k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 23| 22.9k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 24| | // 11110100 101_____ 25| | // 11110101 1001____ 26| | // 11110101 101_____ 27| | // 1111011_ 1001____ 28| | // 1111011_ 101_____ 29| | // 11111___ 1001____ 30| | // 11111___ 101_____ 31| 22.9k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 32| | // 11110101 1000____ 33| | // 1111011_ 1000____ 34| | // 11111___ 1000____ 35| 22.9k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 36| | 37| 22.9k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 38| | // 0_______ ________ 39| 22.9k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 40| 22.9k| TOO_LONG, 41| | // 10______ ________ 42| 22.9k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 43| | // 1100____ ________ 44| 22.9k| TOO_SHORT | OVERLONG_2, 45| | // 1101____ ________ 46| 22.9k| TOO_SHORT, 47| | // 1110____ ________ 48| 22.9k| TOO_SHORT | OVERLONG_3 | SURROGATE, 49| | // 1111____ ________ 50| 22.9k| TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); 51| 22.9k| constexpr const uint8_t CARRY = 52| 22.9k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 53| 22.9k| const simd8 byte_1_low = 54| 22.9k| (prev1 & 0x0F) 55| 22.9k| .lookup_16( 56| | // ____0000 ________ 57| 22.9k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 58| | // ____0001 ________ 59| 22.9k| CARRY | OVERLONG_2, 60| | // ____001_ ________ 61| 22.9k| CARRY, CARRY, 62| | 63| | // ____0100 ________ 64| 22.9k| CARRY | TOO_LARGE, 65| | // ____0101 ________ 66| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 67| | // ____011_ ________ 68| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 69| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 70| | 71| | // ____1___ ________ 72| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 73| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 74| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 75| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 76| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 77| | // ____1101 ________ 78| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, 79| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000, 80| 22.9k| CARRY | TOO_LARGE | TOO_LARGE_1000); 81| 22.9k| const simd8 byte_2_high = input.shr<4>().lookup_16( 82| | // ________ 0_______ 83| 22.9k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 84| 22.9k| TOO_SHORT, TOO_SHORT, 85| | 86| | // ________ 1000____ 87| 22.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 88| 22.9k| OVERLONG_4, 89| | // ________ 1001____ 90| 22.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 91| | // ________ 101_____ 92| 22.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 93| 22.9k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 94| | 95| | // ________ 11______ 96| 22.9k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 97| 22.9k| return (byte_1_high & byte_1_low & byte_2_high); 98| 22.9k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation23check_multibyte_lengthsENS1_4simd5simd8IhEES5_S5_: 102| 22.9k| const simd8 sc) { 103| 22.9k| simd8 prev2 = input.prev<2>(prev_input); 104| 22.9k| simd8 prev3 = input.prev<3>(prev_input); 105| 22.9k| simd8 must23 = 106| 22.9k| simd8(must_be_2_3_continuation(prev2, prev3)); 107| 22.9k| simd8 must23_80 = must23 & uint8_t(0x80); 108| 22.9k| return must23_80 ^ sc; 109| 22.9k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation13is_incompleteENS1_4simd5simd8IhEE: 115| 11.4k|simdutf_really_inline simd8 is_incomplete(const simd8 input) { 116| | // If the previous input's last 3 bytes match this, they're too short (they 117| | // ended at EOF): 118| | // ... 1111____ 111_____ 11______ 119| 11.4k| static const uint8_t max_array[32] = {255, 120| 11.4k| 255, 121| 11.4k| 255, 122| 11.4k| 255, 123| 11.4k| 255, 124| 11.4k| 255, 125| 11.4k| 255, 126| 11.4k| 255, 127| 11.4k| 255, 128| 11.4k| 255, 129| 11.4k| 255, 130| 11.4k| 255, 131| 11.4k| 255, 132| 11.4k| 255, 133| 11.4k| 255, 134| 11.4k| 255, 135| 11.4k| 255, 136| 11.4k| 255, 137| 11.4k| 255, 138| 11.4k| 255, 139| 11.4k| 255, 140| 11.4k| 255, 141| 11.4k| 255, 142| 11.4k| 255, 143| 11.4k| 255, 144| 11.4k| 255, 145| 11.4k| 255, 146| 11.4k| 255, 147| 11.4k| 255, 148| 11.4k| 0b11110000u - 1, 149| 11.4k| 0b11100000u - 1, 150| 11.4k| 0b11000000u - 1}; 151| 11.4k| const simd8 max_value( 152| 11.4k| &max_array[sizeof(max_array) - sizeof(simd8)]); 153| 11.4k| return input.gt_bits(max_value); 154| 11.4k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation12utf8_checker9check_eofEv: 207| 33.8k| simdutf_really_inline void check_eof() { 208| | // If the previous block had incomplete UTF-8 characters at the end, an 209| | // ASCII block can't possibly finish them. 210| 33.8k| this->error |= this->prev_incomplete; 211| 33.8k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_115utf8_validation12utf8_checker6errorsEv: 252| 35.2k| simdutf_really_inline bool errors() const { 253| 35.2k| return this->error.any_bits_set_anywhere(); 254| 35.2k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation12utf8_checker23check_overlapping_inputERKNS1_4simd8simd8x64IhEENS4_5simd8IhEE: 246| 1.20k| const simd8 prev_input) { 247| 1.20k| this->prev_input_block = prev_input; 248| 1.20k| this->check_next_input(input); 249| 1.20k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation12utf8_checker16check_next_inputERKNS1_4simd8simd8x64IhEE: 215| 16.0k| simdutf_really_inline bool check_next_input(const simd8x64 &input) { 216| 16.0k| if (simdutf_likely(is_ascii(input))) { ------------------ | | 90| 16.0k| #define simdutf_likely(x) __builtin_expect(!!(x), 1) | | ------------------ | | | Branch (90:31): [True: 4.37k, False: 11.7k] | | ------------------ ------------------ 217| 4.37k| this->error |= this->prev_incomplete; 218| 4.37k| return true; 219| 11.7k| } else { 220| | // you might think that a for-loop would work, but under Visual Studio, it 221| | // is not good enough. 222| 11.7k| static_assert((simd8x64::NUM_CHUNKS == 2) || 223| 11.7k| (simd8x64::NUM_CHUNKS == 4), 224| 11.7k| "We support either two or four chunks per 64-byte block."); 225| | if constexpr (simd8x64::NUM_CHUNKS == 2) { 226| | this->check_utf8_bytes(input.chunks[0], this->prev_input_block); 227| | this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 228| 11.7k| } else if constexpr (simd8x64::NUM_CHUNKS == 4) { 229| 11.7k| this->check_utf8_bytes(input.chunks[0], this->prev_input_block); 230| 11.7k| this->check_utf8_bytes(input.chunks[1], input.chunks[0]); 231| 11.7k| this->check_utf8_bytes(input.chunks[2], input.chunks[1]); 232| 11.7k| this->check_utf8_bytes(input.chunks[3], input.chunks[2]); 233| 11.7k| } 234| 11.7k| this->prev_incomplete = 235| 11.7k| is_incomplete(input.chunks[simd8x64::NUM_CHUNKS - 1]); 236| 11.7k| this->prev_input_block = input.chunks[simd8x64::NUM_CHUNKS - 1]; 237| 11.7k| return false; 238| 11.7k| } 239| 16.0k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation12utf8_checker16check_utf8_bytesENS1_4simd5simd8IhEES6_: 195| 46.8k| const simd8 prev_input) { 196| | // Flip prev1...prev3 so we can easily determine if they are 2+, 3+ or 4+ 197| | // lead bytes (2, 3, 4-byte leads become large positive numbers instead of 198| | // small negative numbers) 199| 46.8k| simd8 prev1 = input.prev<1>(prev_input); 200| 46.8k| simd8 sc = check_special_cases(input, prev1); 201| 46.8k| this->error |= check_multibyte_lengths(input, prev_input, sc); 202| 46.8k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation19check_special_casesENS1_4simd5simd8IhEES5_: 9| 46.8k|check_special_cases(const simd8 input, const simd8 prev1) { 10| | // Bit 0 = Too Short (lead byte/ASCII followed by lead byte/ASCII) 11| | // Bit 1 = Too Long (ASCII followed by continuation) 12| | // Bit 2 = Overlong 3-byte 13| | // Bit 4 = Surrogate 14| | // Bit 5 = Overlong 2-byte 15| | // Bit 7 = Two Continuations 16| 46.8k| constexpr const uint8_t TOO_SHORT = 1 << 0; // 11______ 0_______ 17| | // 11______ 11______ 18| 46.8k| constexpr const uint8_t TOO_LONG = 1 << 1; // 0_______ 10______ 19| 46.8k| constexpr const uint8_t OVERLONG_3 = 1 << 2; // 11100000 100_____ 20| 46.8k| constexpr const uint8_t SURROGATE = 1 << 4; // 11101101 101_____ 21| 46.8k| constexpr const uint8_t OVERLONG_2 = 1 << 5; // 1100000_ 10______ 22| 46.8k| constexpr const uint8_t TWO_CONTS = 1 << 7; // 10______ 10______ 23| 46.8k| constexpr const uint8_t TOO_LARGE = 1 << 3; // 11110100 1001____ 24| | // 11110100 101_____ 25| | // 11110101 1001____ 26| | // 11110101 101_____ 27| | // 1111011_ 1001____ 28| | // 1111011_ 101_____ 29| | // 11111___ 1001____ 30| | // 11111___ 101_____ 31| 46.8k| constexpr const uint8_t TOO_LARGE_1000 = 1 << 6; 32| | // 11110101 1000____ 33| | // 1111011_ 1000____ 34| | // 11111___ 1000____ 35| 46.8k| constexpr const uint8_t OVERLONG_4 = 1 << 6; // 11110000 1000____ 36| | 37| 46.8k| const simd8 byte_1_high = prev1.shr<4>().lookup_16( 38| | // 0_______ ________ 39| 46.8k| TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, TOO_LONG, 40| 46.8k| TOO_LONG, 41| | // 10______ ________ 42| 46.8k| TWO_CONTS, TWO_CONTS, TWO_CONTS, TWO_CONTS, 43| | // 1100____ ________ 44| 46.8k| TOO_SHORT | OVERLONG_2, 45| | // 1101____ ________ 46| 46.8k| TOO_SHORT, 47| | // 1110____ ________ 48| 46.8k| TOO_SHORT | OVERLONG_3 | SURROGATE, 49| | // 1111____ ________ 50| 46.8k| TOO_SHORT | TOO_LARGE | TOO_LARGE_1000 | OVERLONG_4); 51| 46.8k| constexpr const uint8_t CARRY = 52| 46.8k| TOO_SHORT | TOO_LONG | TWO_CONTS; // These all have ____ in byte 1 . 53| 46.8k| const simd8 byte_1_low = 54| 46.8k| (prev1 & 0x0F) 55| 46.8k| .lookup_16( 56| | // ____0000 ________ 57| 46.8k| CARRY | OVERLONG_3 | OVERLONG_2 | OVERLONG_4, 58| | // ____0001 ________ 59| 46.8k| CARRY | OVERLONG_2, 60| | // ____001_ ________ 61| 46.8k| CARRY, CARRY, 62| | 63| | // ____0100 ________ 64| 46.8k| CARRY | TOO_LARGE, 65| | // ____0101 ________ 66| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 67| | // ____011_ ________ 68| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 69| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 70| | 71| | // ____1___ ________ 72| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 73| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 74| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 75| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 76| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 77| | // ____1101 ________ 78| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000 | SURROGATE, 79| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000, 80| 46.8k| CARRY | TOO_LARGE | TOO_LARGE_1000); 81| 46.8k| const simd8 byte_2_high = input.shr<4>().lookup_16( 82| | // ________ 0_______ 83| 46.8k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT, 84| 46.8k| TOO_SHORT, TOO_SHORT, 85| | 86| | // ________ 1000____ 87| 46.8k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE_1000 | 88| 46.8k| OVERLONG_4, 89| | // ________ 1001____ 90| 46.8k| TOO_LONG | OVERLONG_2 | TWO_CONTS | OVERLONG_3 | TOO_LARGE, 91| | // ________ 101_____ 92| 46.8k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 93| 46.8k| TOO_LONG | OVERLONG_2 | TWO_CONTS | SURROGATE | TOO_LARGE, 94| | 95| | // ________ 11______ 96| 46.8k| TOO_SHORT, TOO_SHORT, TOO_SHORT, TOO_SHORT); 97| 46.8k| return (byte_1_high & byte_1_low & byte_2_high); 98| 46.8k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation23check_multibyte_lengthsENS1_4simd5simd8IhEES5_S5_: 102| 46.8k| const simd8 sc) { 103| 46.8k| simd8 prev2 = input.prev<2>(prev_input); 104| 46.8k| simd8 prev3 = input.prev<3>(prev_input); 105| 46.8k| simd8 must23 = 106| 46.8k| simd8(must_be_2_3_continuation(prev2, prev3)); 107| 46.8k| simd8 must23_80 = must23 & uint8_t(0x80); 108| 46.8k| return must23_80 ^ sc; 109| 46.8k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation13is_incompleteENS1_4simd5simd8IhEE: 115| 11.7k|simdutf_really_inline simd8 is_incomplete(const simd8 input) { 116| | // If the previous input's last 3 bytes match this, they're too short (they 117| | // ended at EOF): 118| | // ... 1111____ 111_____ 11______ 119| 11.7k| static const uint8_t max_array[32] = {255, 120| 11.7k| 255, 121| 11.7k| 255, 122| 11.7k| 255, 123| 11.7k| 255, 124| 11.7k| 255, 125| 11.7k| 255, 126| 11.7k| 255, 127| 11.7k| 255, 128| 11.7k| 255, 129| 11.7k| 255, 130| 11.7k| 255, 131| 11.7k| 255, 132| 11.7k| 255, 133| 11.7k| 255, 134| 11.7k| 255, 135| 11.7k| 255, 136| 11.7k| 255, 137| 11.7k| 255, 138| 11.7k| 255, 139| 11.7k| 255, 140| 11.7k| 255, 141| 11.7k| 255, 142| 11.7k| 255, 143| 11.7k| 255, 144| 11.7k| 255, 145| 11.7k| 255, 146| 11.7k| 255, 147| 11.7k| 255, 148| 11.7k| 0b11110000u - 1, 149| 11.7k| 0b11100000u - 1, 150| 11.7k| 0b11000000u - 1}; 151| 11.7k| const simd8 max_value( 152| 11.7k| &max_array[sizeof(max_array) - sizeof(simd8)]); 153| 11.7k| return input.gt_bits(max_value); 154| 11.7k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation12utf8_checker9check_eofEv: 207| 33.7k| simdutf_really_inline void check_eof() { 208| | // If the previous block had incomplete UTF-8 characters at the end, an 209| | // ASCII block can't possibly finish them. 210| 33.7k| this->error |= this->prev_incomplete; 211| 33.7k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_115utf8_validation12utf8_checker6errorsEv: 252| 35.4k| simdutf_really_inline bool errors() const { 253| 35.4k| return this->error.any_bits_set_anywhere(); 254| 35.4k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation12utf8_checker23check_overlapping_inputERKNS1_4simd8simd8x64IhEENS4_5simd8IhEE: 246| 1.29k| const simd8 prev_input) { 247| 1.29k| this->prev_input_block = prev_input; 248| 1.29k| this->check_next_input(input); 249| 1.29k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation21generic_validate_utf8EPKcm: 58| 27.2k|bool generic_validate_utf8(const char *input, size_t length) { 59| 27.2k| return generic_validate_utf8( 60| 27.2k| reinterpret_cast(input), length); 61| 27.2k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation21generic_validate_utf8INS2_12utf8_checkerEEEbPKhm: 46| 27.2k|bool generic_validate_utf8(const uint8_t *input, size_t length) { 47| 27.2k| checker c{}; 48| 27.2k| buf_block_reader<64> reader(input, length); 49| 36.6k| while (reader.has_full_block()) { ------------------ | Branch (49:10): [True: 9.41k, False: 27.2k] ------------------ 50| 9.41k| simd::simd8x64 in(reader.full_block()); 51| 9.41k| c.check_next_input(in); 52| 9.41k| reader.advance(); 53| 9.41k| } 54| 27.2k| check_remainder(c, input, length, reader.block_index()); 55| 27.2k| return !c.errors(); 56| 27.2k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation15check_remainderINS2_12utf8_checkerEEEvRT_PKhmm: 15| 33.8k| size_t length, size_t idx) { 16| 33.8k| constexpr size_t block = sizeof(simd::simd8x64); 17| 33.8k| constexpr size_t width = sizeof(simd::simd8); 18| 33.8k| static_assert(block == 8 * sizeof(uint64_t), "one mask bit per byte"); 19| 33.8k| if (length >= block + width) { ------------------ | Branch (19:7): [True: 1.67k, False: 32.1k] ------------------ 20| | // The window starts at or before idx: buf_block_reader leaves 1 to 64 21| | // bytes, so 0 <= idx - start < 64. 22| 1.67k| const size_t start = length - block; 23| 1.67k| const simd::simd8x64 in(input + start); 24| | // Bit i is set if byte start + i is not ASCII, so the bytes from idx on 25| | // are the top length - idx bits. is_ascii() goes first because GCC, up 26| | // to at least GCC 17, builds gteq_unsigned(0x80) from min and compare 27| | // instead of one movemask, which costs ASCII tails about 0.5 ns. 28| 1.67k| if (!in.is_ascii() && (in.gteq_unsigned(0x80) >> (idx - start)) != 0) { ------------------ | Branch (28:9): [True: 1.34k, False: 330] | Branch (28:27): [True: 1.20k, False: 134] ------------------ 29| 1.20k| c.check_overlapping_input(in, 30| 1.20k| simd::simd8(input + start - width)); 31| 1.20k| } 32| 32.1k| } else if (idx != length) { ------------------ | Branch (32:14): [True: 3.68k, False: 28.4k] ------------------ 33| | // Only an empty input leaves no bytes. Its pointer may be null, and 34| | // memcpy from a null pointer is undefined. 35| 3.68k| uint8_t buffer[block]{}; 36| 3.68k| std::memcpy(buffer, input + idx, length - idx); 37| 3.68k| c.check_next_input(simd::simd8x64(buffer)); 38| 3.68k| } 39| 33.8k| c.check_eof(); 40| 33.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation33generic_validate_utf8_with_errorsEPKcm: 104| 6.85k|result generic_validate_utf8_with_errors(const char *input, size_t length) { 105| 6.85k| return generic_validate_utf8_with_errors( 106| 6.85k| reinterpret_cast(input), length); 107| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115utf8_validation33generic_validate_utf8_with_errorsINS2_12utf8_checkerEEENS_6resultEPKhm: 67| 6.85k|result generic_validate_utf8_with_errors(const uint8_t *input, size_t length) { 68| 6.85k| checker c{}; 69| 6.85k| buf_block_reader<64> reader(input, length); 70| 6.85k| size_t count{0}; 71| 7.97k| while (reader.has_full_block()) { ------------------ | Branch (71:10): [True: 1.39k, False: 6.58k] ------------------ 72| 1.39k| simd::simd8x64 in(reader.full_block()); 73| 1.39k| c.check_next_input(in); 74| 1.39k| if (c.errors()) { ------------------ | Branch (74:9): [True: 273, False: 1.12k] ------------------ 75| 273| if (count != 0) { ------------------ | Branch (75:11): [True: 27, False: 246] ------------------ 76| 27| count--; 77| 27| } // Sometimes the error is only detected in the next chunk 78| 273| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 273|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 79| 273| result res = scalar::utf8::rewind_and_validate_with_errors( 80| 273| reinterpret_cast(input), 81| 273| reinterpret_cast(input + count), length - count); 82| 273| res.count += count; 83| 273| return res; 84| 273| } 85| 1.12k| reader.advance(); 86| 1.12k| count += 64; 87| 1.12k| } 88| 6.58k| check_remainder(c, input, length, reader.block_index()); 89| 6.58k| if (c.errors()) { ------------------ | Branch (89:7): [True: 561, False: 6.02k] ------------------ 90| 561| if (count != 0) { ------------------ | Branch (90:9): [True: 176, False: 385] ------------------ 91| 176| count--; 92| 176| } // Sometimes the error is only detected in the next chunk 93| 561| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 561|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 94| 561| result res = scalar::utf8::rewind_and_validate_with_errors( 95| 561| reinterpret_cast(input), 96| 561| reinterpret_cast(input) + count, length - count); 97| 561| res.count += count; 98| 561| return res; 99| 6.02k| } else { 100| 6.02k| return result(error_code::SUCCESS, length); 101| 6.02k| } 102| 6.58k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation21generic_validate_utf8EPKcm: 58| 27.2k|bool generic_validate_utf8(const char *input, size_t length) { 59| 27.2k| return generic_validate_utf8( 60| 27.2k| reinterpret_cast(input), length); 61| 27.2k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation21generic_validate_utf8INS2_12utf8_checkerEEEbPKhm: 46| 27.2k|bool generic_validate_utf8(const uint8_t *input, size_t length) { 47| 27.2k| checker c{}; 48| 27.2k| buf_block_reader<64> reader(input, length); 49| 38.0k| while (reader.has_full_block()) { ------------------ | Branch (49:10): [True: 10.8k, False: 27.2k] ------------------ 50| 10.8k| simd::simd8x64 in(reader.full_block()); 51| 10.8k| c.check_next_input(in); 52| 10.8k| reader.advance(); 53| 10.8k| } 54| 27.2k| check_remainder(c, input, length, reader.block_index()); 55| 27.2k| return !c.errors(); 56| 27.2k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation15check_remainderINS2_12utf8_checkerEEEvRT_PKhmm: 15| 33.7k| size_t length, size_t idx) { 16| 33.7k| constexpr size_t block = sizeof(simd::simd8x64); 17| 33.7k| constexpr size_t width = sizeof(simd::simd8); 18| 33.7k| static_assert(block == 8 * sizeof(uint64_t), "one mask bit per byte"); 19| 33.7k| if (length >= block + width) { ------------------ | Branch (19:7): [True: 1.83k, False: 31.9k] ------------------ 20| | // The window starts at or before idx: buf_block_reader leaves 1 to 64 21| | // bytes, so 0 <= idx - start < 64. 22| 1.83k| const size_t start = length - block; 23| 1.83k| const simd::simd8x64 in(input + start); 24| | // Bit i is set if byte start + i is not ASCII, so the bytes from idx on 25| | // are the top length - idx bits. is_ascii() goes first because GCC, up 26| | // to at least GCC 17, builds gteq_unsigned(0x80) from min and compare 27| | // instead of one movemask, which costs ASCII tails about 0.5 ns. 28| 1.83k| if (!in.is_ascii() && (in.gteq_unsigned(0x80) >> (idx - start)) != 0) { ------------------ | Branch (28:9): [True: 1.45k, False: 383] | Branch (28:27): [True: 1.29k, False: 159] ------------------ 29| 1.29k| c.check_overlapping_input(in, 30| 1.29k| simd::simd8(input + start - width)); 31| 1.29k| } 32| 31.9k| } else if (idx != length) { ------------------ | Branch (32:14): [True: 2.30k, False: 29.6k] ------------------ 33| | // Only an empty input leaves no bytes. Its pointer may be null, and 34| | // memcpy from a null pointer is undefined. 35| 2.30k| uint8_t buffer[block]{}; 36| 2.30k| std::memcpy(buffer, input + idx, length - idx); 37| 2.30k| c.check_next_input(simd::simd8x64(buffer)); 38| 2.30k| } 39| 33.7k| c.check_eof(); 40| 33.7k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation33generic_validate_utf8_with_errorsEPKcm: 104| 6.85k|result generic_validate_utf8_with_errors(const char *input, size_t length) { 105| 6.85k| return generic_validate_utf8_with_errors( 106| 6.85k| reinterpret_cast(input), length); 107| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115utf8_validation33generic_validate_utf8_with_errorsINS2_12utf8_checkerEEENS_6resultEPKhm: 67| 6.85k|result generic_validate_utf8_with_errors(const uint8_t *input, size_t length) { 68| 6.85k| checker c{}; 69| 6.85k| buf_block_reader<64> reader(input, length); 70| 6.85k| size_t count{0}; 71| 8.24k| while (reader.has_full_block()) { ------------------ | Branch (71:10): [True: 1.65k, False: 6.59k] ------------------ 72| 1.65k| simd::simd8x64 in(reader.full_block()); 73| 1.65k| c.check_next_input(in); 74| 1.65k| if (c.errors()) { ------------------ | Branch (74:9): [True: 263, False: 1.39k] ------------------ 75| 263| if (count != 0) { ------------------ | Branch (75:11): [True: 36, False: 227] ------------------ 76| 36| count--; 77| 36| } // Sometimes the error is only detected in the next chunk 78| 263| SIMDUTF_VZEROUPPER(); ------------------ | | 16| 263|#define SIMDUTF_VZEROUPPER() (void)0 ------------------ 79| 263| result res = scalar::utf8::rewind_and_validate_with_errors( 80| 263| reinterpret_cast(input), 81| 263| reinterpret_cast(input + count), length - count); 82| 263| res.count += count; 83| 263| return res; 84| 263| } 85| 1.39k| reader.advance(); 86| 1.39k| count += 64; 87| 1.39k| } 88| 6.59k| check_remainder(c, input, length, reader.block_index()); 89| 6.59k| if (c.errors()) { ------------------ | Branch (89:7): [True: 325, False: 6.26k] ------------------ 90| 325| if (count != 0) { ------------------ | Branch (90:9): [True: 143, False: 182] ------------------ 91| 143| count--; 92| 143| } // Sometimes the error is only detected in the next chunk 93| 325| SIMDUTF_VZEROUPPER(); ------------------ | | 16| 325|#define SIMDUTF_VZEROUPPER() (void)0 ------------------ 94| 325| result res = scalar::utf8::rewind_and_validate_with_errors( 95| 325| reinterpret_cast(input), 96| 325| reinterpret_cast(input) + count, length - count); 97| 325| res.count += count; 98| 325| return res; 99| 6.26k| } else { 100| 6.26k| return result(error_code::SUCCESS, length); 101| 6.26k| } 102| 6.59k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf1626validate_utf16_with_errorsILNS_10endiannessE0EEEKNS_6resultEPKDsm: 49| 9.78k|const result validate_utf16_with_errors(const char16_t *input, size_t size) { 50| 9.78k| if (simdutf_unlikely(size == 0)) { ------------------ | | 93| 9.78k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 6.41k, False: 3.37k] | | ------------------ ------------------ 51| 6.41k| return result(error_code::SUCCESS, 0); 52| 6.41k| } 53| | 54| 3.37k| const char16_t *start = input; 55| 3.37k| const char16_t *end = input + size; 56| | 57| 3.37k| const auto v_d8 = simd8::splat(0xd8); 58| 3.37k| const auto v_f8 = simd8::splat(0xf8); 59| 3.37k| const auto v_fc = simd8::splat(0xfc); 60| 3.37k| const auto v_dc = simd8::splat(0xdc); 61| | 62| 23.4k| while (input + simd16::SIZE * 2 < end) { ------------------ | Branch (62:10): [True: 20.1k, False: 3.27k] ------------------ 63| | // 0. Load data: since the validation takes into account only higher 64| | // byte of each word, we compress the two vectors into one which 65| | // consists only the higher bytes. 66| 20.1k| auto in0 = simd16(input); 67| 20.1k| auto in1 = 68| 20.1k| simd16(input + simd16::SIZE / sizeof(char16_t)); 69| | 70| | // Function `utf16_gather_high_bytes` consumes two vectors of UTF-16 71| | // and yields a single vector having only higher bytes of characters. 72| 20.1k| const auto in = utf16_gather_high_bytes(in0, in1); 73| | 74| | // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). 75| 20.1k| const auto surrogates_wordmask = (in & v_f8) == v_d8; 76| 20.1k| const uint16_t surrogates_bitmask = 77| 20.1k| static_cast(surrogates_wordmask.to_bitmask()); 78| 20.1k| if (surrogates_bitmask == 0x0000) { ------------------ | Branch (78:9): [True: 19.1k, False: 1.07k] ------------------ 79| 19.1k| input += 16; 80| 19.1k| } else { 81| | // 2. We have some surrogates that have to be distinguished: 82| | // - low surrogates: 0b1101'10xx'yyyy'yyyy (0xD800..0xDBFF) 83| | // - high surrogates: 0b1101'11xx'yyyy'yyyy (0xDC00..0xDFFF) 84| | // 85| | // Fact: high surrogate has 11th bit set (3rd bit in the higher byte) 86| | 87| | // V - non-surrogate code units 88| | // V = not surrogates_wordmask 89| 1.07k| const uint16_t V = static_cast(~surrogates_bitmask); 90| | 91| | // H - word-mask for high surrogates: the six highest bits are 0b1101'11 92| 1.07k| const auto vH = (in & v_fc) == v_dc; 93| 1.07k| const uint16_t H = static_cast(vH.to_bitmask()); 94| | 95| | // L - word mask for low surrogates 96| | // L = not H and surrogates_wordmask 97| 1.07k| const uint16_t L = static_cast(~H & surrogates_bitmask); 98| | 99| 1.07k| const uint16_t a = static_cast( 100| 1.07k| L & (H >> 1)); // A low surrogate must be followed by high one. 101| | // (A low surrogate placed in the 7th register's word 102| | // is an exception we handle.) 103| 1.07k| const uint16_t b = static_cast( 104| 1.07k| a << 1); // Just mark that the opinput - startite fact is hold, 105| | // thanks to that we have only two masks for valid case. 106| 1.07k| const uint16_t c = static_cast( 107| 1.07k| V | a | b); // Combine all the masks into the final one. 108| | 109| 1.07k| if (c == 0xffff) { ------------------ | Branch (109:11): [True: 653, False: 418] ------------------ 110| | // The whole input register contains valid UTF-16, i.e., 111| | // either single code units or proper surrogate pairs. 112| 653| input += 16; 113| 653| } else if (c == 0x7fff) { ------------------ | Branch (113:18): [True: 322, False: 96] ------------------ 114| | // The 15 lower code units of the input register contains valid UTF-16. 115| | // The 15th word may be either a low or high surrogate. It the next 116| | // iteration we 1) check if the low surrogate is followed by a high 117| | // one, 2) reject sole high surrogate. 118| 322| input += 15; 119| 322| } else { 120| 96| return result(error_code::SURROGATE, input - start); 121| 96| } 122| 1.07k| } 123| 20.1k| } 124| | 125| 3.27k| return result(error_code::SUCCESS, input - start); 126| 3.37k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf1626validate_utf16_with_errorsILNS_10endiannessE1EEEKNS_6resultEPKDsm: 49| 9.70k|const result validate_utf16_with_errors(const char16_t *input, size_t size) { 50| 9.70k| if (simdutf_unlikely(size == 0)) { ------------------ | | 93| 9.70k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 6.49k, False: 3.21k] | | ------------------ ------------------ 51| 6.49k| return result(error_code::SUCCESS, 0); 52| 6.49k| } 53| | 54| 3.21k| const char16_t *start = input; 55| 3.21k| const char16_t *end = input + size; 56| | 57| 3.21k| const auto v_d8 = simd8::splat(0xd8); 58| 3.21k| const auto v_f8 = simd8::splat(0xf8); 59| 3.21k| const auto v_fc = simd8::splat(0xfc); 60| 3.21k| const auto v_dc = simd8::splat(0xdc); 61| | 62| 23.1k| while (input + simd16::SIZE * 2 < end) { ------------------ | Branch (62:10): [True: 20.0k, False: 3.14k] ------------------ 63| | // 0. Load data: since the validation takes into account only higher 64| | // byte of each word, we compress the two vectors into one which 65| | // consists only the higher bytes. 66| 20.0k| auto in0 = simd16(input); 67| 20.0k| auto in1 = 68| 20.0k| simd16(input + simd16::SIZE / sizeof(char16_t)); 69| | 70| | // Function `utf16_gather_high_bytes` consumes two vectors of UTF-16 71| | // and yields a single vector having only higher bytes of characters. 72| 20.0k| const auto in = utf16_gather_high_bytes(in0, in1); 73| | 74| | // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). 75| 20.0k| const auto surrogates_wordmask = (in & v_f8) == v_d8; 76| 20.0k| const uint16_t surrogates_bitmask = 77| 20.0k| static_cast(surrogates_wordmask.to_bitmask()); 78| 20.0k| if (surrogates_bitmask == 0x0000) { ------------------ | Branch (78:9): [True: 19.0k, False: 999] ------------------ 79| 19.0k| input += 16; 80| 19.0k| } else { 81| | // 2. We have some surrogates that have to be distinguished: 82| | // - low surrogates: 0b1101'10xx'yyyy'yyyy (0xD800..0xDBFF) 83| | // - high surrogates: 0b1101'11xx'yyyy'yyyy (0xDC00..0xDFFF) 84| | // 85| | // Fact: high surrogate has 11th bit set (3rd bit in the higher byte) 86| | 87| | // V - non-surrogate code units 88| | // V = not surrogates_wordmask 89| 999| const uint16_t V = static_cast(~surrogates_bitmask); 90| | 91| | // H - word-mask for high surrogates: the six highest bits are 0b1101'11 92| 999| const auto vH = (in & v_fc) == v_dc; 93| 999| const uint16_t H = static_cast(vH.to_bitmask()); 94| | 95| | // L - word mask for low surrogates 96| | // L = not H and surrogates_wordmask 97| 999| const uint16_t L = static_cast(~H & surrogates_bitmask); 98| | 99| 999| const uint16_t a = static_cast( 100| 999| L & (H >> 1)); // A low surrogate must be followed by high one. 101| | // (A low surrogate placed in the 7th register's word 102| | // is an exception we handle.) 103| 999| const uint16_t b = static_cast( 104| 999| a << 1); // Just mark that the opinput - startite fact is hold, 105| | // thanks to that we have only two masks for valid case. 106| 999| const uint16_t c = static_cast( 107| 999| V | a | b); // Combine all the masks into the final one. 108| | 109| 999| if (c == 0xffff) { ------------------ | Branch (109:11): [True: 631, False: 368] ------------------ 110| | // The whole input register contains valid UTF-16, i.e., 111| | // either single code units or proper surrogate pairs. 112| 631| input += 16; 113| 631| } else if (c == 0x7fff) { ------------------ | Branch (113:18): [True: 298, False: 70] ------------------ 114| | // The 15 lower code units of the input register contains valid UTF-16. 115| | // The 15th word may be either a low or high surrogate. It the next 116| | // iteration we 1) check if the low surrogate is followed by a high 117| | // one, 2) reject sole high surrogate. 118| 298| input += 15; 119| 298| } else { 120| 70| return result(error_code::SURROGATE, input - start); 121| 70| } 122| 999| } 123| 20.0k| } 124| | 125| 3.14k| return result(error_code::SUCCESS, input - start); 126| 3.21k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf1626validate_utf16_with_errorsILNS_10endiannessE0EEEKNS_6resultEPKDsm: 49| 9.19k|const result validate_utf16_with_errors(const char16_t *input, size_t size) { 50| 9.19k| if (simdutf_unlikely(size == 0)) { ------------------ | | 93| 9.19k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 6.46k, False: 2.72k] | | ------------------ ------------------ 51| 6.46k| return result(error_code::SUCCESS, 0); 52| 6.46k| } 53| | 54| 2.72k| const char16_t *start = input; 55| 2.72k| const char16_t *end = input + size; 56| | 57| 2.72k| const auto v_d8 = simd8::splat(0xd8); 58| 2.72k| const auto v_f8 = simd8::splat(0xf8); 59| 2.72k| const auto v_fc = simd8::splat(0xfc); 60| 2.72k| const auto v_dc = simd8::splat(0xdc); 61| | 62| 29.1k| while (input + simd16::SIZE * 2 < end) { ------------------ | Branch (62:10): [True: 26.5k, False: 2.60k] ------------------ 63| | // 0. Load data: since the validation takes into account only higher 64| | // byte of each word, we compress the two vectors into one which 65| | // consists only the higher bytes. 66| 26.5k| auto in0 = simd16(input); 67| 26.5k| auto in1 = 68| 26.5k| simd16(input + simd16::SIZE / sizeof(char16_t)); 69| | 70| | // Function `utf16_gather_high_bytes` consumes two vectors of UTF-16 71| | // and yields a single vector having only higher bytes of characters. 72| 26.5k| const auto in = utf16_gather_high_bytes(in0, in1); 73| | 74| | // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). 75| 26.5k| const auto surrogates_wordmask = (in & v_f8) == v_d8; 76| 26.5k| const uint16_t surrogates_bitmask = 77| 26.5k| static_cast(surrogates_wordmask.to_bitmask()); 78| 26.5k| if (surrogates_bitmask == 0x0000) { ------------------ | Branch (78:9): [True: 25.2k, False: 1.29k] ------------------ 79| 25.2k| input += 16; 80| 25.2k| } else { 81| | // 2. We have some surrogates that have to be distinguished: 82| | // - low surrogates: 0b1101'10xx'yyyy'yyyy (0xD800..0xDBFF) 83| | // - high surrogates: 0b1101'11xx'yyyy'yyyy (0xDC00..0xDFFF) 84| | // 85| | // Fact: high surrogate has 11th bit set (3rd bit in the higher byte) 86| | 87| | // V - non-surrogate code units 88| | // V = not surrogates_wordmask 89| 1.29k| const uint16_t V = static_cast(~surrogates_bitmask); 90| | 91| | // H - word-mask for high surrogates: the six highest bits are 0b1101'11 92| 1.29k| const auto vH = (in & v_fc) == v_dc; 93| 1.29k| const uint16_t H = static_cast(vH.to_bitmask()); 94| | 95| | // L - word mask for low surrogates 96| | // L = not H and surrogates_wordmask 97| 1.29k| const uint16_t L = static_cast(~H & surrogates_bitmask); 98| | 99| 1.29k| const uint16_t a = static_cast( 100| 1.29k| L & (H >> 1)); // A low surrogate must be followed by high one. 101| | // (A low surrogate placed in the 7th register's word 102| | // is an exception we handle.) 103| 1.29k| const uint16_t b = static_cast( 104| 1.29k| a << 1); // Just mark that the opinput - startite fact is hold, 105| | // thanks to that we have only two masks for valid case. 106| 1.29k| const uint16_t c = static_cast( 107| 1.29k| V | a | b); // Combine all the masks into the final one. 108| | 109| 1.29k| if (c == 0xffff) { ------------------ | Branch (109:11): [True: 838, False: 455] ------------------ 110| | // The whole input register contains valid UTF-16, i.e., 111| | // either single code units or proper surrogate pairs. 112| 838| input += 16; 113| 838| } else if (c == 0x7fff) { ------------------ | Branch (113:18): [True: 337, False: 118] ------------------ 114| | // The 15 lower code units of the input register contains valid UTF-16. 115| | // The 15th word may be either a low or high surrogate. It the next 116| | // iteration we 1) check if the low surrogate is followed by a high 117| | // one, 2) reject sole high surrogate. 118| 337| input += 15; 119| 337| } else { 120| 118| return result(error_code::SURROGATE, input - start); 121| 118| } 122| 1.29k| } 123| 26.5k| } 124| | 125| 2.60k| return result(error_code::SUCCESS, input - start); 126| 2.72k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf1626validate_utf16_with_errorsILNS_10endiannessE1EEEKNS_6resultEPKDsm: 49| 9.14k|const result validate_utf16_with_errors(const char16_t *input, size_t size) { 50| 9.14k| if (simdutf_unlikely(size == 0)) { ------------------ | | 93| 9.14k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 6.52k, False: 2.61k] | | ------------------ ------------------ 51| 6.52k| return result(error_code::SUCCESS, 0); 52| 6.52k| } 53| | 54| 2.61k| const char16_t *start = input; 55| 2.61k| const char16_t *end = input + size; 56| | 57| 2.61k| const auto v_d8 = simd8::splat(0xd8); 58| 2.61k| const auto v_f8 = simd8::splat(0xf8); 59| 2.61k| const auto v_fc = simd8::splat(0xfc); 60| 2.61k| const auto v_dc = simd8::splat(0xdc); 61| | 62| 28.8k| while (input + simd16::SIZE * 2 < end) { ------------------ | Branch (62:10): [True: 26.3k, False: 2.49k] ------------------ 63| | // 0. Load data: since the validation takes into account only higher 64| | // byte of each word, we compress the two vectors into one which 65| | // consists only the higher bytes. 66| 26.3k| auto in0 = simd16(input); 67| 26.3k| auto in1 = 68| 26.3k| simd16(input + simd16::SIZE / sizeof(char16_t)); 69| | 70| | // Function `utf16_gather_high_bytes` consumes two vectors of UTF-16 71| | // and yields a single vector having only higher bytes of characters. 72| 26.3k| const auto in = utf16_gather_high_bytes(in0, in1); 73| | 74| | // 1. Check whether we have any 0xD800..DFFF word (0b1101'1xxx'yyyy'yyyy). 75| 26.3k| const auto surrogates_wordmask = (in & v_f8) == v_d8; 76| 26.3k| const uint16_t surrogates_bitmask = 77| 26.3k| static_cast(surrogates_wordmask.to_bitmask()); 78| 26.3k| if (surrogates_bitmask == 0x0000) { ------------------ | Branch (78:9): [True: 25.1k, False: 1.24k] ------------------ 79| 25.1k| input += 16; 80| 25.1k| } else { 81| | // 2. We have some surrogates that have to be distinguished: 82| | // - low surrogates: 0b1101'10xx'yyyy'yyyy (0xD800..0xDBFF) 83| | // - high surrogates: 0b1101'11xx'yyyy'yyyy (0xDC00..0xDFFF) 84| | // 85| | // Fact: high surrogate has 11th bit set (3rd bit in the higher byte) 86| | 87| | // V - non-surrogate code units 88| | // V = not surrogates_wordmask 89| 1.24k| const uint16_t V = static_cast(~surrogates_bitmask); 90| | 91| | // H - word-mask for high surrogates: the six highest bits are 0b1101'11 92| 1.24k| const auto vH = (in & v_fc) == v_dc; 93| 1.24k| const uint16_t H = static_cast(vH.to_bitmask()); 94| | 95| | // L - word mask for low surrogates 96| | // L = not H and surrogates_wordmask 97| 1.24k| const uint16_t L = static_cast(~H & surrogates_bitmask); 98| | 99| 1.24k| const uint16_t a = static_cast( 100| 1.24k| L & (H >> 1)); // A low surrogate must be followed by high one. 101| | // (A low surrogate placed in the 7th register's word 102| | // is an exception we handle.) 103| 1.24k| const uint16_t b = static_cast( 104| 1.24k| a << 1); // Just mark that the opinput - startite fact is hold, 105| | // thanks to that we have only two masks for valid case. 106| 1.24k| const uint16_t c = static_cast( 107| 1.24k| V | a | b); // Combine all the masks into the final one. 108| | 109| 1.24k| if (c == 0xffff) { ------------------ | Branch (109:11): [True: 782, False: 467] ------------------ 110| | // The whole input register contains valid UTF-16, i.e., 111| | // either single code units or proper surrogate pairs. 112| 782| input += 16; 113| 782| } else if (c == 0x7fff) { ------------------ | Branch (113:18): [True: 345, False: 122] ------------------ 114| | // The 15 lower code units of the input register contains valid UTF-16. 115| | // The 15th word may be either a low or high surrogate. It the next 116| | // iteration we 1) check if the low surrogate is followed by a high 117| | // one, 2) reject sole high surrogate. 118| 345| input += 15; 119| 345| } else { 120| 122| return result(error_code::SURROGATE, input - start); 121| 122| } 122| 1.24k| } 123| 26.3k| } 124| | 125| 2.49k| return result(error_code::SUCCESS, input - start); 126| 2.61k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf328validateEPKDim: 6| 12.8k|simdutf_really_inline bool validate(const char32_t *input, size_t size) { 7| 12.8k| if (simdutf_unlikely(size == 0)) { ------------------ | | 93| 12.8k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 10.3k, False: 2.48k] | | ------------------ ------------------ 8| | // empty input is valid UTF-32. protect the implementation from 9| | // handling nullptr 10| 10.3k| return true; 11| 10.3k| } 12| | 13| 2.48k| const char32_t *end = input + size; 14| | 15| 2.48k| using vector_u32 = simd32; 16| | 17| 2.48k| const auto standardmax = vector_u32::splat(0x10ffff); 18| 2.48k| const auto offset = vector_u32::splat(0xffff2000); 19| 2.48k| const auto standardoffsetmax = vector_u32::splat(0xfffff7ff); 20| 2.48k| auto currentmax = vector_u32::zero(); 21| 2.48k| auto currentoffsetmax = vector_u32::zero(); 22| | 23| 2.48k| constexpr size_t N = vector_u32::ELEMENTS; 24| | 25| 48.6k| while (input + N < end) { ------------------ | Branch (25:10): [True: 46.1k, False: 2.48k] ------------------ 26| 46.1k| auto in = vector_u32(input); 27| 46.1k| if constexpr (!match_system(endianness::BIG)) { 28| 46.1k| in.swap_bytes(); 29| 46.1k| } 30| | 31| 46.1k| currentmax = max(currentmax, in); 32| 46.1k| currentoffsetmax = max(currentoffsetmax, in + offset); 33| 46.1k| input += N; 34| 46.1k| } 35| | 36| 2.48k| const auto too_large = currentmax > standardmax; 37| 2.48k| if (too_large.any()) { ------------------ | Branch (37:7): [True: 0, False: 2.48k] ------------------ 38| 0| return false; 39| 0| } 40| | 41| 2.48k| const auto surrogate = currentoffsetmax > standardoffsetmax; 42| 2.48k| if (surrogate.any()) { ------------------ | Branch (42:7): [True: 0, False: 2.48k] ------------------ 43| 0| return false; 44| 0| } 45| | 46| 2.48k| return scalar::utf32::validate(input, end - input); 47| 2.48k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_15utf328validateEPKDim: 6| 13.1k|simdutf_really_inline bool validate(const char32_t *input, size_t size) { 7| 13.1k| if (simdutf_unlikely(size == 0)) { ------------------ | | 93| 13.1k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 11.1k, False: 1.95k] | | ------------------ ------------------ 8| | // empty input is valid UTF-32. protect the implementation from 9| | // handling nullptr 10| 11.1k| return true; 11| 11.1k| } 12| | 13| 1.95k| const char32_t *end = input + size; 14| | 15| 1.95k| using vector_u32 = simd32; 16| | 17| 1.95k| const auto standardmax = vector_u32::splat(0x10ffff); 18| 1.95k| const auto offset = vector_u32::splat(0xffff2000); 19| 1.95k| const auto standardoffsetmax = vector_u32::splat(0xfffff7ff); 20| 1.95k| auto currentmax = vector_u32::zero(); 21| 1.95k| auto currentoffsetmax = vector_u32::zero(); 22| | 23| 1.95k| constexpr size_t N = vector_u32::ELEMENTS; 24| | 25| 101k| while (input + N < end) { ------------------ | Branch (25:10): [True: 99.6k, False: 1.95k] ------------------ 26| 99.6k| auto in = vector_u32(input); 27| 99.6k| if constexpr (!match_system(endianness::BIG)) { 28| 99.6k| in.swap_bytes(); 29| 99.6k| } 30| | 31| 99.6k| currentmax = max(currentmax, in); 32| 99.6k| currentoffsetmax = max(currentoffsetmax, in + offset); 33| 99.6k| input += N; 34| 99.6k| } 35| | 36| 1.95k| const auto too_large = currentmax > standardmax; 37| 1.95k| if (too_large.any()) { ------------------ | Branch (37:7): [True: 0, False: 1.95k] ------------------ 38| 0| return false; 39| 0| } 40| | 41| 1.95k| const auto surrogate = currentoffsetmax > standardoffsetmax; 42| 1.95k| if (surrogate.any()) { ------------------ | Branch (42:7): [True: 0, False: 1.95k] ------------------ 43| 0| return false; 44| 0| } 45| | 46| 1.95k| return scalar::utf32::validate(input, end - input); 47| 1.95k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block64C2EPKc: 488| 20.5k| simdutf_really_inline block64(const char *src) { 489| 20.5k| chunks[0] = _mm256_loadu_si256(reinterpret_cast(src)); 490| 20.5k| chunks[1] = _mm256_loadu_si256(reinterpret_cast(src + 32)); 491| 20.5k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6419base64_decode_blockILb0EEEvPc: 529| 4.85k| simdutf_really_inline void base64_decode_block(char *out) { 530| 4.85k| base64_decode(out, chunks[0]); 531| 4.85k| base64_decode(out + 24, chunks[1]); 532| 4.85k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113base64_decodeILb0EEEvPcDv4_x: 450| 21.6k|simdutf_really_inline void base64_decode(char *out, __m256i str) { 451| | // credit: aqrit 452| 21.6k| const __m256i pack_shuffle = 453| 21.6k| _mm256_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1, 454| 21.6k| 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1); 455| 21.6k| const __m256i t0 = _mm256_maddubs_epi16(str, _mm256_set1_epi32(0x01400140)); 456| 21.6k| const __m256i t1 = _mm256_madd_epi16(t0, _mm256_set1_epi32(0x00011000)); 457| 21.6k| const __m256i t2 = _mm256_shuffle_epi8(t1, pack_shuffle); 458| | 459| | // Store the output: 460| 21.6k| _mm_storeu_si128((__m128i *)out, _mm256_castsi256_si128(t2)); 461| 21.6k| const __m128i hi = _mm256_extracti128_si256(t2, 1); 462| | if constexpr (exact_tail) { 463| | _mm_storel_epi64((__m128i *)(out + 12), hi); 464| | const int32_t last = _mm_extract_epi32(hi, 2); 465| | std::memcpy(out + 20, &last, sizeof(last)); 466| 21.6k| } else { 467| 21.6k| _mm_storeu_si128((__m128i *)(out + 12), hi); 468| 21.6k| } 469| 21.6k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6419base64_decode_blockILb1EEEvPc: 529| 8.85k| simdutf_really_inline void base64_decode_block(char *out) { 530| 8.85k| base64_decode(out, chunks[0]); 531| 8.85k| base64_decode(out + 24, chunks[1]); 532| 8.85k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113base64_decodeILb1EEEvPcDv4_x: 450| 9.17k|simdutf_really_inline void base64_decode(char *out, __m256i str) { 451| | // credit: aqrit 452| 9.17k| const __m256i pack_shuffle = 453| 9.17k| _mm256_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1, 454| 9.17k| 2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1); 455| 9.17k| const __m256i t0 = _mm256_maddubs_epi16(str, _mm256_set1_epi32(0x01400140)); 456| 9.17k| const __m256i t1 = _mm256_madd_epi16(t0, _mm256_set1_epi32(0x00011000)); 457| 9.17k| const __m256i t2 = _mm256_shuffle_epi8(t1, pack_shuffle); 458| | 459| | // Store the output: 460| 9.17k| _mm_storeu_si128((__m128i *)out, _mm256_castsi256_si128(t2)); 461| 9.17k| const __m128i hi = _mm256_extracti128_si256(t2, 1); 462| 9.17k| if constexpr (exact_tail) { 463| 9.17k| _mm_storel_epi64((__m128i *)(out + 12), hi); 464| 9.17k| const int32_t last = _mm_extract_epi32(hi, 2); 465| 9.17k| std::memcpy(out + 20, &last, sizeof(last)); 466| | } else { 467| | _mm_storeu_si128((__m128i *)(out + 12), hi); 468| | } 469| 9.17k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6414compress_blockEmPc: 671| 2.81k| simdutf_really_inline uint64_t compress_block(uint64_t mask, char *output) { 672| 2.81k| if (is_power_of_two(mask)) { ------------------ | Branch (672:9): [True: 978, False: 1.83k] ------------------ 673| 978| return compress_block_single(mask, output); 674| 978| } 675| | 676| 1.83k| uint64_t nmask = ~mask; 677| 1.83k| compress(chunks[0], uint32_t(mask), output); 678| 1.83k| compress(chunks[1], uint32_t(mask >> 32), 679| 1.83k| output + count_ones(nmask & 0xFFFFFFFF)); 680| 1.83k| return count_ones(nmask); 681| 2.81k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6421compress_block_singleEmPc: 684| 978| char *output) { 685| 978| const size_t pos64 = trailing_zeroes(mask); 686| 978| const int8_t pos = pos64 & 0xf; 687| 978| switch (pos64 >> 4) { ------------------ | Branch (687:13): [True: 978, False: 0] ------------------ 688| 188| case 0b00: { ------------------ | Branch (688:5): [True: 188, False: 790] ------------------ 689| 188| const __m128i lane0 = _mm256_extracti128_si256(chunks[0], 0); 690| 188| const __m128i lane1 = _mm256_extracti128_si256(chunks[0], 1); 691| | 692| 188| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 693| 188| const __m128i v1 = 694| 188| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 695| 188| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 696| 188| const __m128i sh = _mm_sub_epi8(v1, v2); 697| 188| const __m128i compressed = _mm_shuffle_epi8(lane0, sh); 698| | 699| 188| _mm_storeu_si128((__m128i *)(output + 0 * 16), compressed); 700| 188| _mm_storeu_si128((__m128i *)(output + 1 * 16 - 1), lane1); 701| 188| _mm256_storeu_si256((__m256i *)(output + 2 * 16 - 1), chunks[1]); 702| 188| } break; 703| 239| case 0b01: { ------------------ | Branch (703:5): [True: 239, False: 739] ------------------ 704| 239| const __m128i lane0 = _mm256_extracti128_si256(chunks[0], 0); 705| 239| const __m128i lane1 = _mm256_extracti128_si256(chunks[0], 1); 706| 239| _mm_storeu_si128((__m128i *)(output + 0 * 16), lane0); 707| | 708| 239| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 709| 239| const __m128i v1 = 710| 239| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 711| 239| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 712| 239| const __m128i sh = _mm_sub_epi8(v1, v2); 713| 239| const __m128i compressed = _mm_shuffle_epi8(lane1, sh); 714| | 715| 239| _mm_storeu_si128((__m128i *)(output + 1 * 16), compressed); 716| 239| _mm256_storeu_si256((__m256i *)(output + 2 * 16 - 1), chunks[1]); 717| 239| } break; 718| 266| case 0b10: { ------------------ | Branch (718:5): [True: 266, False: 712] ------------------ 719| 266| const __m128i lane2 = _mm256_extracti128_si256(chunks[1], 0); 720| 266| const __m128i lane3 = _mm256_extracti128_si256(chunks[1], 1); 721| | 722| 266| _mm256_storeu_si256((__m256i *)(output + 0 * 16), chunks[0]); 723| | 724| 266| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 725| 266| const __m128i v1 = 726| 266| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 727| 266| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 728| 266| const __m128i sh = _mm_sub_epi8(v1, v2); 729| 266| const __m128i compressed = _mm_shuffle_epi8(lane2, sh); 730| | 731| 266| _mm_storeu_si128((__m128i *)(output + 2 * 16), compressed); 732| 266| _mm_storeu_si128((__m128i *)(output + 3 * 16 - 1), lane3); 733| 266| } break; 734| 285| case 0b11: { ------------------ | Branch (734:5): [True: 285, False: 693] ------------------ 735| 285| const __m128i lane2 = _mm256_extracti128_si256(chunks[1], 0); 736| 285| const __m128i lane3 = _mm256_extracti128_si256(chunks[1], 1); 737| | 738| 285| _mm256_storeu_si256((__m256i *)(output + 0 * 16), chunks[0]); 739| 285| _mm_storeu_si128((__m128i *)(output + 2 * 16), lane2); 740| | 741| 285| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 742| 285| const __m128i v1 = 743| 285| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 744| 285| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 745| 285| const __m128i sh = _mm_sub_epi8(v1, v2); 746| 285| const __m128i compressed = _mm_shuffle_epi8(lane3, sh); 747| | 748| 285| _mm_storeu_si128((__m128i *)(output + 3 * 16), compressed); 749| 285| } break; 750| 978| } 751| | 752| 978| return 63; 753| 978| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_18compressIvEEvDv4_xjPc: 435| 3.67k|simdutf_really_inline void compress(__m256i data, uint32_t mask, char *output) { 436| 3.67k| if (mask == 0) { ------------------ | Branch (436:7): [True: 635, False: 3.04k] ------------------ 437| 635| _mm256_storeu_si256(reinterpret_cast<__m256i *>(output), data); 438| 635| return; 439| 635| } 440| 3.04k| compress(_mm256_castsi256_si128(data), uint16_t(mask), output); 441| | compress(_mm256_extracti128_si256(data, 1), uint16_t(mask >> 16), 442| 3.04k| output + count_ones(~mask & 0xFFFF)); 443| 3.04k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_18compressEDv2_xtPc: 398| 6.08k|static inline void compress(__m128i data, uint16_t mask, char *output) { 399| 6.08k| if (mask == 0) { ------------------ | Branch (399:7): [True: 1.25k, False: 4.83k] ------------------ 400| 1.25k| _mm_storeu_si128(reinterpret_cast<__m128i *>(output), data); 401| 1.25k| return; 402| 1.25k| } 403| | // this particular implementation was inspired by work done by @animetosho 404| | // we do it in two steps, first 8 bytes and then second 8 bytes 405| 4.83k| uint8_t mask1 = uint8_t(mask); // least significant 8 bits 406| 4.83k| uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits 407| | // next line just loads the 64-bit values thintable_epi8[mask1] and 408| | // thintable_epi8[mask2] into a 128-bit register, using only 409| | // two instructions on most compilers. 410| | 411| 4.83k| __m128i shufmask = _mm_set_epi64x(tables::base64::thintable_epi8[mask2], 412| 4.83k| tables::base64::thintable_epi8[mask1]); 413| | // we increment by 0x08 the second half of the mask 414| 4.83k| shufmask = 415| 4.83k| _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); 416| | // this is the version "nearly pruned" 417| 4.83k| __m128i pruned = _mm_shuffle_epi8(data, shufmask); 418| | // we still need to put the two halves together. 419| | // we compute the popcount of the first half: 420| 4.83k| int pop1 = tables::base64::BitsSetTable256mul2[mask1]; 421| | // then load the corresponding mask, what it does is to write 422| | // only the first pop1 bytes from the first 8 bytes, and then 423| | // it fills in with the bytes from the second 8 bytes + some filling 424| | // at the end. 425| 4.83k| __m128i compactmask = _mm_loadu_si128(reinterpret_cast( 426| 4.83k| tables::base64::pshufb_combine_table + pop1 * 8)); 427| 4.83k| __m128i answer = _mm_shuffle_epi8(pruned, compactmask); 428| | 429| 4.83k| _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); 430| 4.83k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6410copy_blockEPc: 522| 337| simdutf_really_inline void copy_block(char *output) { 523| 337| _mm256_storeu_si256(reinterpret_cast<__m256i *>(output), chunks[0]); 524| 337| _mm256_storeu_si256(reinterpret_cast<__m256i *>(output + 32), chunks[1]); 525| 337| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_119base64_decode_blockILb0EEEvPcPKc: 472| 1.38k|simdutf_really_inline void base64_decode_block(char *out, const char *src) { 473| 1.38k| base64_decode(out, 474| 1.38k| _mm256_loadu_si256(reinterpret_cast(src))); 475| 1.38k| base64_decode( 476| 1.38k| out + 24, 477| 1.38k| _mm256_loadu_si256(reinterpret_cast(src + 32))); 478| 1.38k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_119base64_decode_blockILb1EEEvPcPKc: 472| 313|simdutf_really_inline void base64_decode_block(char *out, const char *src) { 473| 313| base64_decode(out, 474| 313| _mm256_loadu_si256(reinterpret_cast(src))); 475| 313| base64_decode( 476| 313| out + 24, 477| 313| _mm256_loadu_si256(reinterpret_cast(src + 32))); 478| 313|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6414to_base64_maskILb0ELb0ELb0EEEmPm: 535| 20.5k| simdutf_really_inline uint64_t to_base64_mask(uint64_t *error) { 536| 20.5k| uint32_t err0 = 0; 537| 20.5k| uint32_t err1 = 0; 538| 20.5k| uint64_t m0 = to_base64_mask( 539| 20.5k| &chunks[0], &err0); 540| 20.5k| uint64_t m1 = to_base64_mask( 541| 20.5k| &chunks[1], &err1); 542| 20.5k| if (!ignore_garbage) { ------------------ | Branch (542:9): [True: 20.5k, Folded] ------------------ 543| 20.5k| *error = err0 | ((uint64_t)err1 << 32); 544| 20.5k| } 545| 20.5k| return m0 | (m1 << 32); 546| 20.5k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_17block6414to_base64_maskILb0ELb0ELb0EEEjPDv4_xPj: 549| 41.0k| simdutf_really_inline uint32_t to_base64_mask(__m256i *src, uint32_t *error) { 550| 41.0k| const __m256i ascii_space_tbl = 551| 41.0k| _mm256_setr_epi8(0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9, 0xa, 552| 41.0k| 0x0, 0xc, 0xd, 0x0, 0x0, 0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 553| 41.0k| 0x0, 0x0, 0x0, 0x9, 0xa, 0x0, 0xc, 0xd, 0x0, 0x0); 554| | // credit: aqrit 555| 41.0k| __m256i delta_asso; 556| 41.0k| if (default_or_url) { ------------------ | Branch (556:9): [Folded, False: 41.0k] ------------------ 557| 0| delta_asso = _mm256_setr_epi8( 558| 0| 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 559| 0| 0x00, 0x00, 0x11, 0x00, 0x16, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 560| 0| 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x16); 561| 41.0k| } else if (base64_url) { ------------------ | Branch (561:16): [Folded, False: 41.0k] ------------------ 562| 0| delta_asso = _mm256_setr_epi8(0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, 563| 0| 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF, 0x1, 0x1, 564| 0| 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, 0x0, 0x0, 565| 0| 0x0, 0x0, 0xF, 0x0, 0xF); 566| 41.0k| } else { 567| 41.0k| delta_asso = _mm256_setr_epi8( 568| 41.0k| 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 0x00, 0x00, 569| 41.0k| 0x00, 0x00, 0x0F, 0x00, 0x0F, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 570| 41.0k| 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x0F); 571| 41.0k| } 572| | 573| 41.0k| __m256i delta_values; 574| 41.0k| if (default_or_url) { ------------------ | Branch (574:9): [Folded, False: 41.0k] ------------------ 575| 0| delta_values = _mm256_setr_epi8( 576| 0| uint8_t(0xBF), uint8_t(0xE0), uint8_t(0xB9), uint8_t(0x13), 577| 0| uint8_t(0x04), uint8_t(0xBF), uint8_t(0xBF), uint8_t(0xB9), 578| 0| uint8_t(0xB9), uint8_t(0x00), uint8_t(0xFF), uint8_t(0x11), 579| 0| uint8_t(0xFF), uint8_t(0xBF), uint8_t(0x10), uint8_t(0xB9), 580| 0| uint8_t(0xBF), uint8_t(0xE0), uint8_t(0xB9), uint8_t(0x13), 581| 0| uint8_t(0x04), uint8_t(0xBF), uint8_t(0xBF), uint8_t(0xB9), 582| 0| uint8_t(0xB9), uint8_t(0x00), uint8_t(0xFF), uint8_t(0x11), 583| 0| uint8_t(0xFF), uint8_t(0xBF), uint8_t(0x10), uint8_t(0xB9)); 584| 41.0k| } else if (base64_url) { ------------------ | Branch (584:16): [Folded, False: 41.0k] ------------------ 585| 0| delta_values = _mm256_setr_epi8( 586| 0| 0x0, 0x0, 0x0, 0x13, 0x4, uint8_t(0xBF), uint8_t(0xBF), uint8_t(0xB9), 587| 0| uint8_t(0xB9), 0x0, 0x11, uint8_t(0xC3), uint8_t(0xBF), uint8_t(0xE0), 588| 0| uint8_t(0xB9), uint8_t(0xB9), 0x0, 0x0, 0x0, 0x13, 0x4, uint8_t(0xBF), 589| 0| uint8_t(0xBF), uint8_t(0xB9), uint8_t(0xB9), 0x0, 0x11, uint8_t(0xC3), 590| 0| uint8_t(0xBF), uint8_t(0xE0), uint8_t(0xB9), uint8_t(0xB9)); 591| 41.0k| } else { 592| 41.0k| delta_values = _mm256_setr_epi8( 593| 41.0k| int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), int8_t(0x04), 594| 41.0k| int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9), int8_t(0x00), 595| 41.0k| int8_t(0x10), int8_t(0xC3), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), 596| 41.0k| int8_t(0xB9), int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), 597| 41.0k| int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9), 598| 41.0k| int8_t(0x00), int8_t(0x10), int8_t(0xC3), int8_t(0xBF), int8_t(0xBF), 599| 41.0k| int8_t(0xB9), int8_t(0xB9)); 600| 41.0k| } 601| | 602| 41.0k| __m256i check_asso; 603| 41.0k| if (default_or_url) { ------------------ | Branch (603:9): [Folded, False: 41.0k] ------------------ 604| 0| check_asso = _mm256_setr_epi8( 605| 0| 0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x03, 606| 0| 0x07, 0x0B, 0x0E, 0x0B, 0x06, 0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 607| 0| 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0E, 0x0B, 0x06); 608| 41.0k| } else if (base64_url) { ------------------ | Branch (608:16): [Folded, False: 41.0k] ------------------ 609| 0| check_asso = _mm256_setr_epi8(0xD, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 610| 0| 0x1, 0x3, 0x7, 0xB, 0xE, 0xB, 0x6, 0xD, 0x1, 611| 0| 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x3, 612| 0| 0x7, 0xB, 0xE, 0xB, 0x6); 613| 41.0k| } else { 614| 41.0k| check_asso = _mm256_setr_epi8( 615| 41.0k| 0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x03, 616| 41.0k| 0x07, 0x0B, 0x0B, 0x0B, 0x0F, 0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 617| 41.0k| 0x01, 0x01, 0x01, 0x01, 0x03, 0x07, 0x0B, 0x0B, 0x0B, 0x0F); 618| 41.0k| } 619| 41.0k| __m256i check_values; 620| 41.0k| if (default_or_url) { ------------------ | Branch (620:9): [Folded, False: 41.0k] ------------------ 621| 0| check_values = _mm256_setr_epi8( 622| 0| uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), 623| 0| uint8_t(0xCF), uint8_t(0xBF), uint8_t(0xD5), uint8_t(0xA6), 624| 0| uint8_t(0xB5), uint8_t(0xA1), uint8_t(0x00), uint8_t(0x80), 625| 0| uint8_t(0x00), uint8_t(0x80), uint8_t(0x00), uint8_t(0x80), 626| 0| uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), 627| 0| uint8_t(0xCF), uint8_t(0xBF), uint8_t(0xD5), uint8_t(0xA6), 628| 0| uint8_t(0xB5), uint8_t(0xA1), uint8_t(0x00), uint8_t(0x80), 629| 0| uint8_t(0x00), uint8_t(0x80), uint8_t(0x00), uint8_t(0x80)); 630| 41.0k| } else if (base64_url) { ------------------ | Branch (630:16): [Folded, False: 41.0k] ------------------ 631| 0| check_values = _mm256_setr_epi8( 632| 0| uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), 633| 0| uint8_t(0xCF), uint8_t(0xBF), uint8_t(0xB6), uint8_t(0xA6), 634| 0| uint8_t(0xB5), uint8_t(0xA1), 0x0, uint8_t(0x80), 0x0, uint8_t(0x80), 635| 0| 0x0, uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), 636| 0| uint8_t(0x80), uint8_t(0xCF), uint8_t(0xBF), uint8_t(0xB6), 637| 0| uint8_t(0xA6), uint8_t(0xB5), uint8_t(0xA1), 0x0, uint8_t(0x80), 0x0, 638| 0| uint8_t(0x80), 0x0, uint8_t(0x80)); 639| 41.0k| } else { 640| 41.0k| check_values = _mm256_setr_epi8( 641| 41.0k| int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0xCF), 642| 41.0k| int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), int8_t(0xB5), int8_t(0x86), 643| 41.0k| int8_t(0xD1), int8_t(0x80), int8_t(0xB1), int8_t(0x80), int8_t(0x91), 644| 41.0k| int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), 645| 41.0k| int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), int8_t(0xB5), 646| 41.0k| int8_t(0x86), int8_t(0xD1), int8_t(0x80), int8_t(0xB1), int8_t(0x80), 647| 41.0k| int8_t(0x91), int8_t(0x80)); 648| 41.0k| } 649| 41.0k| const __m256i shifted = _mm256_srli_epi32(*src, 3); 650| 41.0k| __m256i delta_hash = 651| 41.0k| _mm256_avg_epu8(_mm256_shuffle_epi8(delta_asso, *src), shifted); 652| 41.0k| if (default_or_url) { ------------------ | Branch (652:9): [Folded, False: 41.0k] ------------------ 653| 0| delta_hash = _mm256_and_si256(delta_hash, _mm256_set1_epi8(0xf)); 654| 0| } 655| 41.0k| const __m256i check_hash = 656| 41.0k| _mm256_avg_epu8(_mm256_shuffle_epi8(check_asso, *src), shifted); 657| 41.0k| const __m256i out = 658| 41.0k| _mm256_adds_epi8(_mm256_shuffle_epi8(delta_values, delta_hash), *src); 659| 41.0k| const __m256i chk = 660| 41.0k| _mm256_adds_epi8(_mm256_shuffle_epi8(check_values, check_hash), *src); 661| 41.0k| const int mask = _mm256_movemask_epi8(chk); 662| 41.0k| if (!ignore_garbage && mask) { ------------------ | Branch (662:9): [True: 41.0k, Folded] | Branch (662:28): [True: 9.88k, False: 31.1k] ------------------ 663| 9.88k| __m256i ascii_space = 664| 9.88k| _mm256_cmpeq_epi8(_mm256_shuffle_epi8(ascii_space_tbl, *src), *src); 665| 9.88k| *error = (mask ^ _mm256_movemask_epi8(ascii_space)); 666| 9.88k| } 667| 41.0k| *src = out; 668| 41.0k| return (uint32_t)mask; 669| 41.0k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113load_12_bytesEPKh: 57| 640|simdutf_really_inline __m128i load_12_bytes(const uint8_t *input) { 58| 640| __m128i value = _mm_loadl_epi64(reinterpret_cast(input)); 59| 640| uint32_t tail; 60| 640| std::memcpy(&tail, input + 8, sizeof(tail)); 61| | return _mm_insert_epi32(value, int(tail), 2); 62| 640|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_113encode_base64ILb0EEEmPcPKcmNS_14base64_optionsE: 394| 18.0k| base64_options options) { 395| 18.0k| return avx2_encode_base64_impl(dst, src, srclen, options); 396| 18.0k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_123avx2_encode_base64_implILb0ELb0EEEmPcPKcmNS_14base64_optionsEm: 124| 18.0k| size_t line_length = simdutf::default_line_length) { 125| 18.0k| size_t offset = 0; 126| | 127| 18.0k| if (line_length < 4) { ------------------ | Branch (127:7): [True: 0, False: 18.0k] ------------------ 128| 0| line_length = 4; // We do not support line_length less than 4 129| 0| } 130| | // credit: Wojciech Muła 131| 18.0k| const uint8_t *input = (const uint8_t *)src; 132| | 133| 18.0k| uint8_t *out = (uint8_t *)dst; 134| 18.0k| const __m256i shuf = 135| 18.0k| _mm256_set_epi8(10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1, 136| | 137| 18.0k| 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); 138| 18.0k| const __m256i shuf_overlap = 139| 18.0k| _mm256_setr_epi8(5, 4, 6, 5, 8, 7, 9, 8, 11, 10, 12, 11, 14, 13, 15, 14, 140| 18.0k| 1, 0, 2, 1, 4, 3, 5, 4, 7, 6, 8, 7, 10, 9, 11, 10); 141| 18.0k| size_t i = 0; 142| 20.0k| for (; i + 100 <= srclen; i += 96) { ------------------ | Branch (142:10): [True: 2.07k, False: 18.0k] ------------------ 143| 2.07k| const __m128i lo0 = _mm_loadu_si128( 144| 2.07k| reinterpret_cast(input + i + 4 * 3 * 0)); 145| 2.07k| const __m128i hi0 = _mm_loadu_si128( 146| 2.07k| reinterpret_cast(input + i + 4 * 3 * 1)); 147| 2.07k| __m256i in0 = _mm256_shuffle_epi8(_mm256_set_m128i(hi0, lo0), shuf); ------------------ | | 4| 2.07k| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 5| 2.07k| _mm256_castsi128_si256(xmm2), 2) ------------------ 148| 2.07k| __m256i in1 = _mm256_shuffle_epi8( 149| 2.07k| _mm256_loadu_si256( 150| 2.07k| reinterpret_cast(input + i + 24 - 4)), 151| 2.07k| shuf_overlap); 152| 2.07k| __m256i in2 = _mm256_shuffle_epi8( 153| 2.07k| _mm256_loadu_si256( 154| 2.07k| reinterpret_cast(input + i + 48 - 4)), 155| 2.07k| shuf_overlap); 156| 2.07k| __m256i in3 = _mm256_shuffle_epi8( 157| 2.07k| _mm256_loadu_si256( 158| 2.07k| reinterpret_cast(input + i + 72 - 4)), 159| 2.07k| shuf_overlap); 160| | 161| 2.07k| const __m256i t0_0 = _mm256_and_si256(in0, _mm256_set1_epi32(0x0fc0fc00)); 162| 2.07k| const __m256i t0_1 = _mm256_and_si256(in1, _mm256_set1_epi32(0x0fc0fc00)); 163| 2.07k| const __m256i t0_2 = _mm256_and_si256(in2, _mm256_set1_epi32(0x0fc0fc00)); 164| 2.07k| const __m256i t0_3 = _mm256_and_si256(in3, _mm256_set1_epi32(0x0fc0fc00)); 165| | 166| 2.07k| const __m256i t1_0 = 167| 2.07k| _mm256_mulhi_epu16(t0_0, _mm256_set1_epi32(0x04000040)); 168| 2.07k| const __m256i t1_1 = 169| 2.07k| _mm256_mulhi_epu16(t0_1, _mm256_set1_epi32(0x04000040)); 170| 2.07k| const __m256i t1_2 = 171| 2.07k| _mm256_mulhi_epu16(t0_2, _mm256_set1_epi32(0x04000040)); 172| 2.07k| const __m256i t1_3 = 173| 2.07k| _mm256_mulhi_epu16(t0_3, _mm256_set1_epi32(0x04000040)); 174| | 175| 2.07k| const __m256i t2_0 = _mm256_and_si256(in0, _mm256_set1_epi32(0x003f03f0)); 176| 2.07k| const __m256i t2_1 = _mm256_and_si256(in1, _mm256_set1_epi32(0x003f03f0)); 177| 2.07k| const __m256i t2_2 = _mm256_and_si256(in2, _mm256_set1_epi32(0x003f03f0)); 178| 2.07k| const __m256i t2_3 = _mm256_and_si256(in3, _mm256_set1_epi32(0x003f03f0)); 179| | 180| 2.07k| const __m256i t3_0 = 181| 2.07k| _mm256_mullo_epi16(t2_0, _mm256_set1_epi32(0x01000010)); 182| 2.07k| const __m256i t3_1 = 183| 2.07k| _mm256_mullo_epi16(t2_1, _mm256_set1_epi32(0x01000010)); 184| 2.07k| const __m256i t3_2 = 185| 2.07k| _mm256_mullo_epi16(t2_2, _mm256_set1_epi32(0x01000010)); 186| 2.07k| const __m256i t3_3 = 187| 2.07k| _mm256_mullo_epi16(t2_3, _mm256_set1_epi32(0x01000010)); 188| | 189| 2.07k| const __m256i input0 = _mm256_or_si256(t1_0, t3_0); 190| 2.07k| const __m256i input1 = _mm256_or_si256(t1_1, t3_1); 191| 2.07k| const __m256i input2 = _mm256_or_si256(t1_2, t3_2); 192| 2.07k| const __m256i input3 = _mm256_or_si256(t1_3, t3_3); 193| | 194| 2.07k| if (use_lines) { ------------------ | Branch (194:9): [Folded, False: 2.07k] ------------------ 195| 0| if (line_length >= 32) { // fast path ------------------ | Branch (195:11): [True: 0, False: 0] ------------------ 196| 0| __m256i result; 197| 0| result = lookup_pshufb_improved(input0); 198| 0| if (offset + 32 > line_length) { ------------------ | Branch (198:13): [True: 0, False: 0] ------------------ 199| 0| size_t location_end = line_length - offset; 200| 0| size_t to_move = 32 - location_end; 201| | // We could do this, or extract instead. 202| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); 203| 0| _mm256_storeu_si256( 204| 0| reinterpret_cast<__m256i *>(out), 205| 0| insert_line_feed32(result, static_cast(location_end))); 206| 0| offset = to_move; 207| 0| out += 32 + 1; 208| 0| } else { 209| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); 210| 0| offset += 32; 211| 0| out += 32; 212| 0| } 213| 0| result = lookup_pshufb_improved(input1); 214| | 215| 0| if (offset + 32 > line_length) { ------------------ | Branch (215:13): [True: 0, False: 0] ------------------ 216| 0| size_t location_end = line_length - offset; 217| 0| size_t to_move = 32 - location_end; 218| | 219| | // We could do this, or extract instead. 220| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); 221| 0| _mm256_storeu_si256( 222| 0| reinterpret_cast<__m256i *>(out), 223| 0| insert_line_feed32(result, static_cast(location_end))); 224| | // see above. 225| | // out[32] = static_cast(_mm256_extract_epi8(result, 31)); 226| 0| offset = to_move; 227| 0| out += 32 + 1; 228| 0| } else { 229| | 230| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); 231| | 232| 0| offset += 32; 233| 0| out += 32; 234| 0| } 235| 0| result = lookup_pshufb_improved(input2); 236| | 237| 0| if (offset + 32 > line_length) { ------------------ | Branch (237:13): [True: 0, False: 0] ------------------ 238| 0| size_t location_end = line_length - offset; 239| 0| size_t to_move = 32 - location_end; 240| | 241| | // We could do this, or extract instead. 242| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); 243| 0| _mm256_storeu_si256( 244| 0| reinterpret_cast<__m256i *>(out), 245| 0| insert_line_feed32(result, static_cast(location_end))); 246| | // see above. 247| | // out[32] = static_cast(_mm256_extract_epi8(result, 31)); 248| 0| offset = to_move; 249| 0| out += 32 + 1; 250| 0| } else { 251| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); 252| 0| offset += 32; 253| 0| out += 32; 254| 0| } 255| 0| result = lookup_pshufb_improved(input3); 256| | 257| 0| if (offset + 32 > line_length) { ------------------ | Branch (257:13): [True: 0, False: 0] ------------------ 258| 0| size_t location_end = line_length - offset; 259| 0| size_t to_move = 32 - location_end; 260| | 261| | // We could do this, or extract instead. 262| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 1), result); 263| 0| _mm256_storeu_si256( 264| 0| reinterpret_cast<__m256i *>(out), 265| 0| insert_line_feed32(result, static_cast(location_end))); 266| | // see above. 267| | // out[32] = static_cast(_mm256_extract_epi8(result, 31)); 268| 0| offset = to_move; 269| 0| out += 32 + 1; 270| 0| } else { 271| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), result); 272| 0| offset += 32; 273| 0| out += 32; 274| 0| } 275| 0| } else { // slow path 276| | // could be optimized 277| 0| uint8_t buffer[128]; 278| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer), 279| 0| lookup_pshufb_improved(input0)); 280| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + 32), 281| 0| lookup_pshufb_improved(input1)); 282| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + 64), 283| 0| lookup_pshufb_improved(input2)); 284| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer + 96), 285| 0| lookup_pshufb_improved(input3)); 286| 0| size_t out_pos = 0; 287| 0| size_t local_offset = offset; 288| 0| for (size_t j = 0; j < 128;) { ------------------ | Branch (288:28): [True: 0, False: 0] ------------------ 289| 0| if (local_offset == line_length) { ------------------ | Branch (289:15): [True: 0, False: 0] ------------------ 290| 0| out[out_pos++] = '\n'; 291| 0| local_offset = 0; 292| 0| } 293| 0| out[out_pos++] = buffer[j++]; 294| 0| local_offset++; 295| 0| } 296| 0| offset = local_offset; 297| 0| out += out_pos; 298| 0| } 299| 2.07k| } else { 300| 2.07k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), 301| 2.07k| lookup_pshufb_improved(input0)); 302| 2.07k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 32), 303| 2.07k| lookup_pshufb_improved(input1)); 304| 2.07k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 64), 305| 2.07k| lookup_pshufb_improved(input2)); 306| 2.07k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out + 96), 307| 2.07k| lookup_pshufb_improved(input3)); 308| | 309| 2.07k| out += 128; 310| 2.07k| } 311| 2.07k| } 312| | // The high half is loaded 4 bytes early so that we never read past 313| | // input + i + 24: its shuffle indices are offset by 4. 314| 18.0k| const __m256i shuf_hi_shifted = 315| 18.0k| _mm256_set_epi8(14, 15, 13, 14, 11, 12, 10, 11, 8, 9, 7, 8, 5, 6, 4, 5, 316| | 317| 18.0k| 10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); 318| 20.2k| for (; i + 24 <= srclen; i += 24) { ------------------ | Branch (318:10): [True: 2.18k, False: 18.0k] ------------------ 319| | // lo = [xxxx|DDDC|CCBB|BAAA] 320| | // hi = [HHHG|GGFF|FEEE|xxxx] 321| 2.18k| const __m128i lo = 322| 2.18k| _mm_loadu_si128(reinterpret_cast(input + i)); 323| 2.18k| const __m128i hi = 324| 2.18k| _mm_loadu_si128(reinterpret_cast(input + i + 8)); 325| | 326| | // bytes from groups A, B and C are needed in separate 32-bit lanes 327| | // in = [0HHH|0GGG|0FFF|0EEE[0DDD|0CCC|0BBB|0AAA] 328| 2.18k| __m256i in = _mm256_shuffle_epi8(_mm256_set_m128i(hi, lo), shuf_hi_shifted); ------------------ | | 4| 2.18k| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 5| 2.18k| _mm256_castsi128_si256(xmm2), 2) ------------------ 329| | 330| | // this part is well commented in encode.sse.cpp 331| | 332| 2.18k| const __m256i t0 = _mm256_and_si256(in, _mm256_set1_epi32(0x0fc0fc00)); 333| 2.18k| const __m256i t1 = _mm256_mulhi_epu16(t0, _mm256_set1_epi32(0x04000040)); 334| 2.18k| const __m256i t2 = _mm256_and_si256(in, _mm256_set1_epi32(0x003f03f0)); 335| 2.18k| const __m256i t3 = _mm256_mullo_epi16(t2, _mm256_set1_epi32(0x01000010)); 336| 2.18k| const __m256i indices = _mm256_or_si256(t1, t3); 337| | 338| 2.18k| if (use_lines) { ------------------ | Branch (338:9): [Folded, False: 2.18k] ------------------ 339| 0| if (line_length >= 32) { // fast path ------------------ | Branch (339:11): [True: 0, False: 0] ------------------ 340| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), 341| 0| lookup_pshufb_improved(indices)); 342| | 343| 0| if (offset + 32 > line_length) { ------------------ | Branch (343:13): [True: 0, False: 0] ------------------ 344| 0| size_t location_end = line_length - offset; 345| 0| size_t to_move = 32 - location_end; 346| 0| std::memmove(out + location_end + 1, out + location_end, to_move); 347| 0| out[location_end] = '\n'; 348| 0| offset = to_move; 349| 0| out += 32 + 1; 350| 0| } else { 351| 0| offset += 32; 352| 0| out += 32; 353| 0| } 354| 0| } else { // slow path 355| | // could be optimized 356| 0| alignas(32) uint8_t buffer[32]; 357| 0| _mm256_storeu_si256(reinterpret_cast<__m256i *>(buffer), 358| 0| lookup_pshufb_improved(indices)); 359| 0| std::memcpy(out, buffer, 32); 360| 0| size_t out_pos = 0; 361| 0| size_t local_offset = offset; 362| 0| for (size_t j = 0; j < 32;) { ------------------ | Branch (362:28): [True: 0, False: 0] ------------------ 363| 0| if (local_offset == line_length) { ------------------ | Branch (363:15): [True: 0, False: 0] ------------------ 364| 0| out[out_pos++] = '\n'; 365| 0| local_offset = 0; 366| 0| } 367| 0| out[out_pos++] = buffer[j++]; 368| 0| local_offset++; 369| 0| } 370| 0| offset = local_offset; 371| 0| out += out_pos; 372| 0| } 373| 2.18k| } else { 374| 2.18k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(out), 375| 2.18k| lookup_pshufb_improved(indices)); 376| | 377| 2.18k| out += 32; 378| 2.18k| } 379| 2.18k| } 380| 18.0k| if constexpr (!use_lines) { 381| 18.0k| if (i + 12 <= srclen) { ------------------ | Branch (381:9): [True: 640, False: 17.3k] ------------------ 382| 640| encode_base64_12(out, input + i); 383| 640| i += 12; 384| 640| out += 16; 385| 640| } 386| 18.0k| } 387| 18.0k| return ((char *)out - (char *)dst) + 388| 18.0k| scalar::base64::tail_encode_base64_impl( 389| 18.0k| (char *)out, src + i, srclen - i, options, line_length, offset); 390| 18.0k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_122lookup_pshufb_improvedILb0EEEDv4_xS3_: 30| 11.1k|simdutf_really_inline __m256i lookup_pshufb_improved(const __m256i input) { 31| | // Precomputed shuffle masks for K = 1 to 16 32| | // credit: Wojciech Muła 33| 11.1k| __m256i result = _mm256_subs_epu8(input, _mm256_set1_epi8(51)); 34| 11.1k| result = 35| 11.1k| _mm256_sub_epi8(result, _mm256_cmpgt_epi8(input, _mm256_set1_epi8(25))); 36| 11.1k| __m256i shift_LUT; 37| 11.1k| if (base64_url) { ------------------ | Branch (37:7): [Folded, False: 11.1k] ------------------ 38| 0| shift_LUT = _mm256_setr_epi8( 39| 0| 'A', 'a' - 26, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, 40| 0| '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '-' - 62, '_' - 63, 0, 41| 0| 0, 'A', 'a' - 26, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, 42| 0| '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '-' - 62, '_' - 63, 0, 43| 0| 0); 44| 11.1k| } else { 45| 11.1k| shift_LUT = _mm256_setr_epi8( 46| 11.1k| 'A', 'a' - 26, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, 47| 11.1k| '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '+' - 62, '/' - 63, 0, 48| 11.1k| 0, 'A', 'a' - 26, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, 49| 11.1k| '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, '+' - 62, '/' - 63, 0, 50| 11.1k| 0); 51| 11.1k| } 52| | 53| 11.1k| result = _mm256_shuffle_epi8(shift_LUT, result); 54| 11.1k| return _mm256_add_epi8(result, input); 55| 11.1k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_116encode_base64_12ILb0EEEvPhPKh: 68| 640| const uint8_t *input) { 69| 640| const __m128i shuf = 70| 640| _mm_set_epi8(10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); 71| 640| const __m128i in = _mm_shuffle_epi8(load_12_bytes(input), shuf); 72| 640| const __m256i in256 = _mm256_castsi128_si256(in); 73| 640| const __m256i t0 = _mm256_and_si256(in256, _mm256_set1_epi32(0x0fc0fc00)); 74| 640| const __m256i t1 = _mm256_mulhi_epu16(t0, _mm256_set1_epi32(0x04000040)); 75| 640| const __m256i t2 = _mm256_and_si256(in256, _mm256_set1_epi32(0x003f03f0)); 76| 640| const __m256i t3 = _mm256_mullo_epi16(t2, _mm256_set1_epi32(0x01000010)); 77| 640| const __m256i encoded = 78| 640| lookup_pshufb_improved(_mm256_or_si256(t1, t3)); 79| 640| _mm_storeu_si128(reinterpret_cast<__m128i *>(output), 80| 640| _mm256_castsi256_si128(encoded)); 81| 640|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128avx2_convert_latin1_to_utf16ILNS_10endiannessE0EEENSt3__14pairIPKcPDsEES7_mS8_: 4| 6.85k| char16_t *utf16_output) { 5| 6.85k| size_t rounded_len = len & ~0xF; // Round down to nearest multiple of 16 6| | 7| 6.85k| size_t i = 0; 8| 26.3k| for (; i < rounded_len; i += 16) { ------------------ | Branch (8:10): [True: 19.5k, False: 6.85k] ------------------ 9| | // Load 16 bytes from the address (input + i) into a xmm register 10| 19.5k| const __m128i latin1 = 11| 19.5k| _mm_loadu_si128(reinterpret_cast(latin1_input + i)); 12| | 13| | // Zero extend each byte in `in` to word 14| 19.5k| __m256i utf16 = _mm256_cvtepu8_epi16(latin1); 15| | 16| 19.5k| if (big_endian) { ------------------ | Branch (16:9): [Folded, False: 19.5k] ------------------ 17| 0| const __m128i swap128 = 18| 0| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 19| 0| const __m256i swap = _mm256_set_m128i(swap128, swap128); ------------------ | | 4| 0| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 5| 0| _mm256_castsi128_si256(xmm2), 2) ------------------ 20| 0| utf16 = _mm256_shuffle_epi8(utf16, swap); 21| 0| } 22| | 23| | // Store the contents of xmm1 into the address pointed by (output + i) 24| 19.5k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf16_output + i), utf16); 25| 19.5k| } 26| | 27| 6.85k| return std::make_pair(latin1_input + rounded_len, utf16_output + rounded_len); 28| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128avx2_convert_latin1_to_utf16ILNS_10endiannessE1EEENSt3__14pairIPKcPDsEES7_mS8_: 4| 6.85k| char16_t *utf16_output) { 5| 6.85k| size_t rounded_len = len & ~0xF; // Round down to nearest multiple of 16 6| | 7| 6.85k| size_t i = 0; 8| 26.3k| for (; i < rounded_len; i += 16) { ------------------ | Branch (8:10): [True: 19.5k, False: 6.85k] ------------------ 9| | // Load 16 bytes from the address (input + i) into a xmm register 10| 19.5k| const __m128i latin1 = 11| 19.5k| _mm_loadu_si128(reinterpret_cast(latin1_input + i)); 12| | 13| | // Zero extend each byte in `in` to word 14| 19.5k| __m256i utf16 = _mm256_cvtepu8_epi16(latin1); 15| | 16| 19.5k| if (big_endian) { ------------------ | Branch (16:9): [True: 19.5k, Folded] ------------------ 17| 19.5k| const __m128i swap128 = 18| 19.5k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 19| 19.5k| const __m256i swap = _mm256_set_m128i(swap128, swap128); ------------------ | | 4| 19.5k| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 5| 19.5k| _mm256_castsi128_si256(xmm2), 2) ------------------ 20| 19.5k| utf16 = _mm256_shuffle_epi8(utf16, swap); 21| 19.5k| } 22| | 23| | // Store the contents of xmm1 into the address pointed by (output + i) 24| 19.5k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf16_output + i), utf16); 25| 19.5k| } 26| | 27| 6.85k| return std::make_pair(latin1_input + rounded_len, utf16_output + rounded_len); 28| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128avx2_convert_latin1_to_utf32EPKcmPDi: 3| 6.85k| char32_t *utf32_output) { 4| 6.85k| size_t rounded_len = ((len | 7) ^ 7); // Round down to nearest multiple of 8 5| | 6| 46.1k| for (size_t i = 0; i < rounded_len; i += 8) { ------------------ | Branch (6:22): [True: 39.2k, False: 6.85k] ------------------ 7| | // Load 8 Latin1 characters into a 64-bit register 8| 39.2k| __m128i in = _mm_loadl_epi64((__m128i *)&buf[i]); 9| | 10| | // Zero extend each set of 8 Latin1 characters to 8 32-bit integers using 11| | // vpmovzxbd 12| 39.2k| __m256i out = _mm256_cvtepu8_epi32(in); 13| | 14| | // Store the results back to memory 15| 39.2k| _mm256_storeu_si256((__m256i *)&utf32_output[i], out); 16| 39.2k| } 17| | 18| | // return pointers pointing to where we left off 19| 6.85k| return std::make_pair(buf + rounded_len, utf32_output + rounded_len); 20| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_127avx2_convert_latin1_to_utf8EPKcmPc: 3| 7.13k| char *utf8_output) { 4| 7.13k| const char *end = latin1_input + len; 5| 7.13k| const __m256i v_0000 = _mm256_setzero_si256(); 6| 7.13k| const __m256i v_c080 = _mm256_set1_epi16((int16_t)0xc080); 7| 7.13k| const __m256i v_ff80 = _mm256_set1_epi16((int16_t)0xff80); 8| 7.13k| const size_t safety_margin = 12; 9| | 10| 27.5k| while (end - latin1_input >= std::ptrdiff_t(16 + safety_margin)) { ------------------ | Branch (10:10): [True: 20.3k, False: 7.13k] ------------------ 11| 20.3k| __m128i in8 = _mm_loadu_si128((__m128i *)latin1_input); 12| | // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes 13| 20.3k| const __m128i v_80 = _mm_set1_epi8((char)0x80); 14| 20.3k| if (_mm_testz_si128(in8, v_80)) { // ASCII fast path!!!! ------------------ | Branch (14:9): [True: 12.9k, False: 7.46k] ------------------ 15| | // 1. store (16 bytes) 16| 12.9k| _mm_storeu_si128((__m128i *)utf8_output, in8); 17| | // 2. adjust pointers 18| 12.9k| latin1_input += 16; 19| 12.9k| utf8_output += 16; 20| 12.9k| continue; // we are done for this round! 21| 12.9k| } 22| | // We proceed only with the first 16 bytes. 23| 7.46k| const __m256i in = _mm256_cvtepu8_epi16((in8)); 24| | 25| | // 1. prepare 2-byte values 26| | // input 16-bit word : [0000|0000|aabb|bbbb] x 8 27| | // expected output : [1100|00aa|10bb|bbbb] x 8 28| 7.46k| const __m256i v_1f00 = _mm256_set1_epi16((int16_t)0x1f00); 29| 7.46k| const __m256i v_003f = _mm256_set1_epi16((int16_t)0x003f); 30| | 31| | // t0 = [0000|00aa|bbbb|bb00] 32| 7.46k| const __m256i t0 = _mm256_slli_epi16(in, 2); 33| | // t1 = [0000|00aa|0000|0000] 34| 7.46k| const __m256i t1 = _mm256_and_si256(t0, v_1f00); 35| | // t2 = [0000|0000|00bb|bbbb] 36| 7.46k| const __m256i t2 = _mm256_and_si256(in, v_003f); 37| | // t3 = [000a|aaaa|00bb|bbbb] 38| 7.46k| const __m256i t3 = _mm256_or_si256(t1, t2); 39| | // t4 = [1100|00aa|10bb|bbbb] 40| 7.46k| const __m256i t4 = _mm256_or_si256(t3, v_c080); 41| | 42| | // 2. merge ASCII and 2-byte codewords 43| | 44| | // no bits set above 7th bit 45| 7.46k| const __m256i one_byte_bytemask = 46| 7.46k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_ff80), v_0000); 47| 7.46k| const uint32_t one_byte_bitmask = 48| 7.46k| static_cast(_mm256_movemask_epi8(one_byte_bytemask)); 49| | 50| 7.46k| const __m256i utf8_unpacked = _mm256_blendv_epi8(t4, in, one_byte_bytemask); 51| | 52| | // 3. prepare bitmask for 8-bit lookup 53| 7.46k| const uint32_t M0 = one_byte_bitmask & 0x55555555; 54| 7.46k| const uint32_t M1 = M0 >> 7; 55| 7.46k| const uint32_t M2 = (M1 | M0) & 0x00ff00ff; 56| | // 4. pack the bytes 57| | 58| 7.46k| const uint8_t *row = 59| 7.46k| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2)][0]; 60| 7.46k| const uint8_t *row_2 = 61| 7.46k| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2 >> 16)] 62| 7.46k| [0]; 63| | 64| 7.46k| const __m128i shuffle = _mm_loadu_si128((__m128i *)(row + 1)); 65| 7.46k| const __m128i shuffle_2 = _mm_loadu_si128((__m128i *)(row_2 + 1)); 66| | 67| 7.46k| const __m256i utf8_packed = _mm256_shuffle_epi8( 68| 7.46k| utf8_unpacked, _mm256_setr_m128i(shuffle, shuffle_2)); ------------------ | | 7| 7.46k| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 8| 7.46k| _mm256_castsi128_si256(xmm2), 2) ------------------ 69| | // 5. store bytes 70| 7.46k| _mm_storeu_si128((__m128i *)utf8_output, 71| 7.46k| _mm256_castsi256_si128(utf8_packed)); 72| 7.46k| utf8_output += row[0]; 73| 7.46k| _mm_storeu_si128((__m128i *)utf8_output, 74| 7.46k| _mm256_extractf128_si256(utf8_packed, 1)); 75| 7.46k| utf8_output += row_2[0]; 76| | 77| | // 6. adjust pointers 78| 7.46k| latin1_input += 16; 79| 7.46k| continue; 80| | 81| 20.3k| } // while 82| 7.13k| return std::make_pair(latin1_input, utf8_output); 83| 7.13k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128avx2_convert_utf16_to_latin1ILNS_10endiannessE0EEENSt3__14pairIPKDsPcEES7_mS8_: 4| 6.85k| char *latin1_output) { 5| 6.85k| const char16_t *end = buf + len; 6| 16.4k| while (end - buf >= 32) { ------------------ | Branch (6:10): [True: 9.57k, False: 6.85k] ------------------ 7| | // Load 16 UTF-16 characters into 256-bit AVX2 register 8| 9.57k| __m256i in0 = _mm256_loadu_si256(reinterpret_cast(buf)); 9| 9.57k| __m256i in1 = 10| 9.57k| _mm256_loadu_si256(reinterpret_cast(buf + 16)); 11| | 12| | if constexpr (!match_system(big_endian)) { 13| | const __m256i swap = _mm256_setr_epi8( 14| | 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 15| | 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 16| | in0 = _mm256_shuffle_epi8(in0, swap); 17| | in1 = _mm256_shuffle_epi8(in1, swap); 18| | } 19| | 20| 9.57k| __m256i high_byte_mask = _mm256_set1_epi16((int16_t)0xFF00); 21| 9.57k| if (_mm256_testz_si256(_mm256_or_si256(in0, in1), high_byte_mask)) { ------------------ | Branch (21:9): [True: 9.57k, False: 0] ------------------ 22| | // Pack 16-bit characters into 8-bit and store in latin1_output 23| 9.57k| const __m256i packed = _mm256_packus_epi16(in0, in1); 24| | 25| 9.57k| const __m256i result = _mm256_permute4x64_epi64(packed, 0b11011000); 26| | 27| 9.57k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(latin1_output), result); 28| | // Adjust pointers for the next iteration 29| 9.57k| buf += 32; 30| 9.57k| latin1_output += 32; 31| 9.57k| } else { 32| 0| return std::make_pair(nullptr, reinterpret_cast(latin1_output)); 33| 0| } 34| 9.57k| } // while 35| 6.85k| return std::make_pair(buf, latin1_output); 36| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128avx2_convert_utf16_to_latin1ILNS_10endiannessE1EEENSt3__14pairIPKDsPcEES7_mS8_: 4| 6.85k| char *latin1_output) { 5| 6.85k| const char16_t *end = buf + len; 6| 16.4k| while (end - buf >= 32) { ------------------ | Branch (6:10): [True: 9.57k, False: 6.85k] ------------------ 7| | // Load 16 UTF-16 characters into 256-bit AVX2 register 8| 9.57k| __m256i in0 = _mm256_loadu_si256(reinterpret_cast(buf)); 9| 9.57k| __m256i in1 = 10| 9.57k| _mm256_loadu_si256(reinterpret_cast(buf + 16)); 11| | 12| 9.57k| if constexpr (!match_system(big_endian)) { 13| 9.57k| const __m256i swap = _mm256_setr_epi8( 14| 9.57k| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 15| 9.57k| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 16| 9.57k| in0 = _mm256_shuffle_epi8(in0, swap); 17| 9.57k| in1 = _mm256_shuffle_epi8(in1, swap); 18| 9.57k| } 19| | 20| 9.57k| __m256i high_byte_mask = _mm256_set1_epi16((int16_t)0xFF00); 21| 9.57k| if (_mm256_testz_si256(_mm256_or_si256(in0, in1), high_byte_mask)) { ------------------ | Branch (21:9): [True: 9.57k, False: 0] ------------------ 22| | // Pack 16-bit characters into 8-bit and store in latin1_output 23| 9.57k| const __m256i packed = _mm256_packus_epi16(in0, in1); 24| | 25| 9.57k| const __m256i result = _mm256_permute4x64_epi64(packed, 0b11011000); 26| | 27| 9.57k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(latin1_output), result); 28| | // Adjust pointers for the next iteration 29| 9.57k| buf += 32; 30| 9.57k| latin1_output += 32; 31| 9.57k| } else { 32| 0| return std::make_pair(nullptr, reinterpret_cast(latin1_output)); 33| 0| } 34| 9.57k| } // while 35| 6.85k| return std::make_pair(buf, latin1_output); 36| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_126avx2_convert_utf16_to_utf8ILNS_10endiannessE0EEENSt3__14pairIPKDsPcEES7_mS8_: 56| 12.8k|avx2_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { 57| 12.8k| const char16_t *end = buf + len; 58| 12.8k| const __m256i v_0000 = _mm256_setzero_si256(); 59| 12.8k| const __m256i v_f800 = _mm256_set1_epi16((int16_t)0xf800); 60| 12.8k| const __m256i v_d800 = _mm256_set1_epi16((int16_t)0xd800); 61| 12.8k| const __m256i v_c080 = _mm256_set1_epi16((int16_t)0xc080); 62| 12.8k| const size_t safety_margin = 63| 12.8k| 12; // to avoid overruns, see issue 64| | // https://github.com/simdutf/simdutf/issues/92 65| | 66| 17.8k| while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { ------------------ | Branch (66:10): [True: 5.04k, False: 12.8k] ------------------ 67| 5.04k| __m256i in = _mm256_loadu_si256((__m256i *)buf); 68| 5.04k| if (big_endian) { ------------------ | Branch (68:9): [Folded, False: 5.04k] ------------------ 69| 0| const __m256i swap = _mm256_setr_epi8( 70| 0| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 71| 0| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 72| 0| in = _mm256_shuffle_epi8(in, swap); 73| 0| } 74| | // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes 75| 5.04k| const __m256i v_ff80 = _mm256_set1_epi16((int16_t)0xff80); 76| 5.04k| if (_mm256_testz_si256(in, v_ff80)) { // ASCII fast path!!!! ------------------ | Branch (76:9): [True: 2.48k, False: 2.56k] ------------------ 77| | // 1. pack the bytes 78| 2.48k| const __m128i utf8_packed = _mm_packus_epi16( 79| 2.48k| _mm256_castsi256_si128(in), _mm256_extractf128_si256(in, 1)); 80| | // 2. store (16 bytes) 81| 2.48k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 82| | // 3. adjust pointers 83| 2.48k| buf += 16; 84| 2.48k| utf8_output += 16; 85| 2.48k| continue; // we are done for this round! 86| 2.48k| } 87| | // no bits set above 7th bit 88| 2.56k| const __m256i one_byte_bytemask = 89| 2.56k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_ff80), v_0000); 90| 2.56k| const uint32_t one_byte_bitmask = 91| 2.56k| static_cast(_mm256_movemask_epi8(one_byte_bytemask)); 92| | 93| | // no bits set above 11th bit 94| 2.56k| const __m256i one_or_two_bytes_bytemask = 95| 2.56k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_f800), v_0000); 96| 2.56k| const uint32_t one_or_two_bytes_bitmask = 97| 2.56k| static_cast(_mm256_movemask_epi8(one_or_two_bytes_bytemask)); 98| 2.56k| if (one_or_two_bytes_bitmask == 0xffffffff) { ------------------ | Branch (98:9): [True: 217, False: 2.34k] ------------------ 99| | 100| | // 1. prepare 2-byte values 101| | // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 102| | // expected output : [110a|aaaa|10bb|bbbb] x 8 103| 217| const __m256i v_1f00 = _mm256_set1_epi16((int16_t)0x1f00); 104| 217| const __m256i v_003f = _mm256_set1_epi16((int16_t)0x003f); 105| | 106| | // t0 = [000a|aaaa|bbbb|bb00] 107| 217| const __m256i t0 = _mm256_slli_epi16(in, 2); 108| | // t1 = [000a|aaaa|0000|0000] 109| 217| const __m256i t1 = _mm256_and_si256(t0, v_1f00); 110| | // t2 = [0000|0000|00bb|bbbb] 111| 217| const __m256i t2 = _mm256_and_si256(in, v_003f); 112| | // t3 = [000a|aaaa|00bb|bbbb] 113| 217| const __m256i t3 = _mm256_or_si256(t1, t2); 114| | // t4 = [110a|aaaa|10bb|bbbb] 115| 217| const __m256i t4 = _mm256_or_si256(t3, v_c080); 116| | 117| | // 2. merge ASCII and 2-byte codewords 118| 217| const __m256i utf8_unpacked = 119| 217| _mm256_blendv_epi8(t4, in, one_byte_bytemask); 120| | 121| | // 3. prepare bitmask for 8-bit lookup 122| 217| const uint32_t M0 = one_byte_bitmask & 0x55555555; 123| 217| const uint32_t M1 = M0 >> 7; 124| 217| const uint32_t M2 = (M1 | M0) & 0x00ff00ff; 125| | // 4. pack the bytes 126| | 127| 217| const uint8_t *row = 128| 217| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2)][0]; 129| 217| const uint8_t *row_2 = 130| 217| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2 >> 131| 217| 16)][0]; 132| | 133| 217| const __m128i shuffle = _mm_loadu_si128((__m128i *)(row + 1)); 134| 217| const __m128i shuffle_2 = _mm_loadu_si128((__m128i *)(row_2 + 1)); 135| | 136| 217| const __m256i utf8_packed = _mm256_shuffle_epi8( 137| 217| utf8_unpacked, _mm256_setr_m128i(shuffle, shuffle_2)); ------------------ | | 7| 217| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 8| 217| _mm256_castsi128_si256(xmm2), 2) ------------------ 138| | // 5. store bytes 139| 217| _mm_storeu_si128((__m128i *)utf8_output, 140| 217| _mm256_castsi256_si128(utf8_packed)); 141| 217| utf8_output += row[0]; 142| 217| _mm_storeu_si128((__m128i *)utf8_output, 143| 217| _mm256_extractf128_si256(utf8_packed, 1)); 144| 217| utf8_output += row_2[0]; 145| | 146| | // 6. adjust pointers 147| 217| buf += 16; 148| 217| continue; 149| 217| } 150| | // 1. Check if there are any surrogate word in the input chunk. 151| | // We have also deal with situation when there is a surrogate word 152| | // at the end of a chunk. 153| 2.34k| const __m256i surrogates_bytemask = 154| 2.34k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_f800), v_d800); 155| | 156| | // bitmask = 0x0000 if there are no surrogates 157| | // = 0xc000 if the last word is a surrogate 158| 2.34k| const uint32_t surrogates_bitmask = 159| 2.34k| static_cast(_mm256_movemask_epi8(surrogates_bytemask)); 160| | // It might seem like checking for surrogates_bitmask == 0xc000 could help. 161| | // However, it is likely an uncommon occurrence. 162| 2.34k| if (surrogates_bitmask == 0x00000000) { ------------------ | Branch (162:9): [True: 1.20k, False: 1.14k] ------------------ 163| | // case: code units from register produce either 1, 2 or 3 UTF-8 bytes 164| 1.20k| const __m256i dup_even = _mm256_setr_epi16( 165| 1.20k| 0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e, 166| 1.20k| 0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); 167| | 168| | /* In this branch we handle three cases: 169| | 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - 170| | single UFT-8 byte 171| | 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two 172| | UTF-8 bytes 173| | 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - 174| | three UTF-8 bytes 175| | 176| | We expand the input word (16-bit) into two code units (32-bit), thus 177| | we have room for four bytes. However, we need five distinct bit 178| | layouts. Note that the last byte in cases #2 and #3 is the same. 179| | 180| | We precompute byte 1 for case #1 and the common byte for cases #2 & #3 181| | in register t2. 182| | 183| | We precompute byte 1 for case #3 and -- **conditionally** -- precompute 184| | either byte 1 for case #2 or byte 2 for case #3. Note that they 185| | differ by exactly one bit. 186| | 187| | Finally from these two code units we build proper UTF-8 sequence, taking 188| | into account the case (i.e, the number of bytes to write). 189| | */ 190| | /** 191| | * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: 192| | * t2 => [0ccc|cccc] [10cc|cccc] 193| | * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) 194| | */ 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) 196| | // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] 197| 1.20k| const __m256i t0 = _mm256_shuffle_epi8(in, dup_even); 198| | // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] 199| 1.20k| const __m256i t1 = _mm256_and_si256(t0, simdutf_vec(0b0011111101111111)); ------------------ | | 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 200| | // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] 201| 1.20k| const __m256i t2 = _mm256_or_si256(t1, simdutf_vec(0b1000000000000000)); ------------------ | | 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 202| | 203| | // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] 204| 1.20k| const __m256i s0 = _mm256_srli_epi16(in, 4); 205| | // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] 206| 1.20k| const __m256i s1 = _mm256_and_si256(s0, simdutf_vec(0b0000111111111100)); ------------------ | | 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 207| | // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] 208| 1.20k| const __m256i s2 = _mm256_maddubs_epi16(s1, simdutf_vec(0x0140)); ------------------ | | 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 209| | // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] 210| 1.20k| const __m256i s3 = _mm256_or_si256(s2, simdutf_vec(0b1100000011100000)); ------------------ | | 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 211| 1.20k| const __m256i m0 = _mm256_andnot_si256(one_or_two_bytes_bytemask, 212| 1.20k| simdutf_vec(0b0100000000000000)); ------------------ | | 195| 1.20k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 213| 1.20k| const __m256i s4 = _mm256_xor_si256(s3, m0); 214| 1.20k|#undef simdutf_vec 215| | 216| | // 4. expand code units 16-bit => 32-bit 217| 1.20k| const __m256i out0 = _mm256_unpacklo_epi16(t2, s4); 218| 1.20k| const __m256i out1 = _mm256_unpackhi_epi16(t2, s4); 219| | 220| | // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle 221| 1.20k| const uint32_t mask = (one_byte_bitmask & 0x55555555) | 222| 1.20k| (one_or_two_bytes_bitmask & 0xaaaaaaaa); 223| | // Due to the wider registers, the following path is less likely to be 224| | // useful. 225| | /*if(mask == 0) { 226| | // We only have three-byte code units. Use fast path. 227| | const __m256i shuffle = 228| | _mm256_setr_epi8(2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1, 229| | 2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1); const __m256i utf8_0 = 230| | _mm256_shuffle_epi8(out0, shuffle); const __m256i utf8_1 = 231| | _mm256_shuffle_epi8(out1, shuffle); 232| | _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_0)); 233| | utf8_output += 12; 234| | _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_1)); 235| | utf8_output += 12; 236| | _mm_storeu_si128((__m128i*)utf8_output, 237| | _mm256_extractf128_si256(utf8_0,1)); utf8_output += 12; 238| | _mm_storeu_si128((__m128i*)utf8_output, 239| | _mm256_extractf128_si256(utf8_1,1)); utf8_output += 12; buf += 16; 240| | continue; 241| | }*/ 242| 1.20k| const uint8_t mask0 = uint8_t(mask); 243| 1.20k| const uint8_t *row0 = 244| 1.20k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; 245| 1.20k| const __m128i shuffle0 = _mm_loadu_si128((__m128i *)(row0 + 1)); 246| 1.20k| const __m128i utf8_0 = 247| 1.20k| _mm_shuffle_epi8(_mm256_castsi256_si128(out0), shuffle0); 248| | 249| 1.20k| const uint8_t mask1 = static_cast(mask >> 8); 250| 1.20k| const uint8_t *row1 = 251| 1.20k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; 252| 1.20k| const __m128i shuffle1 = _mm_loadu_si128((__m128i *)(row1 + 1)); 253| 1.20k| const __m128i utf8_1 = 254| 1.20k| _mm_shuffle_epi8(_mm256_castsi256_si128(out1), shuffle1); 255| | 256| 1.20k| const uint8_t mask2 = static_cast(mask >> 16); 257| 1.20k| const uint8_t *row2 = 258| 1.20k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask2][0]; 259| 1.20k| const __m128i shuffle2 = _mm_loadu_si128((__m128i *)(row2 + 1)); 260| 1.20k| const __m128i utf8_2 = 261| 1.20k| _mm_shuffle_epi8(_mm256_extractf128_si256(out0, 1), shuffle2); 262| | 263| 1.20k| const uint8_t mask3 = static_cast(mask >> 24); 264| 1.20k| const uint8_t *row3 = 265| 1.20k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask3][0]; 266| 1.20k| const __m128i shuffle3 = _mm_loadu_si128((__m128i *)(row3 + 1)); 267| 1.20k| const __m128i utf8_3 = 268| 1.20k| _mm_shuffle_epi8(_mm256_extractf128_si256(out1, 1), shuffle3); 269| | 270| 1.20k| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 271| 1.20k| utf8_output += row0[0]; 272| 1.20k| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 273| 1.20k| utf8_output += row1[0]; 274| 1.20k| _mm_storeu_si128((__m128i *)utf8_output, utf8_2); 275| 1.20k| utf8_output += row2[0]; 276| 1.20k| _mm_storeu_si128((__m128i *)utf8_output, utf8_3); 277| 1.20k| utf8_output += row3[0]; 278| 1.20k| buf += 16; 279| | // surrogate pair(s) in a register 280| 1.20k| } else { 281| | // Let us do a scalar fallback. 282| | // It may seem wasteful to use scalar code, but being efficient with SIMD 283| | // in the presence of surrogate pairs may require non-trivial tables. 284| 1.14k| size_t forward = 15; 285| 1.14k| size_t k = 0; 286| 1.14k| if (size_t(end - buf) < forward + 1) { ------------------ | Branch (286:11): [True: 0, False: 1.14k] ------------------ 287| 0| forward = size_t(end - buf - 1); 288| 0| } 289| 15.9k| for (; k < forward; k++) { ------------------ | Branch (289:14): [True: 14.8k, False: 1.08k] ------------------ 290| 14.8k| uint16_t word = scalar::utf16::swap_if_needed(buf[k]); 291| 14.8k| if ((word & 0xFF80) == 0) { ------------------ | Branch (291:13): [True: 8.68k, False: 6.18k] ------------------ 292| 8.68k| *utf8_output++ = char(word); 293| 8.68k| } else if ((word & 0xF800) == 0) { ------------------ | Branch (293:20): [True: 733, False: 5.45k] ------------------ 294| 733| *utf8_output++ = char((word >> 6) | 0b11000000); 295| 733| *utf8_output++ = char((word & 0b111111) | 0b10000000); 296| 5.45k| } else if ((word & 0xF800) != 0xD800) { ------------------ | Branch (296:20): [True: 3.56k, False: 1.89k] ------------------ 297| 3.56k| *utf8_output++ = char((word >> 12) | 0b11100000); 298| 3.56k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 299| 3.56k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 300| 3.56k| } else { 301| | // must be a surrogate pair 302| 1.89k| uint16_t diff = uint16_t(word - 0xD800); 303| 1.89k| uint16_t next_word = 304| 1.89k| scalar::utf16::swap_if_needed(buf[k + 1]); 305| 1.89k| k++; 306| 1.89k| uint16_t diff2 = uint16_t(next_word - 0xDC00); 307| 1.89k| if ((diff | diff2) > 0x3FF) { ------------------ | Branch (307:15): [True: 59, False: 1.83k] ------------------ 308| 59| return std::make_pair(nullptr, utf8_output); 309| 59| } 310| 1.83k| uint32_t value = (diff << 10) + diff2 + 0x10000; 311| 1.83k| *utf8_output++ = char((value >> 18) | 0b11110000); 312| 1.83k| *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); 313| 1.83k| *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); 314| 1.83k| *utf8_output++ = char((value & 0b111111) | 0b10000000); 315| 1.83k| } 316| 14.8k| } 317| 1.08k| buf += k; 318| 1.08k| } 319| 2.34k| } // while 320| 12.8k| return std::make_pair(buf, utf8_output); 321| 12.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_126avx2_convert_utf16_to_utf8ILNS_10endiannessE1EEENSt3__14pairIPKDsPcEES7_mS8_: 56| 12.8k|avx2_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { 57| 12.8k| const char16_t *end = buf + len; 58| 12.8k| const __m256i v_0000 = _mm256_setzero_si256(); 59| 12.8k| const __m256i v_f800 = _mm256_set1_epi16((int16_t)0xf800); 60| 12.8k| const __m256i v_d800 = _mm256_set1_epi16((int16_t)0xd800); 61| 12.8k| const __m256i v_c080 = _mm256_set1_epi16((int16_t)0xc080); 62| 12.8k| const size_t safety_margin = 63| 12.8k| 12; // to avoid overruns, see issue 64| | // https://github.com/simdutf/simdutf/issues/92 65| | 66| 17.7k| while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { ------------------ | Branch (66:10): [True: 4.91k, False: 12.8k] ------------------ 67| 4.91k| __m256i in = _mm256_loadu_si256((__m256i *)buf); 68| 4.91k| if (big_endian) { ------------------ | Branch (68:9): [True: 4.91k, Folded] ------------------ 69| 4.91k| const __m256i swap = _mm256_setr_epi8( 70| 4.91k| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 71| 4.91k| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 72| 4.91k| in = _mm256_shuffle_epi8(in, swap); 73| 4.91k| } 74| | // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes 75| 4.91k| const __m256i v_ff80 = _mm256_set1_epi16((int16_t)0xff80); 76| 4.91k| if (_mm256_testz_si256(in, v_ff80)) { // ASCII fast path!!!! ------------------ | Branch (76:9): [True: 2.53k, False: 2.38k] ------------------ 77| | // 1. pack the bytes 78| 2.53k| const __m128i utf8_packed = _mm_packus_epi16( 79| 2.53k| _mm256_castsi256_si128(in), _mm256_extractf128_si256(in, 1)); 80| | // 2. store (16 bytes) 81| 2.53k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 82| | // 3. adjust pointers 83| 2.53k| buf += 16; 84| 2.53k| utf8_output += 16; 85| 2.53k| continue; // we are done for this round! 86| 2.53k| } 87| | // no bits set above 7th bit 88| 2.38k| const __m256i one_byte_bytemask = 89| 2.38k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_ff80), v_0000); 90| 2.38k| const uint32_t one_byte_bitmask = 91| 2.38k| static_cast(_mm256_movemask_epi8(one_byte_bytemask)); 92| | 93| | // no bits set above 11th bit 94| 2.38k| const __m256i one_or_two_bytes_bytemask = 95| 2.38k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_f800), v_0000); 96| 2.38k| const uint32_t one_or_two_bytes_bitmask = 97| 2.38k| static_cast(_mm256_movemask_epi8(one_or_two_bytes_bytemask)); 98| 2.38k| if (one_or_two_bytes_bitmask == 0xffffffff) { ------------------ | Branch (98:9): [True: 199, False: 2.18k] ------------------ 99| | 100| | // 1. prepare 2-byte values 101| | // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 102| | // expected output : [110a|aaaa|10bb|bbbb] x 8 103| 199| const __m256i v_1f00 = _mm256_set1_epi16((int16_t)0x1f00); 104| 199| const __m256i v_003f = _mm256_set1_epi16((int16_t)0x003f); 105| | 106| | // t0 = [000a|aaaa|bbbb|bb00] 107| 199| const __m256i t0 = _mm256_slli_epi16(in, 2); 108| | // t1 = [000a|aaaa|0000|0000] 109| 199| const __m256i t1 = _mm256_and_si256(t0, v_1f00); 110| | // t2 = [0000|0000|00bb|bbbb] 111| 199| const __m256i t2 = _mm256_and_si256(in, v_003f); 112| | // t3 = [000a|aaaa|00bb|bbbb] 113| 199| const __m256i t3 = _mm256_or_si256(t1, t2); 114| | // t4 = [110a|aaaa|10bb|bbbb] 115| 199| const __m256i t4 = _mm256_or_si256(t3, v_c080); 116| | 117| | // 2. merge ASCII and 2-byte codewords 118| 199| const __m256i utf8_unpacked = 119| 199| _mm256_blendv_epi8(t4, in, one_byte_bytemask); 120| | 121| | // 3. prepare bitmask for 8-bit lookup 122| 199| const uint32_t M0 = one_byte_bitmask & 0x55555555; 123| 199| const uint32_t M1 = M0 >> 7; 124| 199| const uint32_t M2 = (M1 | M0) & 0x00ff00ff; 125| | // 4. pack the bytes 126| | 127| 199| const uint8_t *row = 128| 199| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2)][0]; 129| 199| const uint8_t *row_2 = 130| 199| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2 >> 131| 199| 16)][0]; 132| | 133| 199| const __m128i shuffle = _mm_loadu_si128((__m128i *)(row + 1)); 134| 199| const __m128i shuffle_2 = _mm_loadu_si128((__m128i *)(row_2 + 1)); 135| | 136| 199| const __m256i utf8_packed = _mm256_shuffle_epi8( 137| 199| utf8_unpacked, _mm256_setr_m128i(shuffle, shuffle_2)); ------------------ | | 7| 199| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 8| 199| _mm256_castsi128_si256(xmm2), 2) ------------------ 138| | // 5. store bytes 139| 199| _mm_storeu_si128((__m128i *)utf8_output, 140| 199| _mm256_castsi256_si128(utf8_packed)); 141| 199| utf8_output += row[0]; 142| 199| _mm_storeu_si128((__m128i *)utf8_output, 143| 199| _mm256_extractf128_si256(utf8_packed, 1)); 144| 199| utf8_output += row_2[0]; 145| | 146| | // 6. adjust pointers 147| 199| buf += 16; 148| 199| continue; 149| 199| } 150| | // 1. Check if there are any surrogate word in the input chunk. 151| | // We have also deal with situation when there is a surrogate word 152| | // at the end of a chunk. 153| 2.18k| const __m256i surrogates_bytemask = 154| 2.18k| _mm256_cmpeq_epi16(_mm256_and_si256(in, v_f800), v_d800); 155| | 156| | // bitmask = 0x0000 if there are no surrogates 157| | // = 0xc000 if the last word is a surrogate 158| 2.18k| const uint32_t surrogates_bitmask = 159| 2.18k| static_cast(_mm256_movemask_epi8(surrogates_bytemask)); 160| | // It might seem like checking for surrogates_bitmask == 0xc000 could help. 161| | // However, it is likely an uncommon occurrence. 162| 2.18k| if (surrogates_bitmask == 0x00000000) { ------------------ | Branch (162:9): [True: 1.09k, False: 1.08k] ------------------ 163| | // case: code units from register produce either 1, 2 or 3 UTF-8 bytes 164| 1.09k| const __m256i dup_even = _mm256_setr_epi16( 165| 1.09k| 0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e, 166| 1.09k| 0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); 167| | 168| | /* In this branch we handle three cases: 169| | 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - 170| | single UFT-8 byte 171| | 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two 172| | UTF-8 bytes 173| | 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - 174| | three UTF-8 bytes 175| | 176| | We expand the input word (16-bit) into two code units (32-bit), thus 177| | we have room for four bytes. However, we need five distinct bit 178| | layouts. Note that the last byte in cases #2 and #3 is the same. 179| | 180| | We precompute byte 1 for case #1 and the common byte for cases #2 & #3 181| | in register t2. 182| | 183| | We precompute byte 1 for case #3 and -- **conditionally** -- precompute 184| | either byte 1 for case #2 or byte 2 for case #3. Note that they 185| | differ by exactly one bit. 186| | 187| | Finally from these two code units we build proper UTF-8 sequence, taking 188| | into account the case (i.e, the number of bytes to write). 189| | */ 190| | /** 191| | * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: 192| | * t2 => [0ccc|cccc] [10cc|cccc] 193| | * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) 194| | */ 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) 196| | // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] 197| 1.09k| const __m256i t0 = _mm256_shuffle_epi8(in, dup_even); 198| | // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] 199| 1.09k| const __m256i t1 = _mm256_and_si256(t0, simdutf_vec(0b0011111101111111)); ------------------ | | 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 200| | // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] 201| 1.09k| const __m256i t2 = _mm256_or_si256(t1, simdutf_vec(0b1000000000000000)); ------------------ | | 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 202| | 203| | // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] 204| 1.09k| const __m256i s0 = _mm256_srli_epi16(in, 4); 205| | // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] 206| 1.09k| const __m256i s1 = _mm256_and_si256(s0, simdutf_vec(0b0000111111111100)); ------------------ | | 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 207| | // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] 208| 1.09k| const __m256i s2 = _mm256_maddubs_epi16(s1, simdutf_vec(0x0140)); ------------------ | | 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 209| | // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] 210| 1.09k| const __m256i s3 = _mm256_or_si256(s2, simdutf_vec(0b1100000011100000)); ------------------ | | 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 211| 1.09k| const __m256i m0 = _mm256_andnot_si256(one_or_two_bytes_bytemask, 212| 1.09k| simdutf_vec(0b0100000000000000)); ------------------ | | 195| 1.09k|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 213| 1.09k| const __m256i s4 = _mm256_xor_si256(s3, m0); 214| 1.09k|#undef simdutf_vec 215| | 216| | // 4. expand code units 16-bit => 32-bit 217| 1.09k| const __m256i out0 = _mm256_unpacklo_epi16(t2, s4); 218| 1.09k| const __m256i out1 = _mm256_unpackhi_epi16(t2, s4); 219| | 220| | // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle 221| 1.09k| const uint32_t mask = (one_byte_bitmask & 0x55555555) | 222| 1.09k| (one_or_two_bytes_bitmask & 0xaaaaaaaa); 223| | // Due to the wider registers, the following path is less likely to be 224| | // useful. 225| | /*if(mask == 0) { 226| | // We only have three-byte code units. Use fast path. 227| | const __m256i shuffle = 228| | _mm256_setr_epi8(2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1, 229| | 2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1); const __m256i utf8_0 = 230| | _mm256_shuffle_epi8(out0, shuffle); const __m256i utf8_1 = 231| | _mm256_shuffle_epi8(out1, shuffle); 232| | _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_0)); 233| | utf8_output += 12; 234| | _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_1)); 235| | utf8_output += 12; 236| | _mm_storeu_si128((__m128i*)utf8_output, 237| | _mm256_extractf128_si256(utf8_0,1)); utf8_output += 12; 238| | _mm_storeu_si128((__m128i*)utf8_output, 239| | _mm256_extractf128_si256(utf8_1,1)); utf8_output += 12; buf += 16; 240| | continue; 241| | }*/ 242| 1.09k| const uint8_t mask0 = uint8_t(mask); 243| 1.09k| const uint8_t *row0 = 244| 1.09k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; 245| 1.09k| const __m128i shuffle0 = _mm_loadu_si128((__m128i *)(row0 + 1)); 246| 1.09k| const __m128i utf8_0 = 247| 1.09k| _mm_shuffle_epi8(_mm256_castsi256_si128(out0), shuffle0); 248| | 249| 1.09k| const uint8_t mask1 = static_cast(mask >> 8); 250| 1.09k| const uint8_t *row1 = 251| 1.09k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; 252| 1.09k| const __m128i shuffle1 = _mm_loadu_si128((__m128i *)(row1 + 1)); 253| 1.09k| const __m128i utf8_1 = 254| 1.09k| _mm_shuffle_epi8(_mm256_castsi256_si128(out1), shuffle1); 255| | 256| 1.09k| const uint8_t mask2 = static_cast(mask >> 16); 257| 1.09k| const uint8_t *row2 = 258| 1.09k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask2][0]; 259| 1.09k| const __m128i shuffle2 = _mm_loadu_si128((__m128i *)(row2 + 1)); 260| 1.09k| const __m128i utf8_2 = 261| 1.09k| _mm_shuffle_epi8(_mm256_extractf128_si256(out0, 1), shuffle2); 262| | 263| 1.09k| const uint8_t mask3 = static_cast(mask >> 24); 264| 1.09k| const uint8_t *row3 = 265| 1.09k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask3][0]; 266| 1.09k| const __m128i shuffle3 = _mm_loadu_si128((__m128i *)(row3 + 1)); 267| 1.09k| const __m128i utf8_3 = 268| 1.09k| _mm_shuffle_epi8(_mm256_extractf128_si256(out1, 1), shuffle3); 269| | 270| 1.09k| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 271| 1.09k| utf8_output += row0[0]; 272| 1.09k| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 273| 1.09k| utf8_output += row1[0]; 274| 1.09k| _mm_storeu_si128((__m128i *)utf8_output, utf8_2); 275| 1.09k| utf8_output += row2[0]; 276| 1.09k| _mm_storeu_si128((__m128i *)utf8_output, utf8_3); 277| 1.09k| utf8_output += row3[0]; 278| 1.09k| buf += 16; 279| | // surrogate pair(s) in a register 280| 1.09k| } else { 281| | // Let us do a scalar fallback. 282| | // It may seem wasteful to use scalar code, but being efficient with SIMD 283| | // in the presence of surrogate pairs may require non-trivial tables. 284| 1.08k| size_t forward = 15; 285| 1.08k| size_t k = 0; 286| 1.08k| if (size_t(end - buf) < forward + 1) { ------------------ | Branch (286:11): [True: 0, False: 1.08k] ------------------ 287| 0| forward = size_t(end - buf - 1); 288| 0| } 289| 15.2k| for (; k < forward; k++) { ------------------ | Branch (289:14): [True: 14.2k, False: 1.04k] ------------------ 290| 14.2k| uint16_t word = scalar::utf16::swap_if_needed(buf[k]); 291| 14.2k| if ((word & 0xFF80) == 0) { ------------------ | Branch (291:13): [True: 8.71k, False: 5.49k] ------------------ 292| 8.71k| *utf8_output++ = char(word); 293| 8.71k| } else if ((word & 0xF800) == 0) { ------------------ | Branch (293:20): [True: 647, False: 4.84k] ------------------ 294| 647| *utf8_output++ = char((word >> 6) | 0b11000000); 295| 647| *utf8_output++ = char((word & 0b111111) | 0b10000000); 296| 4.84k| } else if ((word & 0xF800) != 0xD800) { ------------------ | Branch (296:20): [True: 3.07k, False: 1.77k] ------------------ 297| 3.07k| *utf8_output++ = char((word >> 12) | 0b11100000); 298| 3.07k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 299| 3.07k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 300| 3.07k| } else { 301| | // must be a surrogate pair 302| 1.77k| uint16_t diff = uint16_t(word - 0xD800); 303| 1.77k| uint16_t next_word = 304| 1.77k| scalar::utf16::swap_if_needed(buf[k + 1]); 305| 1.77k| k++; 306| 1.77k| uint16_t diff2 = uint16_t(next_word - 0xDC00); 307| 1.77k| if ((diff | diff2) > 0x3FF) { ------------------ | Branch (307:15): [True: 44, False: 1.73k] ------------------ 308| 44| return std::make_pair(nullptr, utf8_output); 309| 44| } 310| 1.73k| uint32_t value = (diff << 10) + diff2 + 0x10000; 311| 1.73k| *utf8_output++ = char((value >> 18) | 0b11110000); 312| 1.73k| *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); 313| 1.73k| *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); 314| 1.73k| *utf8_output++ = char((value & 0b111111) | 0b10000000); 315| 1.73k| } 316| 14.2k| } 317| 1.04k| buf += k; 318| 1.04k| } 319| 2.18k| } // while 320| 12.8k| return std::make_pair(buf, utf8_output); 321| 12.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128avx2_convert_utf32_to_latin1EPKDimPc: 3| 6.85k| char *latin1_output) { 4| 6.85k| const size_t rounded_len = 5| 6.85k| len & ~0x1F; // Round down to nearest multiple of 32 6| | 7| 6.85k| const __m256i high_bytes_mask = _mm256_set1_epi32(0xFFFFFF00); 8| | 9| 16.4k| for (size_t i = 0; i < rounded_len; i += 4 * 8) { ------------------ | Branch (9:22): [True: 9.57k, False: 6.85k] ------------------ 10| 9.57k| __m256i a = _mm256_loadu_si256((__m256i *)(buf + 0 * 8)); 11| 9.57k| __m256i b = _mm256_loadu_si256((__m256i *)(buf + 1 * 8)); 12| 9.57k| __m256i c = _mm256_loadu_si256((__m256i *)(buf + 2 * 8)); 13| 9.57k| __m256i d = _mm256_loadu_si256((__m256i *)(buf + 3 * 8)); 14| | 15| 9.57k| const __m256i check_combined = 16| 9.57k| _mm256_or_si256(_mm256_or_si256(a, b), _mm256_or_si256(c, d)); 17| | 18| 9.57k| if (!_mm256_testz_si256(check_combined, high_bytes_mask)) { ------------------ | Branch (18:9): [True: 0, False: 9.57k] ------------------ 19| 0| return std::make_pair(nullptr, latin1_output); 20| 0| } 21| | 22| 9.57k| b = _mm256_slli_epi32(b, 1 * 8); 23| 9.57k| c = _mm256_slli_epi32(c, 2 * 8); 24| 9.57k| d = _mm256_slli_epi32(d, 3 * 8); 25| | 26| | // clang-format off 27| | 28| | // a = [.. .. .. a7|.. .. .. a6|.. .. .. a5|.. .. .. a4||.. .. .. a3|.. .. .. a2|.. .. .. a1|.. .. .. a0] 29| | // b = [.. .. b7 ..|.. .. b6 ..|.. .. b5 ..|.. .. b4 ..||.. .. b3 ..|.. .. b2 ..|.. .. b1 ..|.. .. b0 ..] 30| | // c = [.. c7 .. ..|.. c6 .. ..|.. c5 .. ..|.. c4 .. ..||.. c3 .. ..|.. c2 .. ..|.. c1 .. ..|.. c0 .. ..] 31| | // d = [d7 .. .. ..|d6 .. .. ..|d5 .. .. ..|d4 .. .. ..||d3 .. .. ..|d2 .. .. ..|d1 .. .. ..|d0 .. .. ..] 32| | 33| | // t0 = [d7 c7 b7 a7|d6 c6 b6 a6|d5 c5 b5 a5|d4 c4 b4 a4||d3 c3 b3 a3|d2 c2 b2 a2|d1 c1 b1 a1|d0 c0 b0 a0] 34| 9.57k| const __m256i t0 = 35| 9.57k| _mm256_or_si256(_mm256_or_si256(a, b), _mm256_or_si256(c, d)); 36| | 37| | // shuffle bytes within 128-bit lanes 38| | // t1 = [d7 d6 d5 d4|c7 c6 c5 c4|b7 b6 b5 b4|a7 a6 a5 a4||d3 d2 d1 d0|c3 c2 c1 c0|b3 b2 b1 b0|a3 a2 a1 a0] 39| 9.57k| const __m256i shuffle_bytes = 40| 9.57k| _mm256_setr_epi8(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15, 41| 9.57k| 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); 42| | 43| 9.57k| const __m256i t1 = _mm256_shuffle_epi8(t0, shuffle_bytes); 44| | 45| | // reshuffle dwords 46| | // t2 = [d7 d6 d5 d4|d3 d2 d1 d0|c7 c6 c5 c4|c3 c2 c1 c0||b7 b6 b5 b4|b3 b2 b1 b0|a7 a6 a5 a4|a3 a2 a1 a0] 47| 9.57k| const __m256i shuffle_dwords = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); 48| 9.57k| const __m256i t2 = _mm256_permutevar8x32_epi32(t1, shuffle_dwords); 49| |// clang format on 50| | 51| 9.57k| _mm256_storeu_si256((__m256i *)latin1_output, t2); 52| | 53| 9.57k| latin1_output += 32; 54| 9.57k| buf += 32; 55| 9.57k| } 56| | 57| 6.85k| return std::make_pair(buf, latin1_output); 58| 6.85k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_126avx2_convert_utf32_to_utf8EPKDimPc: 2| 6.02k|avx2_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_output) { 3| 6.02k| const char32_t *end = buf + len; 4| 6.02k| const __m256i v_0000 = _mm256_setzero_si256(); 5| 6.02k| const __m256i v_ffff0000 = _mm256_set1_epi32((uint32_t)0xffff0000); 6| 6.02k| const __m256i v_ff80 = _mm256_set1_epi16((uint16_t)0xff80); 7| 6.02k| const __m256i v_f800 = _mm256_set1_epi16((uint16_t)0xf800); 8| 6.02k| const __m256i v_c080 = _mm256_set1_epi16((uint16_t)0xc080); 9| 6.02k| const __m256i v_7fffffff = _mm256_set1_epi32((uint32_t)0x7fffffff); 10| 6.02k| __m256i running_max = _mm256_setzero_si256(); 11| 6.02k| __m256i forbidden_bytemask = _mm256_setzero_si256(); 12| | 13| 6.02k| const size_t safety_margin = 14| 6.02k| 12; // to avoid overruns, see issue 15| | // https://github.com/simdutf/simdutf/issues/92 16| | 17| 9.39k| while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { ------------------ | Branch (17:10): [True: 3.37k, False: 6.02k] ------------------ 18| 3.37k| __m256i in = _mm256_loadu_si256((__m256i *)buf); 19| 3.37k| __m256i nextin = _mm256_loadu_si256((__m256i *)buf + 1); 20| 3.37k| running_max = _mm256_max_epu32(_mm256_max_epu32(in, running_max), nextin); 21| | 22| | // Pack 32-bit UTF-32 code units to 16-bit UTF-16 code units with unsigned 23| | // saturation 24| 3.37k| __m256i in_16 = _mm256_packus_epi32(_mm256_and_si256(in, v_7fffffff), 25| 3.37k| _mm256_and_si256(nextin, v_7fffffff)); 26| 3.37k| in_16 = _mm256_permute4x64_epi64(in_16, 0b11011000); 27| | 28| | // Try to apply UTF-16 => UTF-8 routine on 256 bits 29| | // (haswell/avx2_convert_utf16_to_utf8.cpp) 30| | 31| 3.37k| if (_mm256_testz_si256(in_16, v_ff80)) { // ASCII fast path!!!! ------------------ | Branch (31:9): [True: 2.31k, False: 1.06k] ------------------ 32| | // 1. pack the bytes 33| 2.31k| const __m128i utf8_packed = _mm_packus_epi16( 34| 2.31k| _mm256_castsi256_si128(in_16), _mm256_extractf128_si256(in_16, 1)); 35| | // 2. store (16 bytes) 36| 2.31k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 37| | // 3. adjust pointers 38| 2.31k| buf += 16; 39| 2.31k| utf8_output += 16; 40| 2.31k| continue; // we are done for this round! 41| 2.31k| } 42| | // no bits set above 7th bit 43| 1.06k| const __m256i one_byte_bytemask = 44| 1.06k| _mm256_cmpeq_epi16(_mm256_and_si256(in_16, v_ff80), v_0000); 45| 1.06k| const uint32_t one_byte_bitmask = 46| 1.06k| static_cast(_mm256_movemask_epi8(one_byte_bytemask)); 47| | 48| | // no bits set above 11th bit 49| 1.06k| const __m256i one_or_two_bytes_bytemask = 50| 1.06k| _mm256_cmpeq_epi16(_mm256_and_si256(in_16, v_f800), v_0000); 51| 1.06k| const uint32_t one_or_two_bytes_bitmask = 52| 1.06k| static_cast(_mm256_movemask_epi8(one_or_two_bytes_bytemask)); 53| 1.06k| if (one_or_two_bytes_bitmask == 0xffffffff) { ------------------ | Branch (53:9): [True: 190, False: 877] ------------------ 54| | // 1. prepare 2-byte values 55| | // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 56| | // expected output : [110a|aaaa|10bb|bbbb] x 8 57| 190| const __m256i v_1f00 = _mm256_set1_epi16((int16_t)0x1f00); 58| 190| const __m256i v_003f = _mm256_set1_epi16((int16_t)0x003f); 59| | 60| | // t0 = [000a|aaaa|bbbb|bb00] 61| 190| const __m256i t0 = _mm256_slli_epi16(in_16, 2); 62| | // t1 = [000a|aaaa|0000|0000] 63| 190| const __m256i t1 = _mm256_and_si256(t0, v_1f00); 64| | // t2 = [0000|0000|00bb|bbbb] 65| 190| const __m256i t2 = _mm256_and_si256(in_16, v_003f); 66| | // t3 = [000a|aaaa|00bb|bbbb] 67| 190| const __m256i t3 = _mm256_or_si256(t1, t2); 68| | // t4 = [110a|aaaa|10bb|bbbb] 69| 190| const __m256i t4 = _mm256_or_si256(t3, v_c080); 70| | 71| | // 2. merge ASCII and 2-byte codewords 72| 190| const __m256i utf8_unpacked = 73| 190| _mm256_blendv_epi8(t4, in_16, one_byte_bytemask); 74| | 75| | // 3. prepare bitmask for 8-bit lookup 76| 190| const uint32_t M0 = one_byte_bitmask & 0x55555555; 77| 190| const uint32_t M1 = M0 >> 7; 78| 190| const uint32_t M2 = (M1 | M0) & 0x00ff00ff; 79| | // 4. pack the bytes 80| | 81| 190| const uint8_t *row = 82| 190| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2)][0]; 83| 190| const uint8_t *row_2 = 84| 190| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[uint8_t(M2 >> 85| 190| 16)][0]; 86| | 87| 190| const __m128i shuffle = _mm_loadu_si128((__m128i *)(row + 1)); 88| 190| const __m128i shuffle_2 = _mm_loadu_si128((__m128i *)(row_2 + 1)); 89| | 90| 190| const __m256i utf8_packed = _mm256_shuffle_epi8( 91| 190| utf8_unpacked, _mm256_setr_m128i(shuffle, shuffle_2)); ------------------ | | 7| 190| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 8| 190| _mm256_castsi128_si256(xmm2), 2) ------------------ 92| | // 5. store bytes 93| 190| _mm_storeu_si128((__m128i *)utf8_output, 94| 190| _mm256_castsi256_si128(utf8_packed)); 95| 190| utf8_output += row[0]; 96| 190| _mm_storeu_si128((__m128i *)utf8_output, 97| 190| _mm256_extractf128_si256(utf8_packed, 1)); 98| 190| utf8_output += row_2[0]; 99| | 100| | // 6. adjust pointers 101| 190| buf += 16; 102| 190| continue; 103| 190| } 104| | // Must check for overflow in packing 105| 877| const __m256i saturation_bytemask = _mm256_cmpeq_epi32( 106| 877| _mm256_and_si256(_mm256_or_si256(in, nextin), v_ffff0000), v_0000); 107| 877| const uint32_t saturation_bitmask = 108| 877| static_cast(_mm256_movemask_epi8(saturation_bytemask)); 109| 877| if (saturation_bitmask == 0xffffffff) { ------------------ | Branch (109:9): [True: 117, False: 760] ------------------ 110| | // case: code units from register produce either 1, 2 or 3 UTF-8 bytes 111| 117| const __m256i v_d800 = _mm256_set1_epi16((uint16_t)0xd800); 112| 117| forbidden_bytemask = _mm256_or_si256( 113| 117| forbidden_bytemask, 114| 117| _mm256_cmpeq_epi16(_mm256_and_si256(in_16, v_f800), v_d800)); 115| | 116| 117| const __m256i dup_even = _mm256_setr_epi16( 117| 117| 0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e, 118| 117| 0x0000, 0x0202, 0x0404, 0x0606, 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); 119| | 120| | /* In this branch we handle three cases: 121| | 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - 122| | single UFT-8 byte 123| | 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two 124| | UTF-8 bytes 125| | 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - 126| | three UTF-8 bytes 127| | 128| | We expand the input word (16-bit) into two code units (32-bit), thus 129| | we have room for four bytes. However, we need five distinct bit 130| | layouts. Note that the last byte in cases #2 and #3 is the same. 131| | 132| | We precompute byte 1 for case #1 and the common byte for cases #2 & #3 133| | in register t2. 134| | 135| | We precompute byte 1 for case #3 and -- **conditionally** -- precompute 136| | either byte 1 for case #2 or byte 2 for case #3. Note that they 137| | differ by exactly one bit. 138| | 139| | Finally from these two code units we build proper UTF-8 sequence, taking 140| | into account the case (i.e, the number of bytes to write). 141| | */ 142| | /** 143| | * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: 144| | * t2 => [0ccc|cccc] [10cc|cccc] 145| | * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) 146| | */ 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) 148| | // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] 149| 117| const __m256i t0 = _mm256_shuffle_epi8(in_16, dup_even); 150| | // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] 151| 117| const __m256i t1 = _mm256_and_si256(t0, simdutf_vec(0b0011111101111111)); ------------------ | | 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 152| | // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] 153| 117| const __m256i t2 = _mm256_or_si256(t1, simdutf_vec(0b1000000000000000)); ------------------ | | 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 154| | 155| | // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] 156| 117| const __m256i s0 = _mm256_srli_epi16(in_16, 4); 157| | // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] 158| 117| const __m256i s1 = _mm256_and_si256(s0, simdutf_vec(0b0000111111111100)); ------------------ | | 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 159| | // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] 160| 117| const __m256i s2 = _mm256_maddubs_epi16(s1, simdutf_vec(0x0140)); ------------------ | | 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 161| | // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] 162| 117| const __m256i s3 = _mm256_or_si256(s2, simdutf_vec(0b1100000011100000)); ------------------ | | 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 163| 117| const __m256i m0 = _mm256_andnot_si256(one_or_two_bytes_bytemask, 164| 117| simdutf_vec(0b0100000000000000)); ------------------ | | 147| 117|#define simdutf_vec(x) _mm256_set1_epi16(static_cast(x)) ------------------ 165| 117| const __m256i s4 = _mm256_xor_si256(s3, m0); 166| 117|#undef simdutf_vec 167| | 168| | // 4. expand code units 16-bit => 32-bit 169| 117| const __m256i out0 = _mm256_unpacklo_epi16(t2, s4); 170| 117| const __m256i out1 = _mm256_unpackhi_epi16(t2, s4); 171| | 172| | // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle 173| 117| const uint32_t mask = (one_byte_bitmask & 0x55555555) | 174| 117| (one_or_two_bytes_bitmask & 0xaaaaaaaa); 175| | // Due to the wider registers, the following path is less likely to be 176| | // useful. 177| | /*if(mask == 0) { 178| | // We only have three-byte code units. Use fast path. 179| | const __m256i shuffle = 180| | _mm256_setr_epi8(2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1, 181| | 2,3,1,6,7,5,10,11,9,14,15,13,-1,-1,-1,-1); const __m256i utf8_0 = 182| | _mm256_shuffle_epi8(out0, shuffle); const __m256i utf8_1 = 183| | _mm256_shuffle_epi8(out1, shuffle); 184| | _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_0)); 185| | utf8_output += 12; 186| | _mm_storeu_si128((__m128i*)utf8_output, _mm256_castsi256_si128(utf8_1)); 187| | utf8_output += 12; 188| | _mm_storeu_si128((__m128i*)utf8_output, 189| | _mm256_extractf128_si256(utf8_0,1)); utf8_output += 12; 190| | _mm_storeu_si128((__m128i*)utf8_output, 191| | _mm256_extractf128_si256(utf8_1,1)); utf8_output += 12; buf += 16; 192| | continue; 193| | }*/ 194| 117| const uint8_t mask0 = uint8_t(mask); 195| 117| const uint8_t *row0 = 196| 117| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; 197| 117| const __m128i shuffle0 = _mm_loadu_si128((__m128i *)(row0 + 1)); 198| 117| const __m128i utf8_0 = 199| 117| _mm_shuffle_epi8(_mm256_castsi256_si128(out0), shuffle0); 200| | 201| 117| const uint8_t mask1 = static_cast(mask >> 8); 202| 117| const uint8_t *row1 = 203| 117| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; 204| 117| const __m128i shuffle1 = _mm_loadu_si128((__m128i *)(row1 + 1)); 205| 117| const __m128i utf8_1 = 206| 117| _mm_shuffle_epi8(_mm256_castsi256_si128(out1), shuffle1); 207| | 208| 117| const uint8_t mask2 = static_cast(mask >> 16); 209| 117| const uint8_t *row2 = 210| 117| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask2][0]; 211| 117| const __m128i shuffle2 = _mm_loadu_si128((__m128i *)(row2 + 1)); 212| 117| const __m128i utf8_2 = 213| 117| _mm_shuffle_epi8(_mm256_extractf128_si256(out0, 1), shuffle2); 214| | 215| 117| const uint8_t mask3 = static_cast(mask >> 24); 216| 117| const uint8_t *row3 = 217| 117| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask3][0]; 218| 117| const __m128i shuffle3 = _mm_loadu_si128((__m128i *)(row3 + 1)); 219| 117| const __m128i utf8_3 = 220| 117| _mm_shuffle_epi8(_mm256_extractf128_si256(out1, 1), shuffle3); 221| | 222| 117| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 223| 117| utf8_output += row0[0]; 224| 117| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 225| 117| utf8_output += row1[0]; 226| 117| _mm_storeu_si128((__m128i *)utf8_output, utf8_2); 227| 117| utf8_output += row2[0]; 228| 117| _mm_storeu_si128((__m128i *)utf8_output, utf8_3); 229| 117| utf8_output += row3[0]; 230| 117| buf += 16; 231| 760| } else { 232| | // case: at least one 32-bit word is larger than 0xFFFF <=> it will 233| | // produce four UTF-8 bytes. Let us do a scalar fallback. It may seem 234| | // wasteful to use scalar code, but being efficient with SIMD may require 235| | // large, non-trivial tables? 236| 760| size_t forward = 15; 237| 760| size_t k = 0; 238| 760| if (size_t(end - buf) < forward + 1) { ------------------ | Branch (238:11): [True: 0, False: 760] ------------------ 239| 0| forward = size_t(end - buf - 1); 240| 0| } 241| 12.1k| for (; k < forward; k++) { ------------------ | Branch (241:14): [True: 11.4k, False: 760] ------------------ 242| 11.4k| uint32_t word = buf[k]; 243| 11.4k| if ((word & 0xFFFFFF80) == 0) { // 1-byte (ASCII) ------------------ | Branch (243:13): [True: 9.29k, False: 2.10k] ------------------ 244| 9.29k| *utf8_output++ = char(word); 245| 9.29k| } else if ((word & 0xFFFFF800) == 0) { // 2-byte ------------------ | Branch (245:20): [True: 476, False: 1.62k] ------------------ 246| 476| *utf8_output++ = char((word >> 6) | 0b11000000); 247| 476| *utf8_output++ = char((word & 0b111111) | 0b10000000); 248| 1.62k| } else if ((word & 0xFFFF0000) == 0) { // 3-byte ------------------ | Branch (248:20): [True: 489, False: 1.13k] ------------------ 249| 489| if (word >= 0xD800 && word <= 0xDFFF) { ------------------ | Branch (249:15): [True: 231, False: 258] | Branch (249:33): [True: 0, False: 231] ------------------ 250| 0| return std::make_pair(nullptr, utf8_output); 251| 0| } 252| 489| *utf8_output++ = char((word >> 12) | 0b11100000); 253| 489| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 254| 489| *utf8_output++ = char((word & 0b111111) | 0b10000000); 255| 1.13k| } else { // 4-byte 256| 1.13k| if (word > 0x10FFFF) { ------------------ | Branch (256:15): [True: 0, False: 1.13k] ------------------ 257| 0| return std::make_pair(nullptr, utf8_output); 258| 0| } 259| 1.13k| *utf8_output++ = char((word >> 18) | 0b11110000); 260| 1.13k| *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); 261| 1.13k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 262| 1.13k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 263| 1.13k| } 264| 11.4k| } 265| 760| buf += k; 266| 760| } 267| 877| } // while 268| | 269| | // check for invalid input 270| 6.02k| const __m256i v_10ffff = _mm256_set1_epi32((uint32_t)0x10ffff); 271| 6.02k| if (static_cast(_mm256_movemask_epi8(_mm256_cmpeq_epi32( ------------------ | Branch (271:7): [True: 0, False: 6.02k] ------------------ 272| 6.02k| _mm256_max_epu32(running_max, v_10ffff), v_10ffff))) != 0xffffffff) { 273| 0| return std::make_pair(nullptr, utf8_output); 274| 0| } 275| | 276| 6.02k| if (static_cast(_mm256_movemask_epi8(forbidden_bytemask)) != 0) { ------------------ | Branch (276:7): [True: 0, False: 6.02k] ------------------ 277| 0| return std::make_pair(nullptr, utf8_output); 278| 0| } 279| | 280| 6.02k| return std::make_pair(buf, utf8_output); 281| 6.02k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_129convert_masked_utf8_to_latin1EPKcmRPc: 9| 33.5k| char *&latin1_output) { 10| | // we use an approach where we try to process up to 12 input bytes. 11| | // Why 12 input bytes and not 16? Because we are concerned with the size of 12| | // the lookup tables. Also 12 is nicely divisible by two and three. 13| | // 14| | // 15| | // Optimization note: our main path below is load-latency dependent. Thus it 16| | // is maybe beneficial to have fast paths that depend on branch prediction but 17| | // have less latency. This results in more instructions but, potentially, also 18| | // higher speeds. 19| | // 20| 33.5k| const __m128i in = _mm_loadu_si128((__m128i *)input); 21| | 22| 33.5k| const uint16_t input_utf8_end_of_code_point_mask = 23| 33.5k| utf8_end_of_code_point_mask & 24| 33.5k| 0xfff; // we are only processing 12 bytes in case it is not all ASCII 25| | 26| 33.5k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (26:7): [True: 264, False: 33.2k] ------------------ 27| | // We process the data in chunks of 12 bytes. 28| 264| _mm_storeu_si128(reinterpret_cast<__m128i *>(latin1_output), in); 29| 264| latin1_output += 12; // We wrote 12 characters. 30| 264| return 12; // We consumed 1 bytes. 31| 264| } 32| | /// We do not have a fast path available, so we fallback. 33| 33.2k| const uint8_t idx = 34| 33.2k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; 35| 33.2k| const uint8_t consumed = 36| 33.2k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; 37| | // this indicates an invalid input: 38| 33.2k| if (idx >= 64) { ------------------ | Branch (38:7): [True: 5.14k, False: 28.1k] ------------------ 39| 5.14k| return consumed; 40| 5.14k| } 41| | // Here we should have (idx < 64), if not, there is a bug in the validation or 42| | // elsewhere. SIX (6) input code-code units this is a relatively easy scenario 43| | // we process SIX (6) input code-code units. The max length in bytes of six 44| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 45| | // processors where pdep/pext is fast, we might be able to use a small lookup 46| | // table. 47| 28.1k| const __m128i sh = 48| 28.1k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 49| 28.1k| const __m128i perm = _mm_shuffle_epi8(in, sh); 50| 28.1k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 51| 28.1k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 52| 28.1k| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 53| 28.1k| const __m128i latin1_packed = _mm_packus_epi16(composed, composed); 54| | // writing 8 bytes even though we only care about the first 6 bytes. 55| | // performance note: it would be faster to use _mm_storeu_si128, we should 56| | // investigate. 57| 28.1k| _mm_storel_epi64((__m128i *)latin1_output, latin1_packed); 58| 28.1k| latin1_output += 6; // We wrote 6 bytes. 59| 28.1k| return consumed; 60| 33.2k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128convert_masked_utf8_to_utf16ILNS_10endiannessE0EEEmPKcmRPDs: 10| 16.8k| char16_t *&utf16_output) { 11| | // we use an approach where we try to process up to 12 input bytes. 12| | // Why 12 input bytes and not 16? Because we are concerned with the size of 13| | // the lookup tables. Also 12 is nicely divisible by two and three. 14| | // 15| | // 16| | // Optimization note: our main path below is load-latency dependent. Thus it 17| | // is maybe beneficial to have fast paths that depend on branch prediction but 18| | // have less latency. This results in more instructions but, potentially, also 19| | // higher speeds. 20| | // 21| | // We first try a few fast paths. 22| 16.8k| const __m128i swap = 23| 16.8k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 24| 16.8k| const __m128i in = _mm_loadu_si128((__m128i *)input); 25| 16.8k| const uint16_t input_utf8_end_of_code_point_mask = 26| 16.8k| utf8_end_of_code_point_mask & 0xfff; 27| 16.8k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (27:7): [True: 113, False: 16.6k] ------------------ 28| | // We process the data in chunks of 12 bytes. 29| 113| __m256i ascii = _mm256_cvtepu8_epi16(in); 30| 113| if (big_endian) { ------------------ | Branch (30:9): [Folded, False: 113] ------------------ 31| 0| const __m256i swap256 = _mm256_setr_epi8( 32| 0| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 33| 0| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 34| 0| ascii = _mm256_shuffle_epi8(ascii, swap256); 35| 0| } 36| 113| _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf16_output), ascii); 37| 113| utf16_output += 12; // We wrote 12 16-bit characters. 38| 113| return 12; // We consumed 12 bytes. 39| 113| } 40| 16.6k| if (((utf8_end_of_code_point_mask & 0xffff) == 0xaaaa)) { ------------------ | Branch (40:7): [True: 155, False: 16.5k] ------------------ 41| | // We want to take 8 2-byte UTF-8 code units and turn them into 8 2-byte 42| | // UTF-16 code units. There is probably a more efficient sequence, but the 43| | // following might do. 44| 155| const __m128i sh = 45| 155| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 46| 155| const __m128i perm = _mm_shuffle_epi8(in, sh); 47| 155| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 48| 155| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 49| 155| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 50| 155| if (big_endian) ------------------ | Branch (50:9): [Folded, False: 155] ------------------ 51| 0| composed = _mm_shuffle_epi8(composed, swap); 52| 155| _mm_storeu_si128((__m128i *)utf16_output, composed); 53| 155| utf16_output += 8; // We wrote 16 bytes, 8 code points. 54| 155| return 16; 55| 155| } 56| 16.5k| if (input_utf8_end_of_code_point_mask == 0x924) { ------------------ | Branch (56:7): [True: 3.84k, False: 12.6k] ------------------ 57| | // We want to take 4 3-byte UTF-8 code units and turn them into 4 2-byte 58| | // UTF-16 code units. There is probably a more efficient sequence, but the 59| | // following might do. 60| 3.84k| const __m128i sh = 61| 3.84k| _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); 62| 3.84k| const __m128i perm = _mm_shuffle_epi8(in, sh); 63| 3.84k| const __m128i ascii = 64| 3.84k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 65| 3.84k| const __m128i middlebyte = 66| 3.84k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 67| 3.84k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 68| 3.84k| const __m128i highbyte = 69| 3.84k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 70| 3.84k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 71| 3.84k| const __m128i composed = 72| 3.84k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 73| 3.84k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 74| 3.84k| if (big_endian) ------------------ | Branch (74:9): [Folded, False: 3.84k] ------------------ 75| 0| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 76| 3.84k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 77| 3.84k| utf16_output += 4; 78| 3.84k| return 12; 79| 3.84k| } 80| | 81| 12.6k| const uint8_t idx = simdutf::tables::utf8_to_utf16::utf8bigindex 82| 12.6k| [input_utf8_end_of_code_point_mask][0]; 83| 12.6k| const uint8_t consumed = simdutf::tables::utf8_to_utf16::utf8bigindex 84| 12.6k| [input_utf8_end_of_code_point_mask][1]; 85| 12.6k| if (idx < 64) { ------------------ | Branch (85:7): [True: 5.64k, False: 7.05k] ------------------ 86| | // SIX (6) input code-code units 87| | // this is a relatively easy scenario 88| | // we process SIX (6) input code-code units. The max length in bytes of six 89| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 90| | // processors where pdep/pext is fast, we might be able to use a small 91| | // lookup table. 92| 5.64k| const __m128i sh = _mm_loadu_si128( 93| 5.64k| (const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); 94| 5.64k| const __m128i perm = _mm_shuffle_epi8(in, sh); 95| 5.64k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 96| 5.64k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 97| 5.64k| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 98| 5.64k| if (big_endian) ------------------ | Branch (98:9): [Folded, False: 5.64k] ------------------ 99| 0| composed = _mm_shuffle_epi8(composed, swap); 100| 5.64k| _mm_storeu_si128((__m128i *)utf16_output, composed); 101| 5.64k| utf16_output += 6; // We wrote 12 bytes, 6 code points. There is a potential 102| | // overflow of 4 bytes. 103| 7.05k| } else if (idx < 145) { ------------------ | Branch (103:14): [True: 1.89k, False: 5.15k] ------------------ 104| | // FOUR (4) input code-code units 105| 1.89k| const __m128i sh = _mm_loadu_si128( 106| 1.89k| (const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); 107| 1.89k| const __m128i perm = _mm_shuffle_epi8(in, sh); 108| 1.89k| const __m128i ascii = 109| 1.89k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 110| 1.89k| const __m128i middlebyte = 111| 1.89k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 112| 1.89k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 113| 1.89k| const __m128i highbyte = 114| 1.89k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 115| 1.89k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 116| 1.89k| const __m128i composed = 117| 1.89k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 118| 1.89k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 119| 1.89k| if (big_endian) ------------------ | Branch (119:9): [Folded, False: 1.89k] ------------------ 120| 0| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 121| 1.89k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 122| 1.89k| utf16_output += 4; // Here we overflow by 8 bytes. 123| 5.15k| } else if (idx < 209) { ------------------ | Branch (123:14): [True: 4.93k, False: 222] ------------------ 124| | // TWO (2) input code-code units 125| | ////////////// 126| | // There might be garbage inputs where a leading byte mascarades as a 127| | // four-byte leading byte (by being followed by 3 continuation byte), but is 128| | // not greater than 0xf0. This could trigger a buffer overflow if we only 129| | // counted leading bytes of the form 0xf0 as generating surrogate pairs, 130| | // without further UTF-8 validation. Thus we must be careful to ensure that 131| | // only leading bytes at least as large as 0xf0 generate surrogate pairs. We 132| | // do as at the cost of an extra mask. 133| | ///////////// 134| 4.93k| const __m128i sh = _mm_loadu_si128( 135| 4.93k| (const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); 136| 4.93k| const __m128i perm = _mm_shuffle_epi8(in, sh); 137| 4.93k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); 138| 4.93k| const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); 139| 4.93k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 140| 4.93k| __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); 141| | // correct for spurious high bit 142| 4.93k| const __m128i correct = 143| 4.93k| _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); 144| 4.93k| middlehighbyte = _mm_xor_si128(correct, middlehighbyte); 145| 4.93k| const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); 146| | // We deliberately carry the leading four bits in highbyte if they are 147| | // present, we remove them later when computing hightenbits. 148| 4.93k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0xff000000)); 149| 4.93k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); 150| | // When we need to generate a surrogate pair (leading byte > 0xF0), then 151| | // the corresponding 32-bit value in 'composed' will be greater than 152| | // > (0xff00000>>6) or > 0x3c00000. This can be used later to identify the 153| | // location of the surrogate pairs. 154| 4.93k| const __m128i composed = 155| 4.93k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), 156| 4.93k| _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); 157| 4.93k| const __m128i composedminus = 158| 4.93k| _mm_sub_epi32(composed, _mm_set1_epi32(0x10000)); 159| 4.93k| const __m128i lowtenbits = 160| 4.93k| _mm_and_si128(composedminus, _mm_set1_epi32(0x3ff)); 161| | // Notice the 0x3ff mask: 162| 4.93k| const __m128i hightenbits = 163| 4.93k| _mm_and_si128(_mm_srli_epi32(composedminus, 10), _mm_set1_epi32(0x3ff)); 164| 4.93k| const __m128i lowtenbitsadd = 165| 4.93k| _mm_add_epi32(lowtenbits, _mm_set1_epi32(0xDC00)); 166| 4.93k| const __m128i hightenbitsadd = 167| 4.93k| _mm_add_epi32(hightenbits, _mm_set1_epi32(0xD800)); 168| 4.93k| const __m128i lowtenbitsaddshifted = _mm_slli_epi32(lowtenbitsadd, 16); 169| 4.93k| __m128i surrogates = _mm_or_si128(hightenbitsadd, lowtenbitsaddshifted); 170| 4.93k| uint32_t basic_buffer[4]; 171| 4.93k| uint32_t basic_buffer_swap[4]; 172| 4.93k| if (big_endian) { ------------------ | Branch (172:9): [Folded, False: 4.93k] ------------------ 173| 0| _mm_storeu_si128((__m128i *)basic_buffer_swap, 174| 0| _mm_shuffle_epi8(composed, swap)); 175| 0| surrogates = _mm_shuffle_epi8(surrogates, swap); 176| 0| } 177| 4.93k| _mm_storeu_si128((__m128i *)basic_buffer, composed); 178| 4.93k| uint32_t surrogate_buffer[4]; 179| 4.93k| _mm_storeu_si128((__m128i *)surrogate_buffer, surrogates); 180| 19.7k| for (size_t i = 0; i < 3; i++) { ------------------ | Branch (180:24): [True: 14.8k, False: 4.93k] ------------------ 181| 14.8k| if (basic_buffer[i] > 0x3c00000) { ------------------ | Branch (181:11): [True: 2.97k, False: 11.8k] ------------------ 182| 2.97k| utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xffff); 183| 2.97k| utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); 184| 2.97k| utf16_output += 2; 185| 11.8k| } else { 186| 11.8k| utf16_output[0] = big_endian ? uint16_t(basic_buffer_swap[i]) ------------------ | Branch (186:27): [Folded, False: 11.8k] ------------------ 187| 11.8k| : uint16_t(basic_buffer[i]); 188| 11.8k| utf16_output++; 189| 11.8k| } 190| 14.8k| } 191| 4.93k| } else { 192| | // here we know that there is an error but we do not handle errors 193| 222| } 194| 12.6k| return consumed; 195| 16.5k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128convert_masked_utf8_to_utf16ILNS_10endiannessE1EEEmPKcmRPDs: 10| 16.3k| char16_t *&utf16_output) { 11| | // we use an approach where we try to process up to 12 input bytes. 12| | // Why 12 input bytes and not 16? Because we are concerned with the size of 13| | // the lookup tables. Also 12 is nicely divisible by two and three. 14| | // 15| | // 16| | // Optimization note: our main path below is load-latency dependent. Thus it 17| | // is maybe beneficial to have fast paths that depend on branch prediction but 18| | // have less latency. This results in more instructions but, potentially, also 19| | // higher speeds. 20| | // 21| | // We first try a few fast paths. 22| 16.3k| const __m128i swap = 23| 16.3k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 24| 16.3k| const __m128i in = _mm_loadu_si128((__m128i *)input); 25| 16.3k| const uint16_t input_utf8_end_of_code_point_mask = 26| 16.3k| utf8_end_of_code_point_mask & 0xfff; 27| 16.3k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (27:7): [True: 113, False: 16.2k] ------------------ 28| | // We process the data in chunks of 12 bytes. 29| 113| __m256i ascii = _mm256_cvtepu8_epi16(in); 30| 113| if (big_endian) { ------------------ | Branch (30:9): [True: 113, Folded] ------------------ 31| 113| const __m256i swap256 = _mm256_setr_epi8( 32| 113| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 33| 113| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 34| 113| ascii = _mm256_shuffle_epi8(ascii, swap256); 35| 113| } 36| 113| _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf16_output), ascii); 37| 113| utf16_output += 12; // We wrote 12 16-bit characters. 38| 113| return 12; // We consumed 12 bytes. 39| 113| } 40| 16.2k| if (((utf8_end_of_code_point_mask & 0xffff) == 0xaaaa)) { ------------------ | Branch (40:7): [True: 116, False: 16.1k] ------------------ 41| | // We want to take 8 2-byte UTF-8 code units and turn them into 8 2-byte 42| | // UTF-16 code units. There is probably a more efficient sequence, but the 43| | // following might do. 44| 116| const __m128i sh = 45| 116| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 46| 116| const __m128i perm = _mm_shuffle_epi8(in, sh); 47| 116| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 48| 116| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 49| 116| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 50| 116| if (big_endian) ------------------ | Branch (50:9): [True: 116, Folded] ------------------ 51| 116| composed = _mm_shuffle_epi8(composed, swap); 52| 116| _mm_storeu_si128((__m128i *)utf16_output, composed); 53| 116| utf16_output += 8; // We wrote 16 bytes, 8 code points. 54| 116| return 16; 55| 116| } 56| 16.1k| if (input_utf8_end_of_code_point_mask == 0x924) { ------------------ | Branch (56:7): [True: 3.60k, False: 12.5k] ------------------ 57| | // We want to take 4 3-byte UTF-8 code units and turn them into 4 2-byte 58| | // UTF-16 code units. There is probably a more efficient sequence, but the 59| | // following might do. 60| 3.60k| const __m128i sh = 61| 3.60k| _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); 62| 3.60k| const __m128i perm = _mm_shuffle_epi8(in, sh); 63| 3.60k| const __m128i ascii = 64| 3.60k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 65| 3.60k| const __m128i middlebyte = 66| 3.60k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 67| 3.60k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 68| 3.60k| const __m128i highbyte = 69| 3.60k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 70| 3.60k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 71| 3.60k| const __m128i composed = 72| 3.60k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 73| 3.60k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 74| 3.60k| if (big_endian) ------------------ | Branch (74:9): [True: 3.60k, Folded] ------------------ 75| 3.60k| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 76| 3.60k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 77| 3.60k| utf16_output += 4; 78| 3.60k| return 12; 79| 3.60k| } 80| | 81| 12.5k| const uint8_t idx = simdutf::tables::utf8_to_utf16::utf8bigindex 82| 12.5k| [input_utf8_end_of_code_point_mask][0]; 83| 12.5k| const uint8_t consumed = simdutf::tables::utf8_to_utf16::utf8bigindex 84| 12.5k| [input_utf8_end_of_code_point_mask][1]; 85| 12.5k| if (idx < 64) { ------------------ | Branch (85:7): [True: 5.66k, False: 6.85k] ------------------ 86| | // SIX (6) input code-code units 87| | // this is a relatively easy scenario 88| | // we process SIX (6) input code-code units. The max length in bytes of six 89| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 90| | // processors where pdep/pext is fast, we might be able to use a small 91| | // lookup table. 92| 5.66k| const __m128i sh = _mm_loadu_si128( 93| 5.66k| (const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); 94| 5.66k| const __m128i perm = _mm_shuffle_epi8(in, sh); 95| 5.66k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 96| 5.66k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 97| 5.66k| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 98| 5.66k| if (big_endian) ------------------ | Branch (98:9): [True: 5.66k, Folded] ------------------ 99| 5.66k| composed = _mm_shuffle_epi8(composed, swap); 100| 5.66k| _mm_storeu_si128((__m128i *)utf16_output, composed); 101| 5.66k| utf16_output += 6; // We wrote 12 bytes, 6 code points. There is a potential 102| | // overflow of 4 bytes. 103| 6.85k| } else if (idx < 145) { ------------------ | Branch (103:14): [True: 1.72k, False: 5.13k] ------------------ 104| | // FOUR (4) input code-code units 105| 1.72k| const __m128i sh = _mm_loadu_si128( 106| 1.72k| (const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); 107| 1.72k| const __m128i perm = _mm_shuffle_epi8(in, sh); 108| 1.72k| const __m128i ascii = 109| 1.72k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 110| 1.72k| const __m128i middlebyte = 111| 1.72k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 112| 1.72k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 113| 1.72k| const __m128i highbyte = 114| 1.72k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 115| 1.72k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 116| 1.72k| const __m128i composed = 117| 1.72k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 118| 1.72k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 119| 1.72k| if (big_endian) ------------------ | Branch (119:9): [True: 1.72k, Folded] ------------------ 120| 1.72k| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 121| 1.72k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 122| 1.72k| utf16_output += 4; // Here we overflow by 8 bytes. 123| 5.13k| } else if (idx < 209) { ------------------ | Branch (123:14): [True: 4.91k, False: 222] ------------------ 124| | // TWO (2) input code-code units 125| | ////////////// 126| | // There might be garbage inputs where a leading byte mascarades as a 127| | // four-byte leading byte (by being followed by 3 continuation byte), but is 128| | // not greater than 0xf0. This could trigger a buffer overflow if we only 129| | // counted leading bytes of the form 0xf0 as generating surrogate pairs, 130| | // without further UTF-8 validation. Thus we must be careful to ensure that 131| | // only leading bytes at least as large as 0xf0 generate surrogate pairs. We 132| | // do as at the cost of an extra mask. 133| | ///////////// 134| 4.91k| const __m128i sh = _mm_loadu_si128( 135| 4.91k| (const __m128i *)simdutf::tables::utf8_to_utf16::shufutf8[idx]); 136| 4.91k| const __m128i perm = _mm_shuffle_epi8(in, sh); 137| 4.91k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); 138| 4.91k| const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); 139| 4.91k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 140| 4.91k| __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); 141| | // correct for spurious high bit 142| 4.91k| const __m128i correct = 143| 4.91k| _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); 144| 4.91k| middlehighbyte = _mm_xor_si128(correct, middlehighbyte); 145| 4.91k| const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); 146| | // We deliberately carry the leading four bits in highbyte if they are 147| | // present, we remove them later when computing hightenbits. 148| 4.91k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0xff000000)); 149| 4.91k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); 150| | // When we need to generate a surrogate pair (leading byte > 0xF0), then 151| | // the corresponding 32-bit value in 'composed' will be greater than 152| | // > (0xff00000>>6) or > 0x3c00000. This can be used later to identify the 153| | // location of the surrogate pairs. 154| 4.91k| const __m128i composed = 155| 4.91k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), 156| 4.91k| _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); 157| 4.91k| const __m128i composedminus = 158| 4.91k| _mm_sub_epi32(composed, _mm_set1_epi32(0x10000)); 159| 4.91k| const __m128i lowtenbits = 160| 4.91k| _mm_and_si128(composedminus, _mm_set1_epi32(0x3ff)); 161| | // Notice the 0x3ff mask: 162| 4.91k| const __m128i hightenbits = 163| 4.91k| _mm_and_si128(_mm_srli_epi32(composedminus, 10), _mm_set1_epi32(0x3ff)); 164| 4.91k| const __m128i lowtenbitsadd = 165| 4.91k| _mm_add_epi32(lowtenbits, _mm_set1_epi32(0xDC00)); 166| 4.91k| const __m128i hightenbitsadd = 167| 4.91k| _mm_add_epi32(hightenbits, _mm_set1_epi32(0xD800)); 168| 4.91k| const __m128i lowtenbitsaddshifted = _mm_slli_epi32(lowtenbitsadd, 16); 169| 4.91k| __m128i surrogates = _mm_or_si128(hightenbitsadd, lowtenbitsaddshifted); 170| 4.91k| uint32_t basic_buffer[4]; 171| 4.91k| uint32_t basic_buffer_swap[4]; 172| 4.91k| if (big_endian) { ------------------ | Branch (172:9): [True: 4.91k, Folded] ------------------ 173| 4.91k| _mm_storeu_si128((__m128i *)basic_buffer_swap, 174| 4.91k| _mm_shuffle_epi8(composed, swap)); 175| 4.91k| surrogates = _mm_shuffle_epi8(surrogates, swap); 176| 4.91k| } 177| 4.91k| _mm_storeu_si128((__m128i *)basic_buffer, composed); 178| 4.91k| uint32_t surrogate_buffer[4]; 179| 4.91k| _mm_storeu_si128((__m128i *)surrogate_buffer, surrogates); 180| 19.6k| for (size_t i = 0; i < 3; i++) { ------------------ | Branch (180:24): [True: 14.7k, False: 4.91k] ------------------ 181| 14.7k| if (basic_buffer[i] > 0x3c00000) { ------------------ | Branch (181:11): [True: 2.98k, False: 11.7k] ------------------ 182| 2.98k| utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xffff); 183| 2.98k| utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); 184| 2.98k| utf16_output += 2; 185| 11.7k| } else { 186| 11.7k| utf16_output[0] = big_endian ? uint16_t(basic_buffer_swap[i]) ------------------ | Branch (186:27): [True: 11.7k, Folded] ------------------ 187| 11.7k| : uint16_t(basic_buffer[i]); 188| 11.7k| utf16_output++; 189| 11.7k| } 190| 14.7k| } 191| 4.91k| } else { 192| | // here we know that there is an error but we do not handle errors 193| 222| } 194| 12.5k| return consumed; 195| 16.1k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_128convert_masked_utf8_to_utf32EPKcmRPDi: 9| 11.7k| char32_t *&utf32_output) { 10| | // we use an approach where we try to process up to 12 input bytes. 11| | // Why 12 input bytes and not 16? Because we are concerned with the size of 12| | // the lookup tables. Also 12 is nicely divisible by two and three. 13| | // 14| | // 15| | // Optimization note: our main path below is load-latency dependent. Thus it 16| | // is maybe beneficial to have fast paths that depend on branch prediction but 17| | // have less latency. This results in more instructions but, potentially, also 18| | // higher speeds. 19| | // 20| | // We first try a few fast paths. 21| 11.7k| const __m128i in = _mm_loadu_si128((__m128i *)input); 22| 11.7k| const uint16_t input_utf8_end_of_code_point_mask = 23| 11.7k| utf8_end_of_code_point_mask & 0xfff; 24| 11.7k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (24:7): [True: 111, False: 11.5k] ------------------ 25| | // We process the data in chunks of 12 bytes. 26| 111| _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf32_output), 27| 111| _mm256_cvtepu8_epi32(in)); 28| 111| _mm256_storeu_si256(reinterpret_cast<__m256i *>(utf32_output + 8), 29| 111| _mm256_cvtepu8_epi32(_mm_srli_si128(in, 8))); 30| 111| utf32_output += 12; // We wrote 12 32-bit characters. 31| 111| return 12; // We consumed 12 bytes. 32| 111| } 33| 11.5k| if (((utf8_end_of_code_point_mask & 0xffff) == 0xaaaa)) { ------------------ | Branch (33:7): [True: 110, False: 11.4k] ------------------ 34| | // We want to take 8 2-byte UTF-8 code units and turn them into 8 4-byte 35| | // UTF-32 code units. There is probably a more efficient sequence, but the 36| | // following might do. 37| 110| const __m128i sh = 38| 110| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 39| 110| const __m128i perm = _mm_shuffle_epi8(in, sh); 40| 110| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 41| 110| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 42| 110| const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 43| 110| _mm256_storeu_si256((__m256i *)utf32_output, 44| 110| _mm256_cvtepu16_epi32(composed)); 45| 110| utf32_output += 8; // We wrote 16 bytes, 8 code points. 46| 110| return 16; 47| 110| } 48| 11.4k| if (input_utf8_end_of_code_point_mask == 0x924) { ------------------ | Branch (48:7): [True: 196, False: 11.2k] ------------------ 49| | // We want to take 4 3-byte UTF-8 code units and turn them into 4 4-byte 50| | // UTF-32 code units. There is probably a more efficient sequence, but the 51| | // following might do. 52| 196| const __m128i sh = 53| 196| _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); 54| 196| const __m128i perm = _mm_shuffle_epi8(in, sh); 55| 196| const __m128i ascii = 56| 196| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 57| 196| const __m128i middlebyte = 58| 196| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 59| 196| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 60| 196| const __m128i highbyte = 61| 196| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 62| 196| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 63| 196| const __m128i composed = 64| 196| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 65| 196| _mm_storeu_si128((__m128i *)utf32_output, composed); 66| 196| utf32_output += 4; 67| 196| return 12; 68| 196| } 69| | /// We do not have a fast path available, so we fallback. 70| | 71| 11.2k| const uint8_t idx = 72| 11.2k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; 73| 11.2k| const uint8_t consumed = 74| 11.2k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; 75| 11.2k| if (idx < 64) { ------------------ | Branch (75:7): [True: 5.46k, False: 5.82k] ------------------ 76| | // SIX (6) input code-code units 77| | // this is a relatively easy scenario 78| | // we process SIX (6) input code-code units. The max length in bytes of six 79| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 80| | // processors where pdep/pext is fast, we might be able to use a small 81| | // lookup table. 82| 5.46k| const __m128i sh = 83| 5.46k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 84| 5.46k| const __m128i perm = _mm_shuffle_epi8(in, sh); 85| 5.46k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 86| 5.46k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 87| 5.46k| const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 88| 5.46k| _mm256_storeu_si256((__m256i *)utf32_output, 89| 5.46k| _mm256_cvtepu16_epi32(composed)); 90| 5.46k| utf32_output += 6; // We wrote 24 bytes, 6 code points. There is a potential 91| | // overflow of 32 - 24 = 8 bytes. 92| 5.82k| } else if (idx < 145) { ------------------ | Branch (92:14): [True: 1.15k, False: 4.66k] ------------------ 93| | // FOUR (4) input code-code units 94| 1.15k| const __m128i sh = 95| 1.15k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 96| 1.15k| const __m128i perm = _mm_shuffle_epi8(in, sh); 97| 1.15k| const __m128i ascii = 98| 1.15k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 99| 1.15k| const __m128i middlebyte = 100| 1.15k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 101| 1.15k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 102| 1.15k| const __m128i highbyte = 103| 1.15k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 104| 1.15k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 105| 1.15k| const __m128i composed = 106| 1.15k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 107| 1.15k| _mm_storeu_si128((__m128i *)utf32_output, composed); 108| 1.15k| utf32_output += 4; 109| 4.66k| } else if (idx < 209) { ------------------ | Branch (109:14): [True: 4.44k, False: 222] ------------------ 110| | // TWO (2) input code-code units 111| 4.44k| const __m128i sh = 112| 4.44k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 113| 4.44k| const __m128i perm = _mm_shuffle_epi8(in, sh); 114| 4.44k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); 115| 4.44k| const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); 116| 4.44k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 117| 4.44k| __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); 118| | // correct for spurious high bit 119| 4.44k| const __m128i correct = 120| 4.44k| _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); 121| 4.44k| middlehighbyte = _mm_xor_si128(correct, middlehighbyte); 122| 4.44k| const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); 123| 4.44k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0x07000000)); 124| 4.44k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); 125| 4.44k| const __m128i composed = 126| 4.44k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), 127| 4.44k| _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); 128| 4.44k| _mm_storeu_si128((__m128i *)utf32_output, composed); 129| 4.44k| utf32_output += 130| 4.44k| 3; // We wrote 3 * 4 bytes, there is a potential overflow of 4 bytes. 131| 4.44k| } else { 132| | // here we know that there is an error but we do not handle errors 133| 222| } 134| 11.2k| return consumed; 135| 11.4k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf1623utf16_gather_high_bytesILNS_10endiannessE0EEENS1_4simd5simd8IhEERKNS5_6simd16ItEESB_: 3| 20.1k| const simd16 &in1) { 4| 20.1k| if (big_endian) { ------------------ | Branch (4:7): [Folded, False: 20.1k] ------------------ 5| | // we want lower bytes 6| 0| const auto mask = simd16(0x00ff); 7| 0| const auto t0 = in0 & mask; 8| 0| const auto t1 = in1 & mask; 9| | 10| 0| return simd16::pack(t0, t1); 11| 20.1k| } else { 12| 20.1k| const auto t0 = in0.shr<8>(); 13| 20.1k| const auto t1 = in1.shr<8>(); 14| | 15| 20.1k| return simd16::pack(t0, t1); 16| 20.1k| } 17| 20.1k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_15utf1623utf16_gather_high_bytesILNS_10endiannessE1EEENS1_4simd5simd8IhEERKNS5_6simd16ItEESB_: 3| 20.0k| const simd16 &in1) { 4| 20.0k| if (big_endian) { ------------------ | Branch (4:7): [True: 20.0k, Folded] ------------------ 5| | // we want lower bytes 6| 20.0k| const auto mask = simd16(0x00ff); 7| 20.0k| const auto t0 = in0 & mask; 8| 20.0k| const auto t1 = in1 & mask; 9| | 10| 20.0k| return simd16::pack(t0, t1); 11| 20.0k| } else { 12| 0| const auto t0 = in0.shr<8>(); 13| 0| const auto t1 = in1.shr<8>(); 14| | 15| 0| return simd16::pack(t0, t1); 16| 0| } 17| 20.0k|} _ZNK7simdutf7haswell14implementation13validate_utf8EPKcm: 266| 27.2k|implementation::validate_utf8(const char *buf, size_t len) const noexcept { 267| 27.2k| return haswell::utf8_validation::generic_validate_utf8(buf, len); 268| 27.2k|} _ZNK7simdutf7haswell14implementation25validate_utf8_with_errorsEPKcm: 273| 6.85k| const char *buf, size_t len) const noexcept { 274| 6.85k| return haswell::utf8_validation::generic_validate_utf8_with_errors(buf, len); 275| 6.85k|} _ZNK7simdutf7haswell14implementation16validate_utf16leEPKDsm: 312| 19.7k| size_t len) const noexcept { 313| 19.7k| if (simdutf_unlikely(len == 0)) { ------------------ | | 93| 19.7k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 16.7k, False: 2.93k] | | ------------------ ------------------ 314| | // empty input is valid UTF-16. protect the implementation from 315| | // handling nullptr 316| 16.7k| return true; 317| 16.7k| } 318| 2.93k| const auto res = 319| 2.93k| haswell::utf16::validate_utf16_with_errors(buf, len); 320| 2.93k| if (res.is_err()) { ------------------ | Branch (320:7): [True: 48, False: 2.88k] ------------------ 321| 48| return false; 322| 48| } 323| | 324| 2.88k| if (res.count == len) { ------------------ | Branch (324:7): [True: 0, False: 2.88k] ------------------ 325| 0| return true; 326| 0| } 327| | 328| 2.88k| return scalar::utf16::validate(buf + res.count, 329| 2.88k| len - res.count); 330| 2.88k|} _ZNK7simdutf7haswell14implementation16validate_utf16beEPKDsm: 336| 19.7k| size_t len) const noexcept { 337| 19.7k| if (simdutf_unlikely(len == 0)) { ------------------ | | 93| 19.7k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 16.8k, False: 2.85k] | | ------------------ ------------------ 338| | // empty input is valid UTF-16. protect the implementation from 339| | // handling nullptr 340| 16.8k| return true; 341| 16.8k| } 342| 2.85k| const auto res = 343| 2.85k| haswell::utf16::validate_utf16_with_errors(buf, len); 344| 2.85k| if (res.is_err()) { ------------------ | Branch (344:7): [True: 35, False: 2.81k] ------------------ 345| 35| return false; 346| 35| } 347| | 348| 2.81k| if (res.count == len) { ------------------ | Branch (348:7): [True: 0, False: 2.81k] ------------------ 349| 0| return true; 350| 0| } 351| | 352| 2.81k| return scalar::utf16::validate(buf + res.count, 353| 2.81k| len - res.count); 354| 2.81k|} _ZNK7simdutf7haswell14implementation28validate_utf16le_with_errorsEPKDsm: 357| 6.85k| const char16_t *buf, size_t len) const noexcept { 358| | 359| 6.85k| const result res = 360| 6.85k| haswell::utf16::validate_utf16_with_errors(buf, len); 361| 6.85k| if (res.count != len) { ------------------ | Branch (361:7): [True: 443, False: 6.41k] ------------------ 362| 443| const result scalar_res = 363| 443| scalar::utf16::validate_with_errors( 364| 443| buf + res.count, len - res.count); 365| 443| return result(scalar_res.error, res.count + scalar_res.count); 366| 6.41k| } else { 367| 6.41k| return res; 368| 6.41k| } 369| 6.85k|} _ZNK7simdutf7haswell14implementation28validate_utf16be_with_errorsEPKDsm: 372| 6.85k| const char16_t *buf, size_t len) const noexcept { 373| 6.85k| const result res = 374| 6.85k| haswell::utf16::validate_utf16_with_errors(buf, len); 375| 6.85k| if (res.count != len) { ------------------ | Branch (375:7): [True: 361, False: 6.49k] ------------------ 376| 361| const result scalar_res = 377| 361| scalar::utf16::validate_with_errors(buf + res.count, 378| 361| len - res.count); 379| 361| return result(scalar_res.error, res.count + scalar_res.count); 380| 6.49k| } else { 381| 6.49k| return res; 382| 6.49k| } 383| 6.85k|} _ZNK7simdutf7haswell14implementation14validate_utf32EPKDim: 398| 12.8k|implementation::validate_utf32(const char32_t *buf, size_t len) const noexcept { 399| 12.8k| return utf32::validate(buf, len); 400| 12.8k|} _ZNK7simdutf7haswell14implementation22convert_latin1_to_utf8EPKcmPc: 412| 7.13k| const char *buf, size_t len, char *utf8_output) const noexcept { 413| 7.13k| std::pair ret = 414| 7.13k| avx2_convert_latin1_to_utf8(buf, len, utf8_output); 415| 7.13k| size_t converted_chars = ret.second - utf8_output; 416| | 417| 7.13k| if (ret.first != buf + len) { ------------------ | Branch (417:7): [True: 1.92k, False: 5.21k] ------------------ 418| 1.92k| const size_t scalar_converted_chars = scalar::latin1_to_utf8::convert( 419| 1.92k| ret.first, len - (ret.first - buf), ret.second); 420| 1.92k| converted_chars += scalar_converted_chars; 421| 1.92k| } 422| | 423| 7.13k| return converted_chars; 424| 7.13k|} _ZNK7simdutf7haswell14implementation25convert_latin1_to_utf16leEPKcmPDs: 429| 6.85k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 430| 6.85k| std::pair ret = 431| 6.85k| avx2_convert_latin1_to_utf16(buf, len, utf16_output); 432| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (432:7): [True: 0, False: 6.85k] ------------------ 433| 0| return 0; 434| 0| } 435| 6.85k| size_t converted_chars = ret.second - utf16_output; 436| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (436:7): [True: 1.01k, False: 5.84k] ------------------ 437| 1.01k| const size_t scalar_converted_chars = 438| 1.01k| scalar::latin1_to_utf16::convert( 439| 1.01k| ret.first, len - (ret.first - buf), ret.second); 440| 1.01k| if (scalar_converted_chars == 0) { ------------------ | Branch (440:9): [True: 0, False: 1.01k] ------------------ 441| 0| return 0; 442| 0| } 443| 1.01k| converted_chars += scalar_converted_chars; 444| 1.01k| } 445| 6.85k| return converted_chars; 446| 6.85k|} _ZNK7simdutf7haswell14implementation25convert_latin1_to_utf16beEPKcmPDs: 449| 6.85k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 450| 6.85k| std::pair ret = 451| 6.85k| avx2_convert_latin1_to_utf16(buf, len, utf16_output); 452| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (452:7): [True: 0, False: 6.85k] ------------------ 453| 0| return 0; 454| 0| } 455| 6.85k| size_t converted_chars = ret.second - utf16_output; 456| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (456:7): [True: 1.01k, False: 5.84k] ------------------ 457| 1.01k| const size_t scalar_converted_chars = 458| 1.01k| scalar::latin1_to_utf16::convert( 459| 1.01k| ret.first, len - (ret.first - buf), ret.second); 460| 1.01k| if (scalar_converted_chars == 0) { ------------------ | Branch (460:9): [True: 0, False: 1.01k] ------------------ 461| 0| return 0; 462| 0| } 463| 1.01k| converted_chars += scalar_converted_chars; 464| 1.01k| } 465| 6.85k| return converted_chars; 466| 6.85k|} _ZNK7simdutf7haswell14implementation23convert_latin1_to_utf32EPKcmPDi: 471| 6.85k| const char *buf, size_t len, char32_t *utf32_output) const noexcept { 472| 6.85k| std::pair ret = 473| 6.85k| avx2_convert_latin1_to_utf32(buf, len, utf32_output); 474| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (474:7): [True: 0, False: 6.85k] ------------------ 475| 0| return 0; 476| 0| } 477| 6.85k| size_t converted_chars = ret.second - utf32_output; 478| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (478:7): [True: 960, False: 5.89k] ------------------ 479| 960| const size_t scalar_converted_chars = scalar::latin1_to_utf32::convert( 480| 960| ret.first, len - (ret.first - buf), ret.second); 481| 960| if (scalar_converted_chars == 0) { ------------------ | Branch (481:9): [True: 0, False: 960] ------------------ 482| 0| return 0; 483| 0| } 484| 960| converted_chars += scalar_converted_chars; 485| 960| } 486| 6.85k| return converted_chars; 487| 6.85k|} _ZNK7simdutf7haswell14implementation22convert_utf8_to_latin1EPKcmPc: 492| 13.7k| const char *buf, size_t len, char *latin1_output) const noexcept { 493| 13.7k| utf8_to_latin1::validating_transcoder converter; 494| 13.7k| return converter.convert(buf, len, latin1_output); 495| 13.7k|} _ZNK7simdutf7haswell14implementation23convert_utf8_to_utf16leEPKcmPDs: 511| 13.6k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 512| 13.6k| utf8_to_utf16::validating_transcoder converter; 513| 13.6k| return converter.convert(buf, len, utf16_output); 514| 13.6k|} _ZNK7simdutf7haswell14implementation23convert_utf8_to_utf16beEPKcmPDs: 517| 13.6k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 518| 13.6k| utf8_to_utf16::validating_transcoder converter; 519| 13.6k| return converter.convert(buf, len, utf16_output); 520| 13.6k|} _ZNK7simdutf7haswell14implementation21convert_utf8_to_utf32EPKcmPDi: 550| 6.85k| const char *buf, size_t len, char32_t *utf32_output) const noexcept { 551| 6.85k| utf8_to_utf32::validating_transcoder converter; 552| 6.85k| return converter.convert(buf, len, utf32_output); 553| 6.85k|} _ZNK7simdutf7haswell14implementation25convert_utf16le_to_latin1EPKDsmPc: 569| 6.85k| const char16_t *buf, size_t len, char *latin1_output) const noexcept { 570| 6.85k| std::pair ret = 571| 6.85k| haswell::avx2_convert_utf16_to_latin1(buf, len, 572| 6.85k| latin1_output); 573| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (573:7): [True: 0, False: 6.85k] ------------------ 574| 0| return 0; 575| 0| } 576| 6.85k| size_t saved_bytes = ret.second - latin1_output; 577| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (577:7): [True: 1.03k, False: 5.81k] ------------------ 578| 1.03k| const size_t scalar_saved_bytes = 579| 1.03k| scalar::utf16_to_latin1::convert( 580| 1.03k| ret.first, len - (ret.first - buf), ret.second); 581| 1.03k| if (scalar_saved_bytes == 0) { ------------------ | Branch (581:9): [True: 0, False: 1.03k] ------------------ 582| 0| return 0; 583| 0| } 584| 1.03k| saved_bytes += scalar_saved_bytes; 585| 1.03k| } 586| 6.85k| return saved_bytes; 587| 6.85k|} _ZNK7simdutf7haswell14implementation25convert_utf16be_to_latin1EPKDsmPc: 590| 6.85k| const char16_t *buf, size_t len, char *latin1_output) const noexcept { 591| 6.85k| std::pair ret = 592| 6.85k| haswell::avx2_convert_utf16_to_latin1(buf, len, 593| 6.85k| latin1_output); 594| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (594:7): [True: 0, False: 6.85k] ------------------ 595| 0| return 0; 596| 0| } 597| 6.85k| size_t saved_bytes = ret.second - latin1_output; 598| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (598:7): [True: 1.03k, False: 5.81k] ------------------ 599| 1.03k| const size_t scalar_saved_bytes = 600| 1.03k| scalar::utf16_to_latin1::convert( 601| 1.03k| ret.first, len - (ret.first - buf), ret.second); 602| 1.03k| if (scalar_saved_bytes == 0) { ------------------ | Branch (602:9): [True: 0, False: 1.03k] ------------------ 603| 0| return 0; 604| 0| } 605| 1.03k| saved_bytes += scalar_saved_bytes; 606| 1.03k| } 607| 6.85k| return saved_bytes; 608| 6.85k|} _ZNK7simdutf7haswell14implementation23convert_utf16le_to_utf8EPKDsmPc: 679| 12.8k| const char16_t *buf, size_t len, char *utf8_output) const noexcept { 680| 12.8k| std::pair ret = 681| 12.8k| haswell::avx2_convert_utf16_to_utf8(buf, len, 682| 12.8k| utf8_output); 683| 12.8k| if (ret.first == nullptr) { ------------------ | Branch (683:7): [True: 59, False: 12.8k] ------------------ 684| 59| return 0; 685| 59| } 686| 12.8k| size_t saved_bytes = ret.second - utf8_output; 687| 12.8k| if (ret.first != buf + len) { ------------------ | Branch (687:7): [True: 1.23k, False: 11.5k] ------------------ 688| 1.23k| const size_t scalar_saved_bytes = 689| 1.23k| scalar::utf16_to_utf8::convert( 690| 1.23k| ret.first, len - (ret.first - buf), ret.second); 691| 1.23k| if (scalar_saved_bytes == 0) { ------------------ | Branch (691:9): [True: 40, False: 1.19k] ------------------ 692| 40| return 0; 693| 40| } 694| 1.19k| saved_bytes += scalar_saved_bytes; 695| 1.19k| } 696| 12.7k| return saved_bytes; 697| 12.8k|} _ZNK7simdutf7haswell14implementation23convert_utf16be_to_utf8EPKDsmPc: 700| 12.8k| const char16_t *buf, size_t len, char *utf8_output) const noexcept { 701| 12.8k| std::pair ret = 702| 12.8k| haswell::avx2_convert_utf16_to_utf8(buf, len, 703| 12.8k| utf8_output); 704| 12.8k| if (ret.first == nullptr) { ------------------ | Branch (704:7): [True: 44, False: 12.8k] ------------------ 705| 44| return 0; 706| 44| } 707| 12.8k| size_t saved_bytes = ret.second - utf8_output; 708| 12.8k| if (ret.first != buf + len) { ------------------ | Branch (708:7): [True: 1.16k, False: 11.6k] ------------------ 709| 1.16k| const size_t scalar_saved_bytes = 710| 1.16k| scalar::utf16_to_utf8::convert( 711| 1.16k| ret.first, len - (ret.first - buf), ret.second); 712| 1.16k| if (scalar_saved_bytes == 0) { ------------------ | Branch (712:9): [True: 34, False: 1.13k] ------------------ 713| 34| return 0; 714| 34| } 715| 1.13k| saved_bytes += scalar_saved_bytes; 716| 1.13k| } 717| 12.7k| return saved_bytes; 718| 12.8k|} _ZNK7simdutf7haswell14implementation21convert_utf32_to_utf8EPKDimPc: 766| 6.02k| const char32_t *buf, size_t len, char *utf8_output) const noexcept { 767| 6.02k| std::pair ret = 768| 6.02k| avx2_convert_utf32_to_utf8(buf, len, utf8_output); 769| 6.02k| if (ret.first == nullptr) { ------------------ | Branch (769:7): [True: 0, False: 6.02k] ------------------ 770| 0| return 0; 771| 0| } 772| 6.02k| size_t saved_bytes = ret.second - utf8_output; 773| 6.02k| if (ret.first != buf + len) { ------------------ | Branch (773:7): [True: 849, False: 5.17k] ------------------ 774| 849| SIMDUTF_VZEROUPPER(); ------------------ | | 22| 849|#define SIMDUTF_VZEROUPPER() _mm256_zeroupper() ------------------ 775| 849| const size_t scalar_saved_bytes = scalar::utf32_to_utf8::convert( 776| 849| ret.first, len - (ret.first - buf), ret.second); 777| 849| if (scalar_saved_bytes == 0) { ------------------ | Branch (777:9): [True: 0, False: 849] ------------------ 778| 0| return 0; 779| 0| } 780| 849| saved_bytes += scalar_saved_bytes; 781| 849| } 782| 6.02k| return saved_bytes; 783| 6.02k|} _ZNK7simdutf7haswell14implementation23convert_utf32_to_latin1EPKDimPc: 788| 6.85k| const char32_t *buf, size_t len, char *latin1_output) const noexcept { 789| 6.85k| std::pair ret = 790| 6.85k| avx2_convert_utf32_to_latin1(buf, len, latin1_output); 791| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (791:7): [True: 0, False: 6.85k] ------------------ 792| 0| return 0; 793| 0| } 794| 6.85k| size_t saved_bytes = ret.second - latin1_output; 795| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (795:7): [True: 1.03k, False: 5.81k] ------------------ 796| 1.03k| const size_t scalar_saved_bytes = scalar::utf32_to_latin1::convert( 797| 1.03k| ret.first, len - (ret.first - buf), ret.second); 798| 1.03k| if (scalar_saved_bytes == 0) { ------------------ | Branch (798:9): [True: 0, False: 1.03k] ------------------ 799| 0| return 0; 800| 0| } 801| 1.03k| saved_bytes += scalar_saved_bytes; 802| 1.03k| } 803| 6.85k| return saved_bytes; 804| 6.85k|} _ZNK7simdutf7haswell14implementation10count_utf8EPKcm: 1095| 13.7k|implementation::count_utf8(const char *in, size_t size) const noexcept { 1096| 13.7k| return utf8::count_code_points_bytemask(in, size); 1097| 13.7k|} _ZNK7simdutf7haswell14implementation23latin1_length_from_utf8EPKcm: 1107| 13.7k| const char *buf, size_t len) const noexcept { 1108| 13.7k| return count_utf8(buf, len); 1109| 13.7k|} _ZNK7simdutf7haswell14implementation24utf8_length_from_utf16leEPKDsm: 1114| 12.8k| const char16_t *input, size_t length) const noexcept { 1115| 12.8k| return utf16::utf8_length_from_utf16_bytemask(input, 1116| 12.8k| length); 1117| 12.8k|} _ZNK7simdutf7haswell14implementation24utf8_length_from_utf16beEPKDsm: 1120| 12.8k| const char16_t *input, size_t length) const noexcept { 1121| 12.8k| return utf16::utf8_length_from_utf16_bytemask(input, length); 1122| 12.8k|} _ZNK7simdutf7haswell14implementation22utf16_length_from_utf8EPKcm: 1139| 27.2k| const char *input, size_t length) const noexcept { 1140| 27.2k| return utf8::utf16_length_from_utf8_bytemask(input, length); 1141| 27.2k|} _ZNK7simdutf7haswell14implementation23utf8_length_from_latin1EPKcm: 1180| 7.13k| const char *input, size_t len) const noexcept { 1181| 7.13k| const uint8_t *data = reinterpret_cast(input); 1182| 7.13k| size_t answer = len / sizeof(__m256i) * sizeof(__m256i); 1183| 7.13k| size_t i = 0; 1184| 7.13k| if (answer >= 2048) { // long strings optimization ------------------ | Branch (1184:7): [True: 0, False: 7.13k] ------------------ 1185| 0| __m256i four_64bits = _mm256_setzero_si256(); 1186| 0| while (i + sizeof(__m256i) <= len) { ------------------ | Branch (1186:12): [True: 0, False: 0] ------------------ 1187| 0| __m256i runner = _mm256_setzero_si256(); 1188| | // We can do up to 255 loops without overflow. 1189| 0| size_t iterations = (len - i) / sizeof(__m256i); 1190| 0| if (iterations > 255) { ------------------ | Branch (1190:11): [True: 0, False: 0] ------------------ 1191| 0| iterations = 255; 1192| 0| } 1193| 0| size_t max_i = i + iterations * sizeof(__m256i) - sizeof(__m256i); 1194| 0| for (; i + 4 * sizeof(__m256i) <= max_i; i += 4 * sizeof(__m256i)) { ------------------ | Branch (1194:14): [True: 0, False: 0] ------------------ 1195| 0| __m256i input1 = _mm256_loadu_si256((const __m256i *)(data + i)); 1196| 0| __m256i input2 = 1197| 0| _mm256_loadu_si256((const __m256i *)(data + i + sizeof(__m256i))); 1198| 0| __m256i input3 = _mm256_loadu_si256( 1199| 0| (const __m256i *)(data + i + 2 * sizeof(__m256i))); 1200| 0| __m256i input4 = _mm256_loadu_si256( 1201| 0| (const __m256i *)(data + i + 3 * sizeof(__m256i))); 1202| 0| __m256i input12 = 1203| 0| _mm256_add_epi8(_mm256_cmpgt_epi8(_mm256_setzero_si256(), input1), 1204| 0| _mm256_cmpgt_epi8(_mm256_setzero_si256(), input2)); 1205| 0| __m256i input23 = 1206| 0| _mm256_add_epi8(_mm256_cmpgt_epi8(_mm256_setzero_si256(), input3), 1207| 0| _mm256_cmpgt_epi8(_mm256_setzero_si256(), input4)); 1208| 0| __m256i input1234 = _mm256_add_epi8(input12, input23); 1209| 0| runner = _mm256_sub_epi8(runner, input1234); 1210| 0| } 1211| 0| for (; i <= max_i; i += sizeof(__m256i)) { ------------------ | Branch (1211:14): [True: 0, False: 0] ------------------ 1212| 0| __m256i input_256_chunk = 1213| 0| _mm256_loadu_si256((const __m256i *)(data + i)); 1214| 0| runner = _mm256_sub_epi8( 1215| 0| runner, _mm256_cmpgt_epi8(_mm256_setzero_si256(), input_256_chunk)); 1216| 0| } 1217| 0| four_64bits = _mm256_add_epi64( 1218| 0| four_64bits, _mm256_sad_epu8(runner, _mm256_setzero_si256())); 1219| 0| } 1220| 0| answer += _mm256_extract_epi64(four_64bits, 0) + 1221| 0| _mm256_extract_epi64(four_64bits, 1) + 1222| 0| _mm256_extract_epi64(four_64bits, 2) + 1223| 0| _mm256_extract_epi64(four_64bits, 3); 1224| 7.13k| } else if (answer > 0) { ------------------ | Branch (1224:14): [True: 1.36k, False: 5.77k] ------------------ 1225| 12.0k| for (; i + sizeof(__m256i) <= len; i += sizeof(__m256i)) { ------------------ | Branch (1225:12): [True: 10.6k, False: 1.36k] ------------------ 1226| 10.6k| __m256i latin = _mm256_loadu_si256((const __m256i *)(data + i)); 1227| 10.6k| uint32_t non_ascii = _mm256_movemask_epi8(latin); 1228| 10.6k| answer += count_ones(non_ascii); 1229| 10.6k| } 1230| 1.36k| } 1231| 7.13k| return answer + scalar::latin1::utf8_length_from_latin1( 1232| 7.13k| reinterpret_cast(data + i), len - i); 1233| 7.13k|} _ZNK7simdutf7haswell14implementation22utf8_length_from_utf32EPKDim: 1238| 6.02k| const char32_t *input, size_t length) const noexcept { 1239| 6.02k| return utf32::utf8_length_from_utf32(input, length); 1240| 6.02k|} _ZNK7simdutf7haswell14implementation22utf32_length_from_utf8EPKcm: 1266| 6.85k| const char *input, size_t length) const noexcept { 1267| 6.85k| return utf8::count_code_points(input, length); 1268| 6.85k|} _ZNK7simdutf7haswell14implementation16base64_to_binaryEPKcmPcNS_14base64_optionsENS_27last_chunk_handling_optionsE: 1274| 19.2k| last_chunk_handling_options last_chunk_options) const noexcept { 1275| 19.2k| if (options & base64_default_or_url) { ------------------ | Branch (1275:7): [True: 0, False: 19.2k] ------------------ 1276| 0| if (options == base64_options::base64_default_or_url_accept_garbage) { ------------------ | Branch (1276:9): [True: 0, False: 0] ------------------ 1277| 0| return base64::compress_decode_base64( 1278| 0| output, input, length, options, last_chunk_options); 1279| 0| } else { 1280| 0| return base64::compress_decode_base64( 1281| 0| output, input, length, options, last_chunk_options); 1282| 0| } 1283| 19.2k| } else if (options & base64_url) { ------------------ | Branch (1283:14): [True: 0, False: 19.2k] ------------------ 1284| 0| if (options == base64_options::base64_url_accept_garbage) { ------------------ | Branch (1284:9): [True: 0, False: 0] ------------------ 1285| 0| return base64::compress_decode_base64( 1286| 0| output, input, length, options, last_chunk_options); 1287| 0| } else { 1288| 0| return base64::compress_decode_base64( 1289| 0| output, input, length, options, last_chunk_options); 1290| 0| } 1291| 19.2k| } else { 1292| 19.2k| if (options == base64_options::base64_default_accept_garbage) { ------------------ | Branch (1292:9): [True: 0, False: 19.2k] ------------------ 1293| 0| return base64::compress_decode_base64( 1294| 0| output, input, length, options, last_chunk_options); 1295| 19.2k| } else { 1296| 19.2k| return base64::compress_decode_base64( 1297| 19.2k| output, input, length, options, last_chunk_options); 1298| 19.2k| } 1299| 19.2k| } 1300| 19.2k|} _ZNK7simdutf7haswell14implementation24base64_to_binary_detailsEPKcmPcNS_14base64_optionsENS_27last_chunk_handling_optionsE: 1304| 59.2k| last_chunk_handling_options last_chunk_options) const noexcept { 1305| 59.2k| if (options & base64_default_or_url) { ------------------ | Branch (1305:7): [True: 0, False: 59.2k] ------------------ 1306| 0| if (options == base64_options::base64_default_or_url_accept_garbage) { ------------------ | Branch (1306:9): [True: 0, False: 0] ------------------ 1307| 0| return base64::compress_decode_base64( 1308| 0| output, input, length, options, last_chunk_options); 1309| 0| } else { 1310| 0| return base64::compress_decode_base64( 1311| 0| output, input, length, options, last_chunk_options); 1312| 0| } 1313| 59.2k| } else if (options & base64_url) { ------------------ | Branch (1313:14): [True: 0, False: 59.2k] ------------------ 1314| 0| if (options == base64_options::base64_url_accept_garbage) { ------------------ | Branch (1314:9): [True: 0, False: 0] ------------------ 1315| 0| return base64::compress_decode_base64( 1316| 0| output, input, length, options, last_chunk_options); 1317| 0| } else { 1318| 0| return base64::compress_decode_base64( 1319| 0| output, input, length, options, last_chunk_options); 1320| 0| } 1321| 59.2k| } else { 1322| 59.2k| if (options == base64_options::base64_default_accept_garbage) { ------------------ | Branch (1322:9): [True: 0, False: 59.2k] ------------------ 1323| 0| return base64::compress_decode_base64( 1324| 0| output, input, length, options, last_chunk_options); 1325| 59.2k| } else { 1326| 59.2k| return base64::compress_decode_base64( 1327| 59.2k| output, input, length, options, last_chunk_options); 1328| 59.2k| } 1329| 59.2k| } 1330| 59.2k|} _ZNK7simdutf7haswell14implementation16binary_to_base64EPKcmPcNS_14base64_optionsE: 1394| 18.0k| base64_options options) const noexcept { 1395| 18.0k| if (options & base64_url) { ------------------ | Branch (1395:7): [True: 0, False: 18.0k] ------------------ 1396| 0| return encode_base64(output, input, length, options); 1397| 18.0k| } else { 1398| 18.0k| return encode_base64(output, input, length, options); 1399| 18.0k| } 1400| 18.0k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_18is_asciiERKNS1_4simd8simd8x64IhEE: 13| 15.7k|simdutf_really_inline bool is_ascii(const simd8x64 &input) { 14| 15.7k| return input.reduce_or().is_ascii(); 15| 15.7k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_124must_be_2_3_continuationENS1_4simd5simd8IhEES4_: 22| 34.5k| const simd8 prev3) { 23| 34.5k| simd8 is_third_byte = 24| 34.5k| prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be > 0x80 25| 34.5k| simd8 is_fourth_byte = 26| 34.5k| prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be > 0x80 27| 34.5k| return simd8(is_third_byte | is_fourth_byte); 28| 34.5k|} _ZNK7simdutf14implementation27supported_by_runtime_systemEv: 64| 27.4k|bool implementation::supported_by_runtime_system() const { 65| 27.4k| uint32_t required_instruction_sets = this->required_instruction_sets(); 66| 27.4k| uint32_t supported_instruction_sets = 67| 27.4k| internal::detect_supported_architectures(); 68| 27.4k| return ((supported_instruction_sets & required_instruction_sets) == 69| 27.4k| required_instruction_sets); 70| 27.4k|} _ZNK7simdutf14implementation33maximal_binary_length_from_base64EPKcm: 117| 98.0k| const char *input, size_t length) const noexcept { 118| 98.0k| return scalar::base64::maximal_binary_length_from_base64(input, length); 119| 98.0k|} _ZNK7simdutf14implementation25base64_length_from_binaryEmNS_14base64_optionsE: 137| 56.9k| size_t length, base64_options options) const noexcept { 138| 56.9k| return scalar::base64::base64_length_from_binary(length, options); 139| 56.9k|} _ZNK7simdutf8internal29available_implementation_list5beginEv: 1513| 6.85k|available_implementation_list::begin() const noexcept { 1514| 6.85k| return internal::get_available_implementation_pointers().begin(); 1515| 6.85k|} _ZNK7simdutf8internal29available_implementation_list3endEv: 1517| 6.85k|available_implementation_list::end() const noexcept { 1518| 6.85k| return internal::get_available_implementation_pointers().end(); 1519| 6.85k|} _ZNK7simdutf8internal29available_implementation_list21detect_best_supportedEv: 1521| 1|available_implementation_list::detect_best_supported() const noexcept { 1522| | // They are prelisted in priority order, so we just go down the list 1523| 1| uint32_t supported_instruction_sets = 1524| 1| internal::detect_supported_architectures(); 1525| 1| for (const implementation *impl : ------------------ | Branch (1525:35): [True: 2, False: 0] ------------------ 1526| 2| internal::get_available_implementation_pointers()) { 1527| 2| uint32_t required_instruction_sets = impl->required_instruction_sets(); 1528| 2| if ((supported_instruction_sets & required_instruction_sets) == ------------------ | Branch (1528:9): [True: 1, False: 1] ------------------ 1529| 2| required_instruction_sets) { 1530| 1| return impl; 1531| 1| } 1532| 2| } 1533| 0| return get_unsupported_singleton(); // this should never happen? 1534| 1|} _ZNK7simdutf8internal49detect_best_supported_implementation_on_first_use8set_bestEv: 1537| 1|detect_best_supported_implementation_on_first_use::set_best() const noexcept { 1538| 1| SIMDUTF_PUSH_DISABLE_WARNINGS 1539| | SIMDUTF_DISABLE_DEPRECATED_WARNING // Disable CRT_SECURE warning on MSVC: 1540| | // manually verified this is safe 1541| 1| char *force_implementation_name = getenv("SIMDUTF_FORCE_IMPLEMENTATION"); 1542| 1| SIMDUTF_POP_DISABLE_WARNINGS 1543| | 1544| 1| if (force_implementation_name) { ------------------ | Branch (1544:7): [True: 0, False: 1] ------------------ 1545| 0| auto force_implementation = 1546| 0| get_available_implementations()[force_implementation_name]; 1547| 0| if (force_implementation) { ------------------ | Branch (1547:9): [True: 0, False: 0] ------------------ 1548| 0| return get_active_implementation() = force_implementation; 1549| 0| } else { 1550| | // Note: abort() and stderr usage within the library is forbidden. 1551| 0| return get_active_implementation() = get_unsupported_singleton(); 1552| 0| } 1553| 0| } 1554| 1| return get_active_implementation() = 1555| 1| get_available_implementations().detect_best_supported(); 1556| 1|} _ZN7simdutf29get_available_implementationsEv: 1568| 6.85k|get_available_implementations() { 1569| 6.85k|#if !SIMDUTF_USE_STATIC_INITIALIZATION 1570| 6.85k| static const internal::available_implementation_list 1571| 6.85k| available_implementations_instance{}; 1572| 6.85k|#endif 1573| 6.85k| return available_implementations_instance; 1574| 6.85k|} _ZN7simdutf25get_active_implementationEv: 1596| 59.3k|get_active_implementation() { 1597| 59.3k|#if !SIMDUTF_USE_STATIC_INITIALIZATION 1598| 59.3k| #if !SIMDUTF_SINGLE_IMPLEMENTATION 1599| 59.3k| static const internal::detect_best_supported_implementation_on_first_use 1600| 59.3k| detect_best_supported_implementation_on_first_use_singleton; 1601| 59.3k| #endif 1602| 59.3k| static internal::atomic_ptr 1603| 59.3k| active_implementation_instance{ 1604| | #if SIMDUTF_SINGLE_IMPLEMENTATION 1605| | internal::get_single_implementation() 1606| | #else 1607| 59.3k| &detect_best_supported_implementation_on_first_use_singleton 1608| 59.3k| #endif 1609| 59.3k| }; 1610| 59.3k|#endif 1611| 59.3k| return active_implementation_instance; 1612| 59.3k|} _ZN7simdutf21base64_to_binary_safeEPKcmPcRmNS_14base64_optionsENS_27last_chunk_handling_optionsEb: 2755| 59.2k| bool decode_up_to_bad_char) noexcept { 2756| 59.2k| return base64_to_binary_safe_impl(input, length, output, outlen, 2757| 59.2k| options, last_chunk_handling_options, 2758| 59.2k| decode_up_to_bad_char); 2759| 59.2k|} simdutf.cpp:_ZN7simdutf8internalL37get_available_implementation_pointersEv: 938| 13.7k|get_available_implementation_pointers() { 939| 13.7k|#if !SIMDUTF_USE_STATIC_INITIALIZATION 940| 13.7k| static const std::initializer_list 941| 13.7k| available_implementation_pointers{ 942| 13.7k| #if SIMDUTF_IMPLEMENTATION_ICELAKE 943| 13.7k| get_icelake_singleton(), 944| 13.7k| #endif 945| 13.7k| #if SIMDUTF_IMPLEMENTATION_HASWELL 946| 13.7k| get_haswell_singleton(), 947| 13.7k| #endif 948| 13.7k| #if SIMDUTF_IMPLEMENTATION_WESTMERE 949| 13.7k| get_westmere_singleton(), 950| 13.7k| #endif 951| | #if SIMDUTF_IMPLEMENTATION_ARM64 952| | get_arm64_singleton(), 953| | #endif 954| | #if SIMDUTF_IMPLEMENTATION_PPC64 955| | get_ppc64_singleton(), 956| | #endif 957| | #if SIMDUTF_IMPLEMENTATION_RVV 958| | get_rvv_singleton(), 959| | #endif 960| | #if SIMDUTF_IMPLEMENTATION_LASX 961| | get_lasx_singleton(), 962| | #endif 963| | #if SIMDUTF_IMPLEMENTATION_LSX 964| | get_lsx_singleton(), 965| | #endif 966| 13.7k| #if SIMDUTF_IMPLEMENTATION_FALLBACK 967| 13.7k| get_fallback_singleton(), 968| 13.7k| #endif 969| 13.7k| }; 970| 13.7k|#endif 971| 13.7k| return available_implementation_pointers; 972| 13.7k|} simdutf.cpp:_ZN7simdutf8internalL21get_icelake_singletonEv: 157| 1|static const icelake::implementation *get_icelake_singleton() { 158| 1| #if !SIMDUTF_USE_STATIC_INITIALIZATION 159| 1| static const icelake::implementation icelake_singleton{}; 160| 1| #endif 161| 1| return &icelake_singleton; 162| 1|} simdutf.cpp:_ZN7simdutf8internalL21get_haswell_singletonEv: 168| 1|static const haswell::implementation *get_haswell_singleton() { 169| 1| #if !SIMDUTF_USE_STATIC_INITIALIZATION 170| 1| static const haswell::implementation haswell_singleton{}; 171| 1| #endif 172| 1| return &haswell_singleton; 173| 1|} simdutf.cpp:_ZN7simdutf8internalL22get_westmere_singletonEv: 179| 1|static const westmere::implementation *get_westmere_singleton() { 180| 1| #if !SIMDUTF_USE_STATIC_INITIALIZATION 181| 1| static const westmere::implementation westmere_singleton{}; 182| 1| #endif 183| 1| return &westmere_singleton; 184| 1|} simdutf.cpp:_ZN7simdutf8internalL22get_fallback_singletonEv: 245| 1|static const fallback::implementation *get_fallback_singleton() { 246| 1| #if !SIMDUTF_USE_STATIC_INITIALIZATION 247| 1| static const fallback::implementation fallback_singleton{}; 248| 1| #endif 249| 1| return &fallback_singleton; 250| 1|} _ZN7simdutf8internal49detect_best_supported_implementation_on_first_useC2Ev: 892| 1| : implementation("best_supported_detector", 893| 1| "Detects the best supported implementation and sets it", 894| 1| 0) {} _ZNK7simdutf8internal49detect_best_supported_implementation_on_first_use24base64_to_binary_detailsEPKcmPcNS_14base64_optionsENS_27last_chunk_handling_optionsE: 833| 1| last_chunk_handling_options::loose) const noexcept override { 834| 1| return set_best()->base64_to_binary_details(input, length, output, options, 835| 1| last_chunk_handling_options); 836| 1| } _ZN7simdutf8fallback14implementationC2Ev: 16| 1| : simdutf::implementation("fallback", "Generic fallback implementation", 17| 1| 0) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_110count_onesEm: 14| 19.5k|simdutf_really_inline long long int count_ones(uint64_t input_num) { 15| | return _popcnt64(input_num); 16| 19.5k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115trailing_zeroesEm: 20| 2.87k|simdutf_really_inline int trailing_zeroes(uint64_t input_num) { 21| | #if SIMDUTF_REGULAR_VISUAL_STUDIO 22| | return (int)_tzcnt_u64(input_num); 23| | #else // SIMDUTF_REGULAR_VISUAL_STUDIO 24| 2.87k| return __builtin_ctzll(input_num); 25| 2.87k| #endif // SIMDUTF_REGULAR_VISUAL_STUDIO 26| 2.87k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_115is_power_of_twoImEEbT_: 29| 2.81k|template bool is_power_of_two(T x) { return (x & (x - 1)) == 0; } _ZN7simdutf7haswell14implementationC2Ev: 16| 1| : simdutf::implementation("haswell", "Intel/AMD AVX2", 17| 1| internal::instruction_set::AVX2 | 18| 1| internal::instruction_set::BMI1 | 19| 1| internal::instruction_set::BMI2) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IhEC2Ev: 191| 102k| simdutf_really_inline simd8() : base8_numeric() {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhEC2Ev: 124| 102k| simdutf_really_inline base8_numeric() : base8() {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEEC2Ev: 71| 102k| simdutf_really_inline base8() : base>() {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEC2Ev: 14| 102k| simdutf_really_inline base() : value{__m256i()} {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IhE9reduce_orEv: 294| 17.3k| simdutf_really_inline simd8 reduce_or() const { 295| 17.3k| return this->chunks[0] | this->chunks[1]; 296| 17.3k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEorES5_: 50| 133k| simdutf_really_inline Child operator|(const Child other) const { 51| 133k| return _mm256_or_si256(*this, other); 52| 133k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IhEC2EDv4_x: 193| 1.65M| : base8_numeric(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhEC2EDv4_x: 126| 1.65M| : base8(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEEC2EDv4_x: 73| 1.65M| simdutf_really_inline base8(const __m256i _value) : base>(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEC2EDv4_x: 17| 1.65M| simdutf_really_inline base(const __m256i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE8is_asciiEv: 234| 17.3k| simdutf_really_inline bool is_ascii() const { 235| 17.3k| return _mm256_movemask_epi8(*this) == 0; 236| 17.3k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEoRES5_: 59| 82.0k| simdutf_really_inline Child &operator|=(const Child other) { 60| 82.0k| auto this_cast = static_cast(this); 61| 82.0k| *this_cast = *this_cast | other; 62| 82.0k| return *this_cast; 63| 82.0k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEE4prevILi1EEENS4_IhEES8_: 83| 43.9k| simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { 84| | return _mm256_alignr_epi8( 85| 43.9k| *this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); 86| 43.9k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE3shrILi4EEES4_v: 245| 87.9k| template simdutf_really_inline simd8 shr() const { 246| 87.9k| return simd8(_mm256_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); 247| 87.9k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhE9lookup_16IhEENS2_5simd8IT_EES7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_: 157| 131k| L replace15) const { 158| 131k| return lookup_16(simd8::repeat_16( 159| 131k| replace0, replace1, replace2, replace3, replace4, replace5, replace6, 160| 131k| replace7, replace8, replace9, replace10, replace11, replace12, 161| 131k| replace13, replace14, replace15)); 162| 131k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhE9lookup_16IhEENS2_5simd8IT_EES8_: 148| 131k| simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { 149| 131k| return _mm256_shuffle_epi8(lookup_table, *this); 150| 131k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhE9repeat_16Ehhhhhhhhhhhhhhhh: 118| 131k| T v14, T v15) { 119| 131k| return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, 120| 131k| v14, v15, v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, 121| 131k| v12, v13, v14, v15); 122| 131k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IhEC2Ehhhhhhhhhhhhhhhhhhhhhhhhhhhhhhhh: 207| 131k| : simd8(_mm256_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, 208| 131k| v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, 209| 131k| v22, v23, v24, v25, v26, v27, v28, v29, v30, 210| 131k| v31)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IhEC2Eh: 195| 307k| simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEE4prevILi2EEENS4_IhEES8_: 83| 34.5k| simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { 84| | return _mm256_alignr_epi8( 85| 34.5k| *this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); 86| 34.5k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEE4prevILi3EEENS4_IhEES8_: 83| 34.5k| simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { 84| | return _mm256_alignr_epi8( 85| 34.5k| *this, _mm256_permute2x128_si256(prev_chunk, *this, 0x21), 16 - N); 86| 34.5k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE14saturating_subES4_: 214| 80.5k| saturating_sub(const simd8 other) const { 215| 80.5k| return _mm256_subs_epu8(*this, other); 216| 80.5k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IbEC2EDv4_x: 95| 145k| simdutf_really_inline simd8(const __m256i _value) : base8(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5base8IbNS2_5simd8IbEEEC2EDv4_x: 73| 145k| simdutf_really_inline base8(const __m256i _value) : base>(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IbEEEC2EDv4_x: 17| 145k| simdutf_really_inline base(const __m256i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IbEEEcvRKDv4_xEv: 19| 75.2k| simdutf_really_inline operator const __m256i &() const { return this->value; } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEeoES5_: 56| 34.5k| simdutf_really_inline Child operator^(const Child other) const { 57| 34.5k| return _mm256_xor_si256(*this, other); 58| 34.5k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IhEC2EPKh: 197| 12.6k| simdutf_really_inline simd8(const uint8_t values[32]) : simd8(load(values)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhE4loadEPKh: 111| 45.0k| static simdutf_really_inline simd8 load(const T values[32]) { 112| 45.0k| return _mm256_loadu_si256(reinterpret_cast(values)); 113| 45.0k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE7gt_bitsES4_: 225| 11.4k| gt_bits(const simd8 other) const { 226| 11.4k| return this->saturating_sub(other); 227| 11.4k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_6simd16ItEEEC2EDv4_x: 17| 393k| simdutf_really_inline base(const __m256i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_6simd16ItEEEcvRKDv4_xEv: 19| 442k| simdutf_really_inline operator const __m256i &() const { return this->value; } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IbE10to_bitmaskEv: 99| 70.5k| simdutf_really_inline uint32_t to_bitmask() const { 100| 70.5k| return uint32_t(_mm256_movemask_epi8(value)); 101| 70.5k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE21any_bits_set_anywhereEv: 241| 39.7k| simdutf_really_inline bool any_bits_set_anywhere() const { 242| 39.7k| return !bits_not_set_anywhere(); 243| 39.7k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE21bits_not_set_anywhereEv: 237| 39.7k| simdutf_really_inline bool bits_not_set_anywhere() const { 238| 39.7k| return _mm256_testz_si256(*this, *this); 239| 39.7k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IhE8is_asciiEv: 298| 1.67k| simdutf_really_inline bool is_ascii() const { 299| 1.67k| return this->reduce_or().is_ascii(); 300| 1.67k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IhE13gteq_unsignedEh: 341| 1.34k| simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { 342| 1.34k| const simd8 mask = simd8::splat(m); 343| 1.34k| return simd8x64((simd8(__m256i(this->chunks[0])) >= mask), 344| 1.34k| (simd8(__m256i(this->chunks[1])) >= mask)) 345| 1.34k| .to_bitmask(); 346| 1.34k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhEgeES4_: 229| 16.3k| operator>=(const simd8 other) const { 230| 16.3k| return other.min_val(*this) == other; 231| 16.3k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE7min_valES4_: 220| 16.3k| min_val(const simd8 other) const { 221| 16.3k| return _mm256_min_epu8(other, *this); 222| 16.3k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IbEC2ENS2_5simd8IbEES6_: 272| 14.1k| : chunks{chunk0, chunk1} {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IbE10to_bitmaskEv: 282| 14.1k| simdutf_really_inline uint64_t to_bitmask() const { 283| 14.1k| uint64_t r_lo = uint32_t(this->chunks[0].to_bitmask()); 284| 14.1k| uint64_t r_hi = this->chunks[1].to_bitmask(); 285| 14.1k| return r_lo | (r_hi << 32); 286| 14.1k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaEC2EPKa: 274| 19.6k| : chunks{simd8::load(ptr), 275| 19.6k| simd8::load(ptr + sizeof(simd8) / sizeof(T))} {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIaE4loadEPKa: 111| 66.3k| static simdutf_really_inline simd8 load(const T values[32]) { 112| 66.3k| return _mm256_loadu_si256(reinterpret_cast(values)); 113| 66.3k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IaEC2EDv4_x: 169| 123k| : base8_numeric(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIaEC2EDv4_x: 126| 123k| : base8(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5base8IaNS2_5simd8IbEEEC2EDv4_x: 73| 123k| simdutf_really_inline base8(const __m256i _value) : base>(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IaEEEC2EDv4_x: 17| 123k| simdutf_really_inline base(const __m256i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE8is_asciiEv: 298| 17.4k| simdutf_really_inline bool is_ascii() const { 299| 17.4k| return this->reduce_or().is_ascii(); 300| 17.4k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE9reduce_orEv: 294| 17.4k| simdutf_really_inline simd8 reduce_or() const { 295| 17.4k| return this->chunks[0] | this->chunks[1]; 296| 17.4k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IaEEEorES5_: 50| 17.4k| simdutf_really_inline Child operator|(const Child other) const { 51| 17.4k| return _mm256_or_si256(*this, other); 52| 17.4k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IaEEEcvRKDv4_xEv: 19| 176k| simdutf_really_inline operator const __m256i &() const { return this->value; } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IaE8is_asciiEv: 177| 17.4k| simdutf_really_inline bool is_ascii() const { 178| 17.4k| return _mm256_movemask_epi8(*this) == 0; 179| 17.4k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE5storeEPa: 277| 3.06k| simdutf_really_inline void store(T *ptr) const { 278| 3.06k| this->chunks[0].store(ptr + sizeof(simd8) * 0 / sizeof(T)); 279| 3.06k| this->chunks[1].store(ptr + sizeof(simd8) * 1 / sizeof(T)); 280| 3.06k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIaE5storeEPa: 129| 6.13k| simdutf_really_inline void store(T dst[32]) const { 130| 6.13k| return _mm256_storeu_si256(reinterpret_cast<__m256i *>(dst), *this); 131| 6.13k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IaEcvNS3_IhEEEv: 256| 31.6k|simdutf_really_inline simd8::operator simd8() const { 257| 31.6k| return this->value; 258| 31.6k|} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE2ltEa: 325| 10.5k| simdutf_really_inline uint64_t lt(const T m) const { 326| 10.5k| const simd8 mask = simd8::splat(m); 327| 10.5k| return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask) 328| 10.5k| .to_bitmask(); 329| 10.5k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIaE5splatEa: 105| 39.7k| static simdutf_really_inline simd8 splat(T _value) { 106| 39.7k| return _mm256_set1_epi8(_value); 107| 39.7k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IaEltES4_: 184| 21.0k| simdutf_really_inline simd8 operator<(const simd8 other) const { 185| 21.0k| return _mm256_cmpgt_epi8(other, *this); 186| 21.0k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE20store_ascii_as_utf32EPDi: 310| 794| simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { 311| 794| this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8) * 0); 312| 794| this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8) * 1); 313| 794| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IaEEE20store_ascii_as_utf32EPDi: 36| 1.58k| simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { 37| 1.58k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), 38| 1.58k| _mm256_cvtepu8_epi32(_mm256_castsi256_si128(*this))); 39| 1.58k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 8), 40| 1.58k| _mm256_cvtepu8_epi32(_mm256_castsi256_si128( 41| 1.58k| _mm256_srli_si256(*this, 8)))); 42| 1.58k| _mm256_storeu_si256( 43| 1.58k| reinterpret_cast<__m256i *>(ptr + 16), 44| 1.58k| _mm256_cvtepu8_epi32(_mm256_extractf128_si256(*this, 1))); 45| 1.58k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 24), 46| | _mm256_cvtepu8_epi32(_mm_srli_si128( 47| 1.58k| _mm256_extractf128_si256(*this, 1), 8))); 48| 1.58k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhE4zeroEv: 108| 40.9k| static simdutf_really_inline simd8 zero() { 109| 40.9k| return _mm256_setzero_si256(); 110| 40.9k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IaEgtES4_: 181| 31.4k| simdutf_really_inline simd8 operator>(const simd8 other) const { 182| 31.4k| return _mm256_cmpgt_epi8(*this, other); 183| 31.4k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd5simd8IaEC2Ea: 172| 27.0k| simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhEmIENS2_5simd8IhEE: 137| 40.6k| simdutf_really_inline simd8 &operator-=(const simd8 other) { 138| 40.6k| *this = *this - other; 139| 40.6k| return *static_cast *>(this); 140| 40.6k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhEmiENS2_5simd8IhEE: 134| 40.6k| simdutf_really_inline simd8 operator-(const simd8 other) const { 135| 40.6k| return _mm256_sub_epi8(*this, other); 136| 40.6k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd5simd8IhE9sum_bytesEv: 249| 3.21k| simdutf_really_inline uint64_t sum_bytes() const { 250| 3.21k| const auto tmp = _mm256_sad_epu8(value, _mm256_setzero_si256()); 251| | 252| 3.21k| return _mm256_extract_epi64(tmp, 0) + _mm256_extract_epi64(tmp, 1) + 253| 3.21k| _mm256_extract_epi64(tmp, 2) + _mm256_extract_epi64(tmp, 3); 254| 3.21k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE2gtEa: 331| 2.22k| simdutf_really_inline uint64_t gt(const T m) const { 332| 2.22k| const simd8 mask = simd8::splat(m); 333| 2.22k| return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask) 334| 2.22k| .to_bitmask(); 335| 2.22k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_6simd16IbEEEC2EDv4_x: 17| 11.1k| simdutf_really_inline base(const __m256i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_6simd16IbEEEcvRKDv4_xEv: 19| 11.1k| simdutf_really_inline operator const __m256i &() const { return this->value; } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd13base8_numericIhE5splatEh: 105| 335k| static simdutf_really_inline simd8 splat(T _value) { 106| 335k| return _mm256_set1_epi8(_value); 107| 335k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEcvRKDv4_xEv: 19| 2.14M| simdutf_really_inline operator const __m256i &() const { return this->value; } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdeqENS2_5simd8IhEES4_: 76| 58.6k| const simd8 rhs) { 77| 58.6k| return _mm256_cmpeq_epi8(lhs, rhs); 78| 58.6k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IhEC2EPKh: 274| 16.1k| : chunks{simd8::load(ptr), 275| 16.1k| simd8::load(ptr + sizeof(simd8) / sizeof(T))} {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEanES5_: 53| 296k| simdutf_really_inline Child operator&(const Child other) const { 54| 296k| return _mm256_and_si256(*this, other); 55| 296k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_6simd16ItEEEanES5_: 53| 73.4k| simdutf_really_inline Child operator&(const Child other) const { 54| 73.4k| return _mm256_and_si256(*this, other); 55| 73.4k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE20store_ascii_as_utf16ILNS_10endiannessE0EEEvPDs: 303| 812| simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { 304| 812| this->chunks[0].template store_ascii_as_utf16(ptr + 305| 812| sizeof(simd8) * 0); 306| 812| this->chunks[1].template store_ascii_as_utf16(ptr + 307| 812| sizeof(simd8) * 1); 308| 812| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IaEEE20store_ascii_as_utf16ILNS_10endiannessE0EEEvPDs: 22| 1.62k| simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { 23| 1.62k| __m256i first = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(*this)); 24| 1.62k| __m256i second = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(*this, 1)); 25| 1.62k| if (big_endian) { ------------------ | Branch (25:9): [Folded, False: 1.62k] ------------------ 26| 0| const __m256i swap = _mm256_setr_epi8( 27| 0| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 28| 0| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 29| 0| first = _mm256_shuffle_epi8(first, swap); 30| 0| second = _mm256_shuffle_epi8(second, swap); 31| 0| } 32| 1.62k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), first); 33| 1.62k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 16), second); 34| 1.62k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd8simd8x64IaE20store_ascii_as_utf16ILNS_10endiannessE1EEEvPDs: 303| 827| simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { 304| 827| this->chunks[0].template store_ascii_as_utf16(ptr + 305| 827| sizeof(simd8) * 0); 306| 827| this->chunks[1].template store_ascii_as_utf16(ptr + 307| 827| sizeof(simd8) * 1); 308| 827| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd4baseINS2_5simd8IaEEE20store_ascii_as_utf16ILNS_10endiannessE1EEEvPDs: 22| 1.65k| simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { 23| 1.65k| __m256i first = _mm256_cvtepu8_epi16(_mm256_castsi256_si128(*this)); 24| 1.65k| __m256i second = _mm256_cvtepu8_epi16(_mm256_extractf128_si256(*this, 1)); 25| 1.65k| if (big_endian) { ------------------ | Branch (25:9): [True: 1.65k, Folded] ------------------ 26| 1.65k| const __m256i swap = _mm256_setr_epi8( 27| 1.65k| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 28| 1.65k| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 29| 1.65k| first = _mm256_shuffle_epi8(first, swap); 30| 1.65k| second = _mm256_shuffle_epi8(second, swap); 31| 1.65k| } 32| 1.65k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr), first); 33| 1.65k| _mm256_storeu_si256(reinterpret_cast<__m256i *>(ptr + 16), second); 34| 1.65k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd16ItEC2EDv4_x: 97| 393k| : base16_numeric(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd14base16_numericItEC2EDv4_x: 73| 393k| : base16(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6base16ItNS2_6simd16IbEEEC2EDv4_x: 20| 393k| : base>(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd16ItE4packERKS4_S6_: 149| 40.2k| const simd16 &v1) { 150| | // Note: the AVX2 variant of pack operates on 128-bit lanes, thus 151| | // we have to shuffle lanes in order to produce bytes in the 152| | // correct order. 153| | 154| | // get the 0th lanes 155| 40.2k| const __m128i lo_0 = _mm256_extracti128_si256(v0, 0); 156| 40.2k| const __m128i lo_1 = _mm256_extracti128_si256(v1, 0); 157| | 158| | // get the 1st lanes 159| 40.2k| const __m128i hi_0 = _mm256_extracti128_si256(v0, 1); 160| 40.2k| const __m128i hi_1 = _mm256_extracti128_si256(v1, 1); 161| | 162| | // build new vectors (shuffle lanes) 163| 40.2k| const __m256i t0 = _mm256_set_m128i(lo_1, lo_0); ------------------ | | 4| 40.2k| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 5| 40.2k| _mm256_castsi128_si256(xmm2), 2) ------------------ 164| 40.2k| const __m256i t1 = _mm256_set_m128i(hi_1, hi_0); ------------------ | | 4| 40.2k| _mm256_permute2f128_si256(_mm256_castsi128_si256(xmm1), \ | | 5| 40.2k| _mm256_castsi128_si256(xmm2), 2) ------------------ 165| | 166| | // pack code units in linear order from v0 and v1 167| 40.2k| return _mm256_packus_epi16(t0, t1); 168| 40.2k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd14base16_numericItE4loadEPKt: 66| 91.6k| static simdutf_really_inline simd16 load(const T values[8]) { 67| 91.6k| return _mm256_loadu_si256(reinterpret_cast(values)); 68| 91.6k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd16ItE10swap_bytesEv: 139| 5.42k| simdutf_really_inline simd16 swap_bytes() const { 140| 5.42k| const __m256i swap = _mm256_setr_epi8( 141| 5.42k| 1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14, 17, 16, 19, 18, 142| 5.42k| 21, 20, 23, 22, 25, 24, 27, 26, 29, 28, 31, 30); 143| 5.42k| return _mm256_shuffle_epi8(*this, swap); 144| 5.42k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd14base16_numericItE5splatEt: 58| 90.3k| static simdutf_really_inline simd16 splat(T _value) { 59| 90.3k| return _mm256_set1_epi16(_value); 60| 90.3k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdeqENS2_6simd16ItEES4_: 26| 11.1k| const simd16 rhs) { 27| 11.1k| return _mm256_cmpeq_epi16(lhs, rhs); 28| 11.1k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd16IbEC2EDv4_x: 45| 11.1k| simdutf_really_inline simd16(const __m256i _value) : base16(_value) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6base16IbNS2_6simd16IbEEEC2EDv4_x: 20| 11.1k| : base>(_value) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd16ItE3shrILi8EEES4_v: 134| 40.3k| template simdutf_really_inline simd16 shr() const { 135| 40.3k| return simd16(_mm256_srli_epi16(*this, N)); 136| 40.3k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd16ItEC2EPKDs: 104| 80.4k| : simd16(load(reinterpret_cast(values))) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd16ItEC2Et: 100| 64.5k| simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd14base16_numericItE4zeroEv: 62| 25.7k| static simdutf_really_inline simd16 zero() { 63| 25.7k| return _mm256_setzero_si256(); 64| 25.7k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd3minENS2_6simd16ItEES4_: 259| 22.2k|simd16 min(const simd16 a, simd16 b) { 260| 22.2k| return _mm256_min_epu16(a.value, b.value); 261| 22.2k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd14base16_numericItEpLENS2_6simd16ItEE: 87| 33.3k| simdutf_really_inline simd16 &operator+=(const simd16 other) { 88| 33.3k| *this = *this + other; 89| 33.3k| return *static_cast *>(this); 90| 33.3k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd14base16_numericItEplENS2_6simd16ItEE: 84| 33.3k| simdutf_really_inline simd16 operator+(const simd16 other) const { 85| 33.3k| return _mm256_add_epi16(*this, other); 86| 33.3k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd16ItE3sumEv: 170| 25.7k| simdutf_really_inline uint64_t sum() const { 171| 25.7k| const auto lo_u16 = _mm256_and_si256(value, _mm256_set1_epi32(0x0000ffff)); 172| 25.7k| const auto hi_u16 = _mm256_srli_epi32(value, 16); 173| 25.7k| const auto sum_u32 = _mm256_add_epi32(lo_u16, hi_u16); 174| | 175| 25.7k| const auto lo_u32 = 176| 25.7k| _mm256_and_si256(sum_u32, _mm256_set1_epi64x(0xffffffff)); 177| 25.7k| const auto hi_u32 = _mm256_srli_epi64(sum_u32, 32); 178| 25.7k| const auto sum_u64 = _mm256_add_epi64(lo_u32, hi_u32); 179| | 180| 25.7k| return uint64_t(_mm256_extract_epi64(sum_u64, 0)) + 181| 25.7k| uint64_t(_mm256_extract_epi64(sum_u64, 1)) + 182| 25.7k| uint64_t(_mm256_extract_epi64(sum_u64, 2)) + 183| | uint64_t(_mm256_extract_epi64(sum_u64, 3)); 184| 25.7k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd32IjE5splatEj: 46| 31.5k| simdutf_really_inline static simd32 splat(uint32_t v) { 47| 31.5k| return _mm256_set1_epi32(v); 48| 31.5k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd32IjEC2EDv4_x: 9| 267k| simdutf_really_inline simd32(const __m256i v) : value(v) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd32IjE4zeroEv: 42| 5.72k| simdutf_really_inline static simd32 zero() { 43| 5.72k| return _mm256_setzero_si256(); 44| 5.72k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd32IjEC2IDiEEPKT_: 13| 53.7k| : value(_mm256_loadu_si256(reinterpret_cast(ptr))) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd32IjE10swap_bytesEv: 27| 46.1k| simdutf_really_inline simd32 swap_bytes() const { 28| 46.1k| const __m256i shuffle = 29| 46.1k| _mm256_setr_epi8(3, 2, 1, 0, 7, 6, 5, 4, 8, 9, 10, 11, 15, 14, 13, 12, 30| 46.1k| 3, 2, 1, 0, 7, 6, 5, 4, 8, 9, 10, 11, 15, 14, 13, 12); 31| | 32| 46.1k| return _mm256_shuffle_epi8(value, shuffle); 33| 46.1k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd3maxENS2_6simd32IjEES4_: 80| 92.2k| const simd32 b) { 81| 92.2k| return _mm256_max_epu32(a.value, b.value); 82| 92.2k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdplENS2_6simd32IjEES4_: 90| 46.1k| const simd32 b) { 91| 46.1k| return _mm256_add_epi32(a.value, b.value); 92| 46.1k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdgtENS2_6simd32IjEES4_: 109| 4.97k| const simd32 b) { 110| 4.97k| return !(b >= a); 111| 4.97k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdntENS2_6simd32IbEE: 104| 4.97k|simdutf_really_inline simd32 operator!(const simd32 v) { 105| 4.97k| return _mm256_xor_si256(v.value, _mm256_set1_epi8(-1)); 106| 4.97k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd32IbEC2EDv4_x: 59| 9.95k| simdutf_really_inline simd32(const __m256i v) : value(v) {} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdgeENS2_6simd32IjEES4_: 100| 4.97k| const simd32 b) { 101| 4.97k| return _mm256_cmpeq_epi32(_mm256_max_epu32(a.value, b.value), a.value); 102| 4.97k|} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd32IbE3anyEv: 61| 4.97k| simdutf_really_inline bool any() const { 62| 4.97k| return _mm256_movemask_epi8(value) != 0; 63| 4.97k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simdanENS2_6simd32IjEES4_: 85| 22.8k| const simd32 a) { 86| 22.8k| return _mm256_and_si256(a.value, b.value); 87| 22.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd3minENS2_6simd32IjEES4_: 75| 22.8k| const simd32 a) { 76| 22.8k| return _mm256_min_epu32(a.value, b.value); 77| 22.8k|} simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd32IjEpLES4_: 36| 22.8k| simdutf_really_inline simd32 &operator+=(const simd32 other) { 37| 22.8k| value = _mm256_add_epi32(value, other.value); 38| 22.8k| return *this; 39| 22.8k| } simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd32IjE3sumEv: 15| 749| simdutf_really_inline uint64_t sum() const { 16| 749| const __m256i mask = _mm256_set1_epi64x(0xffffffff); 17| 749| const __m256i t0 = _mm256_and_si256(value, mask); 18| 749| const __m256i t1 = _mm256_srli_epi64(value, 32); 19| 749| const __m256i t2 = _mm256_add_epi64(t0, t1); 20| | 21| 749| return uint64_t(_mm256_extract_epi64(t2, 0)) + 22| 749| uint64_t(_mm256_extract_epi64(t2, 1)) + 23| 749| uint64_t(_mm256_extract_epi64(t2, 2)) + 24| | uint64_t(_mm256_extract_epi64(t2, 3)); 25| 749| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd64ImE4zeroEv: 27| 40.9k| simdutf_really_inline static simd64 zero() { 28| 40.9k| return _mm256_setzero_si256(); 29| 40.9k| } simdutf.cpp:_ZN7simdutf7haswell12_GLOBAL__N_14simd6simd64ImEC2EDv4_x: 9| 40.9k| simdutf_really_inline simd64(const __m256i v) : value(v) {} simdutf.cpp:_ZNK7simdutf7haswell12_GLOBAL__N_14simd6simd64ImE3sumEv: 15| 40.9k| simdutf_really_inline uint64_t sum() const { 16| 40.9k| return _mm256_extract_epi64(value, 0) + _mm256_extract_epi64(value, 1) + 17| 40.9k| _mm256_extract_epi64(value, 2) + _mm256_extract_epi64(value, 3); 18| 40.9k| } _ZN7simdutf7icelake14implementationC2Ev: 16| 1| : simdutf::implementation( 17| 1| "icelake", 18| 1| "Intel AVX512 (AVX-512BW, AVX-512CD, AVX-512VL, AVX-512VBMI2 " 19| 1| "extensions)", 20| 1| internal::instruction_set::AVX2 | internal::instruction_set::BMI1 | 21| 1| internal::instruction_set::BMI2 | 22| 1| internal::instruction_set::AVX512BW | 23| 1| internal::instruction_set::AVX512CD | 24| 1| internal::instruction_set::AVX512VL | 25| 1| internal::instruction_set::AVX512VBMI2 | 26| 1| internal::instruction_set::AVX512VPOPCNTDQ) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_110count_onesEm: 14| 28.3k|simdutf_really_inline long long int count_ones(uint64_t input_num) { 15| | return _popcnt64(input_num); 16| 28.3k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115trailing_zeroesEm: 20| 888|simdutf_really_inline int trailing_zeroes(uint64_t input_num) { 21| | #if SIMDUTF_REGULAR_VISUAL_STUDIO 22| | unsigned long ret; 23| | _BitScanForward64(&ret, input_num); 24| | return (int)ret; 25| | #else // SIMDUTF_REGULAR_VISUAL_STUDIO 26| 888| return __builtin_ctzll(input_num); 27| 888| #endif // SIMDUTF_REGULAR_VISUAL_STUDIO 28| 888|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_115is_power_of_twoImEEbT_: 31| 981|template bool is_power_of_two(T x) { return (x & (x - 1)) == 0; } _ZN7simdutf8westmere14implementationC2Ev: 18| 1| : simdutf::implementation("westmere", "Intel/AMD SSE4.2", 19| 1| internal::instruction_set::SSE42) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IhEC2Ev: 183| 102k| simdutf_really_inline simd8() : base8_numeric() {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhEC2Ev: 118| 102k| simdutf_really_inline base8_numeric() : base8() {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEEC2Ev: 68| 102k| simdutf_really_inline base8() : base>() {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEC2Ev: 13| 102k| simdutf_really_inline base() : value{__m128i()} {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IhE9reduce_orEv: 290| 17.9k| simdutf_really_inline simd8 reduce_or() const { 291| 17.9k| return (this->chunks[0] | this->chunks[1]) | 292| 17.9k| (this->chunks[2] | this->chunks[3]); 293| 17.9k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEorES5_: 42| 270k| simdutf_really_inline Child operator|(const Child other) const { 43| 270k| return _mm_or_si128(*this, other); 44| 270k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IhEC2EDv2_x: 185| 3.49M| : base8_numeric(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhEC2EDv2_x: 120| 3.49M| : base8(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEEC2EDv2_x: 69| 3.49M| simdutf_really_inline base8(const __m128i _value) : base>(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEC2EDv2_x: 16| 3.49M| simdutf_really_inline base(const __m128i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE8is_asciiEv: 228| 17.9k| simdutf_really_inline bool is_ascii() const { 229| 17.9k| return _mm_movemask_epi8(*this) == 0; 230| 17.9k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEoRES5_: 51| 138k| simdutf_really_inline Child &operator|=(const Child other) { 52| 138k| auto this_cast = static_cast(this); 53| 138k| *this_cast = *this_cast | other; 54| 138k| return *this_cast; 55| 138k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEE4prevILi1EEENS4_IhEES8_: 79| 100k| simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { 80| | return _mm_alignr_epi8(*this, prev_chunk, 16 - N); 81| 100k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE3shrILi4EEES4_v: 238| 200k| template simdutf_really_inline simd8 shr() const { 239| 200k| return simd8(_mm_srli_epi16(*this, N)) & uint8_t(0xFFu >> N); 240| 200k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhE9lookup_16IhEENS2_5simd8IT_EES7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_S7_: 151| 300k| L replace15) const { 152| 300k| return lookup_16(simd8::repeat_16( 153| 300k| replace0, replace1, replace2, replace3, replace4, replace5, replace6, 154| 300k| replace7, replace8, replace9, replace10, replace11, replace12, 155| 300k| replace13, replace14, replace15)); 156| 300k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhE9lookup_16IhEENS2_5simd8IT_EES8_: 142| 300k| simdutf_really_inline simd8 lookup_16(simd8 lookup_table) const { 143| 300k| return _mm_shuffle_epi8(lookup_table, *this); 144| 300k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhE9repeat_16Ehhhhhhhhhhhhhhhh: 113| 300k| T v14, T v15) { 114| 300k| return simd8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, v12, v13, 115| 300k| v14, v15); 116| 300k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IhEC2Ehhhhhhhhhhhhhhhh: 196| 300k| : simd8(_mm_setr_epi8(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10, v11, 197| 300k| v12, v13, v14, v15)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IhEC2Eh: 188| 630k| simdutf_really_inline simd8(uint8_t _value) : simd8(splat(_value)) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEE4prevILi2EEENS4_IhEES8_: 79| 77.8k| simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { 80| | return _mm_alignr_epi8(*this, prev_chunk, 16 - N); 81| 77.8k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5base8IhNS2_5simd8IbEEE4prevILi3EEENS4_IhEES8_: 79| 77.8k| simdutf_really_inline simd8 prev(const simd8 prev_chunk) const { 80| | return _mm_alignr_epi8(*this, prev_chunk, 16 - N); 81| 77.8k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE14saturating_subES4_: 201| 167k| saturating_sub(const simd8 other) const { 202| 167k| return _mm_subs_epu8(*this, other); 203| 167k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IbEC2EDv2_x: 91| 308k| simdutf_really_inline simd8(const __m128i _value) : base8(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5base8IbNS2_5simd8IbEEEC2EDv2_x: 69| 308k| simdutf_really_inline base8(const __m128i _value) : base>(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IbEEEC2EDv2_x: 16| 308k| simdutf_really_inline base(const __m128i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IbEEEcvRKDv2_xEv: 18| 308k| simdutf_really_inline operator const __m128i &() const { return this->value; } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEeoES5_: 48| 77.8k| simdutf_really_inline Child operator^(const Child other) const { 49| 77.8k| return _mm_xor_si128(*this, other); 50| 77.8k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IhEC2EPKh: 190| 13.0k| simdutf_really_inline simd8(const uint8_t *values) : simd8(load(values)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhE4loadEPKh: 106| 79.5k| static simdutf_really_inline simd8 load(const T values[16]) { 107| 79.5k| return _mm_loadu_si128(reinterpret_cast(values)); 108| 79.5k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE7gt_bitsES4_: 212| 11.7k| gt_bits(const simd8 other) const { 213| 11.7k| return this->saturating_sub(other); 214| 11.7k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_6simd16ItEEEC2EDv2_x: 16| 729k| simdutf_really_inline base(const __m128i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_6simd16ItEEEcvRKDv2_xEv: 18| 703k| simdutf_really_inline operator const __m128i &() const { return this->value; } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IbE10to_bitmaskEv: 95| 125k| simdutf_really_inline int to_bitmask() const { 96| 125k| return _mm_movemask_epi8(*this); 97| 125k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE21any_bits_set_anywhereEv: 235| 39.4k| simdutf_really_inline bool any_bits_set_anywhere() const { 236| 39.4k| return !bits_not_set_anywhere(); 237| 39.4k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE21bits_not_set_anywhereEv: 232| 39.4k| simdutf_really_inline bool bits_not_set_anywhere() const { 233| 39.4k| return _mm_testz_si128(*this, *this); 234| 39.4k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IhE8is_asciiEv: 295| 1.83k| simdutf_really_inline bool is_ascii() const { 296| 1.83k| return this->reduce_or().is_ascii(); 297| 1.83k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IhE13gteq_unsignedEh: 352| 1.45k| simdutf_really_inline uint64_t gteq_unsigned(const uint8_t m) const { 353| 1.45k| const simd8 mask = simd8::splat(m); 354| 1.45k| return simd8x64(simd8(__m128i(this->chunks[0])) >= mask, 355| 1.45k| simd8(__m128i(this->chunks[1])) >= mask, 356| 1.45k| simd8(__m128i(this->chunks[2])) >= mask, 357| 1.45k| simd8(__m128i(this->chunks[3])) >= mask) 358| 1.45k| .to_bitmask(); 359| 1.45k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhEgeES4_: 217| 40.2k| operator>=(const simd8 other) const { 218| 40.2k| return other.min_val(*this) == other; 219| 40.2k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE7min_valES4_: 207| 40.2k| min_val(const simd8 other) const { 208| 40.2k| return _mm_min_epu8(*this, other); 209| 40.2k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IbEC2ENS2_5simd8IbEES6_S6_S6_: 268| 17.5k| : chunks{chunk0, chunk1, chunk2, chunk3} {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IbE10to_bitmaskEv: 318| 17.5k| simdutf_really_inline uint64_t to_bitmask() const { 319| 17.5k| uint64_t r0 = uint32_t(this->chunks[0].to_bitmask()); 320| 17.5k| uint64_t r1 = this->chunks[1].to_bitmask(); 321| 17.5k| uint64_t r2 = this->chunks[2].to_bitmask(); 322| 17.5k| uint64_t r3 = this->chunks[3].to_bitmask(); 323| 17.5k| return r0 | (r1 << 16) | (r2 << 32) | (r3 << 48); 324| 17.5k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaEC2EPKa: 270| 23.2k| : chunks{simd8::load(ptr), 271| 23.2k| simd8::load(ptr + sizeof(simd8) / sizeof(T)), 272| 23.2k| simd8::load(ptr + 2 * sizeof(simd8) / sizeof(T)), 273| 23.2k| simd8::load(ptr + 3 * sizeof(simd8) / sizeof(T))} {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIaE4loadEPKa: 106| 163k| static simdutf_really_inline simd8 load(const T values[16]) { 107| 163k| return _mm_loadu_si128(reinterpret_cast(values)); 108| 163k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IaEC2EDv2_x: 163| 310k| : base8_numeric(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIaEC2EDv2_x: 120| 310k| : base8(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5base8IaNS2_5simd8IbEEEC2EDv2_x: 69| 310k| simdutf_really_inline base8(const __m128i _value) : base>(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IaEEEC2EDv2_x: 16| 310k| simdutf_really_inline base(const __m128i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE8is_asciiEv: 295| 20.4k| simdutf_really_inline bool is_ascii() const { 296| 20.4k| return this->reduce_or().is_ascii(); 297| 20.4k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE9reduce_orEv: 290| 20.4k| simdutf_really_inline simd8 reduce_or() const { 291| 20.4k| return (this->chunks[0] | this->chunks[1]) | 292| 20.4k| (this->chunks[2] | this->chunks[3]); 293| 20.4k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IaEEEorES5_: 42| 61.3k| simdutf_really_inline Child operator|(const Child other) const { 43| 61.3k| return _mm_or_si128(*this, other); 44| 61.3k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IaEEEcvRKDv2_xEv: 18| 458k| simdutf_really_inline operator const __m128i &() const { return this->value; } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IaE8is_asciiEv: 168| 20.4k| simdutf_really_inline bool is_ascii() const { 169| 20.4k| return _mm_movemask_epi8(*this) == 0; 170| 20.4k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE5storeEPa: 275| 3.30k| simdutf_really_inline void store(T *ptr) const { 276| 3.30k| this->chunks[0].store(ptr + sizeof(simd8) * 0 / sizeof(T)); 277| 3.30k| this->chunks[1].store(ptr + sizeof(simd8) * 1 / sizeof(T)); 278| 3.30k| this->chunks[2].store(ptr + sizeof(simd8) * 2 / sizeof(T)); 279| 3.30k| this->chunks[3].store(ptr + sizeof(simd8) * 3 / sizeof(T)); 280| 3.30k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIaE5storeEPa: 123| 13.2k| simdutf_really_inline void store(T dst[16]) const { 124| 13.2k| return _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), *this); 125| 13.2k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IaEcvNS3_IhEEEv: 251| 93.4k|simdutf_really_inline simd8::operator simd8() const { 252| 93.4k| return this->value; 253| 93.4k|} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE2ltEa: 326| 13.3k| simdutf_really_inline uint64_t lt(const T m) const { 327| 13.3k| const simd8 mask = simd8::splat(m); 328| 13.3k| return simd8x64(this->chunks[0] < mask, this->chunks[1] < mask, 329| 13.3k| this->chunks[2] < mask, this->chunks[3] < mask) 330| 13.3k| .to_bitmask(); 331| 13.3k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIaE5splatEa: 102| 86.2k| static simdutf_really_inline simd8 splat(T _value) { 103| 86.2k| return _mm_set1_epi8(_value); 104| 86.2k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IaEltES4_: 176| 53.4k| simdutf_really_inline simd8 operator<(const simd8 other) const { 177| 53.4k| return _mm_cmpgt_epi8(other, *this); 178| 53.4k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE20store_ascii_as_utf32EPDi: 311| 992| simdutf_really_inline void store_ascii_as_utf32(char32_t *ptr) const { 312| 992| this->chunks[0].store_ascii_as_utf32(ptr + sizeof(simd8) * 0); 313| 992| this->chunks[1].store_ascii_as_utf32(ptr + sizeof(simd8) * 1); 314| 992| this->chunks[2].store_ascii_as_utf32(ptr + sizeof(simd8) * 2); 315| 992| this->chunks[3].store_ascii_as_utf32(ptr + sizeof(simd8) * 3); 316| 992| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IaEEE20store_ascii_as_utf32EPDi: 32| 3.96k| simdutf_really_inline void store_ascii_as_utf32(char32_t *p) const { 33| 3.96k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p), _mm_cvtepu8_epi32(*this)); 34| 3.96k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 4), 35| 3.96k| _mm_cvtepu8_epi32(_mm_srli_si128(*this, 4))); 36| 3.96k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 8), 37| 3.96k| _mm_cvtepu8_epi32(_mm_srli_si128(*this, 8))); 38| 3.96k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 12), 39| | _mm_cvtepu8_epi32(_mm_srli_si128(*this, 12))); 40| 3.96k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhE4zeroEv: 105| 40.9k| static simdutf_really_inline simd8 zero() { return _mm_setzero_si128(); } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IaEgtES4_: 173| 81.2k| simdutf_really_inline simd8 operator>(const simd8 other) const { 174| 81.2k| return _mm_cmpgt_epi8(*this, other); 175| 81.2k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd5simd8IaEC2Ea: 165| 70.1k| simdutf_really_inline simd8(int8_t _value) : simd8(splat(_value)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhEmIENS2_5simd8IhEE: 134| 104k| simdutf_really_inline simd8 &operator-=(const simd8 other) { 135| 104k| *this = *this - other; 136| 104k| return *static_cast *>(this); 137| 104k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhEmiENS2_5simd8IhEE: 131| 104k| simdutf_really_inline simd8 operator-(const simd8 other) const { 132| 104k| return _mm_sub_epi8(*this, other); 133| 104k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd5simd8IhE9sum_bytesEv: 245| 4.02k| simdutf_really_inline uint64_t sum_bytes() const { 246| 4.02k| const auto tmp = _mm_sad_epu8(value, _mm_setzero_si128()); 247| 4.02k| return _mm_extract_epi64(tmp, 0) + _mm_extract_epi64(tmp, 1); 248| 4.02k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE2gtEa: 333| 2.77k| simdutf_really_inline uint64_t gt(const T m) const { 334| 2.77k| const simd8 mask = simd8::splat(m); 335| 2.77k| return simd8x64(this->chunks[0] > mask, this->chunks[1] > mask, 336| 2.77k| this->chunks[2] > mask, this->chunks[3] > mask) 337| 2.77k| .to_bitmask(); 338| 2.77k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_6simd16IbEEEC2EDv2_x: 16| 30.2k| simdutf_really_inline base(const __m128i _value) : value(_value) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_6simd16IbEEEcvRKDv2_xEv: 18| 30.2k| simdutf_really_inline operator const __m128i &() const { return this->value; } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd13base8_numericIhE5splatEh: 102| 652k| static simdutf_really_inline simd8 splat(T _value) { 103| 652k| return _mm_set1_epi8(_value); 104| 652k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEcvRKDv2_xEv: 18| 4.27M| simdutf_really_inline operator const __m128i &() const { return this->value; } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdeqENS2_5simd8IhEES4_: 72| 95.7k| const simd8 rhs) { 73| 95.7k| return _mm_cmpeq_epi8(lhs, rhs); 74| 95.7k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IhEC2EPKh: 270| 16.6k| : chunks{simd8::load(ptr), 271| 16.6k| simd8::load(ptr + sizeof(simd8) / sizeof(T)), 272| 16.6k| simd8::load(ptr + 2 * sizeof(simd8) / sizeof(T)), 273| 16.6k| simd8::load(ptr + 3 * sizeof(simd8) / sizeof(T))} {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IhEEEanES5_: 45| 634k| simdutf_really_inline Child operator&(const Child other) const { 46| 634k| return _mm_and_si128(*this, other); 47| 634k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_6simd16ItEEEanES5_: 45| 143k| simdutf_really_inline Child operator&(const Child other) const { 46| 143k| return _mm_and_si128(*this, other); 47| 143k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE20store_ascii_as_utf16ILNS_10endiannessE0EEEvPDs: 300| 1.02k| simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { 301| 1.02k| this->chunks[0].template store_ascii_as_utf16(ptr + 302| 1.02k| sizeof(simd8) * 0); 303| 1.02k| this->chunks[1].template store_ascii_as_utf16(ptr + 304| 1.02k| sizeof(simd8) * 1); 305| 1.02k| this->chunks[2].template store_ascii_as_utf16(ptr + 306| 1.02k| sizeof(simd8) * 2); 307| 1.02k| this->chunks[3].template store_ascii_as_utf16(ptr + 308| 1.02k| sizeof(simd8) * 3); 309| 1.02k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IaEEE20store_ascii_as_utf16ILNS_10endiannessE0EEEvPDs: 20| 4.10k| simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { 21| 4.10k| __m128i first = _mm_cvtepu8_epi16(*this); 22| 4.10k| __m128i second = _mm_cvtepu8_epi16(_mm_srli_si128(*this, 8)); 23| 4.10k| if (big_endian) { ------------------ | Branch (23:9): [Folded, False: 4.10k] ------------------ 24| 0| const __m128i swap = 25| 0| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 26| 0| first = _mm_shuffle_epi8(first, swap); 27| 0| second = _mm_shuffle_epi8(second, swap); 28| 0| } 29| 4.10k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p), first); 30| 4.10k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 8), second); 31| 4.10k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd8simd8x64IaE20store_ascii_as_utf16ILNS_10endiannessE1EEEvPDs: 300| 1.02k| simdutf_really_inline void store_ascii_as_utf16(char16_t *ptr) const { 301| 1.02k| this->chunks[0].template store_ascii_as_utf16(ptr + 302| 1.02k| sizeof(simd8) * 0); 303| 1.02k| this->chunks[1].template store_ascii_as_utf16(ptr + 304| 1.02k| sizeof(simd8) * 1); 305| 1.02k| this->chunks[2].template store_ascii_as_utf16(ptr + 306| 1.02k| sizeof(simd8) * 2); 307| 1.02k| this->chunks[3].template store_ascii_as_utf16(ptr + 308| 1.02k| sizeof(simd8) * 3); 309| 1.02k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd4baseINS2_5simd8IaEEE20store_ascii_as_utf16ILNS_10endiannessE1EEEvPDs: 20| 4.09k| simdutf_really_inline void store_ascii_as_utf16(char16_t *p) const { 21| 4.09k| __m128i first = _mm_cvtepu8_epi16(*this); 22| 4.09k| __m128i second = _mm_cvtepu8_epi16(_mm_srli_si128(*this, 8)); 23| 4.09k| if (big_endian) { ------------------ | Branch (23:9): [True: 4.09k, Folded] ------------------ 24| 4.09k| const __m128i swap = 25| 4.09k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 26| 4.09k| first = _mm_shuffle_epi8(first, swap); 27| 4.09k| second = _mm_shuffle_epi8(second, swap); 28| 4.09k| } 29| 4.09k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p), first); 30| 4.09k| _mm_storeu_si128(reinterpret_cast<__m128i *>(p + 8), second); 31| 4.09k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd16ItEC2EDv2_x: 79| 729k| : base16_numeric(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd14base16_numericItEC2EDv2_x: 54| 729k| : base16(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6base16ItNS2_6simd16IbEEEC2EDv2_x: 8| 729k| : base>(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd16ItE4packERKS4_S6_: 130| 52.8k| const simd16 &v1) { 131| 52.8k| return _mm_packus_epi16(v0, v1); 132| 52.8k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd14base16_numericItE4loadEPKt: 47| 136k| static simdutf_really_inline simd16 load(const T values[8]) { 48| 136k| return _mm_loadu_si128(reinterpret_cast(values)); 49| 136k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd16ItE10swap_bytesEv: 121| 15.0k| simdutf_really_inline simd16 swap_bytes() const { 122| 15.0k| const __m128i swap = 123| 15.0k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 124| 15.0k| return _mm_shuffle_epi8(*this, swap); 125| 15.0k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd14base16_numericItE5splatEt: 41| 173k| static simdutf_really_inline simd16 splat(T _value) { 42| 173k| return _mm_set1_epi16(_value); 43| 173k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdeqENS2_6simd16ItEES4_: 11| 30.2k| const simd16 rhs) { 12| 30.2k| return _mm_cmpeq_epi16(lhs, rhs); 13| 30.2k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd16IbEC2EDv2_x: 28| 30.2k| simdutf_really_inline simd16(const __m128i _value) : base16(_value) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6base16IbNS2_6simd16IbEEEC2EDv2_x: 8| 30.2k| : base>(_value) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd16ItE3shrILi8EEES4_v: 116| 53.0k| template simdutf_really_inline simd16 shr() const { 117| 53.0k| return simd16(_mm_srli_epi16(*this, N)); 118| 53.0k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd16ItEC2EPKDs: 86| 105k| : simd16(load(reinterpret_cast(values))) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd16ItEC2Et: 82| 147k| simdutf_really_inline simd16(uint16_t _value) : simd16(splat(_value)) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd14base16_numericItE4zeroEv: 45| 26.2k| static simdutf_really_inline simd16 zero() { return _mm_setzero_si128(); } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd3minENS2_6simd16ItEES4_: 240| 60.5k|simd16 min(const simd16 a, simd16 b) { 241| 60.5k| return _mm_min_epu16(a.value, b.value); 242| 60.5k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd14base16_numericItEpLENS2_6simd16ItEE: 68| 90.8k| simdutf_really_inline simd16 &operator+=(const simd16 other) { 69| 90.8k| *this = *this + other; 70| 90.8k| return *static_cast *>(this); 71| 90.8k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd14base16_numericItEplENS2_6simd16ItEE: 65| 90.8k| simdutf_really_inline simd16 operator+(const simd16 other) const { 66| 90.8k| return _mm_add_epi16(*this, other); 67| 90.8k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd16ItE3sumEv: 134| 26.2k| simdutf_really_inline uint64_t sum() const { 135| 26.2k| const auto lo_u16 = _mm_and_si128(value, _mm_set1_epi32(0x0000ffff)); 136| 26.2k| const auto hi_u16 = _mm_srli_epi32(value, 16); 137| 26.2k| const auto sum_u32 = _mm_add_epi32(lo_u16, hi_u16); 138| | 139| 26.2k| const auto lo_u32 = _mm_and_si128(sum_u32, _mm_set1_epi64x(0xffffffff)); 140| 26.2k| const auto hi_u32 = _mm_srli_epi64(sum_u32, 32); 141| 26.2k| const auto sum_u64 = _mm_add_epi64(lo_u32, hi_u32); 142| | 143| 26.2k| return uint64_t(_mm_extract_epi64(sum_u64, 0)) + 144| | uint64_t(_mm_extract_epi64(sum_u64, 1)); 145| 26.2k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd32IjE5splatEj: 56| 30.9k| simdutf_really_inline static simd32 splat(uint32_t v) { 57| 30.9k| return _mm_set1_epi32(v); 58| 30.9k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd32IjEC2EDv2_x: 9| 550k| simdutf_really_inline simd32(const __m128i v) : value(v) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd32IjE4zeroEv: 52| 4.86k| simdutf_really_inline static simd32 zero() { 53| 4.86k| return _mm_setzero_si128(); 54| 4.86k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd32IjEC2IDiEEPKT_: 13| 119k| : value(_mm_loadu_si128(reinterpret_cast(ptr))) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd32IjE10swap_bytesEv: 22| 99.6k| simdutf_really_inline simd32 swap_bytes() const { 23| 99.6k| const __m128i shuffle = 24| 99.6k| _mm_setr_epi8(3, 2, 1, 0, 7, 6, 5, 4, 8, 9, 10, 11, 15, 14, 13, 12); 25| | 26| 99.6k| return _mm_shuffle_epi8(value, shuffle); 27| 99.6k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd3maxENS2_6simd32IjEES4_: 94| 199k| const simd32 b) { 95| 199k| return _mm_max_epu32(a.value, b.value); 96| 199k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdplENS2_6simd32IjEES4_: 119| 99.6k| const simd32 b) { 120| 99.6k| return _mm_add_epi32(a.value, b.value); 121| 99.6k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdgtENS2_6simd32IjEES4_: 143| 3.90k| const simd32 b) { 144| 3.90k| return !(b >= a); 145| 3.90k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdntENS2_6simd32IbEE: 138| 3.90k|simdutf_really_inline simd32 operator!(const simd32 v) { 139| 3.90k| return _mm_xor_si128(v.value, _mm_set1_epi8(-1)); 140| 3.90k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd32IbEC2EDv2_x: 69| 7.81k| simdutf_really_inline simd32(const __m128i v) : value(v) {} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdgeENS2_6simd32IjEES4_: 134| 3.90k| const simd32 b) { 135| 3.90k| return _mm_cmpeq_epi32(_mm_max_epu32(a.value, b.value), a.value); 136| 3.90k|} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd32IbE3anyEv: 71| 3.90k| simdutf_really_inline bool any() const { 72| 3.90k| return _mm_movemask_epi8(value) != 0; 73| 3.90k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simdanENS2_6simd32IjEES4_: 104| 58.0k| const simd32 b) { 105| 58.0k| return _mm_and_si128(a.value, b.value); 106| 58.0k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd3minENS2_6simd32IjEES4_: 89| 58.0k| const simd32 b) { 90| 58.0k| return _mm_min_epu32(a.value, b.value); 91| 58.0k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd32IjEpLES4_: 46| 58.0k| simdutf_really_inline simd32 &operator+=(const simd32 other) { 47| 58.0k| value = _mm_add_epi32(value, other.value); 48| 58.0k| return *this; 49| 58.0k| } simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd32IjE3sumEv: 15| 957| simdutf_really_inline uint64_t sum() const { 16| 957| return uint64_t(_mm_extract_epi32(value, 0)) + 17| 957| uint64_t(_mm_extract_epi32(value, 1)) + 18| 957| uint64_t(_mm_extract_epi32(value, 2)) + 19| | uint64_t(_mm_extract_epi32(value, 3)); 20| 957| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd64ImE4zeroEv: 26| 40.9k| simdutf_really_inline static simd64 zero() { 27| 40.9k| return _mm_setzero_si128(); 28| 40.9k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_14simd6simd64ImEC2EDv2_x: 9| 40.9k| simdutf_really_inline simd64(const __m128i v) : value(v) {} simdutf.cpp:_ZNK7simdutf8westmere12_GLOBAL__N_14simd6simd64ImE3sumEv: 15| 40.9k| simdutf_really_inline uint64_t sum() const { 16| 40.9k| return _mm_extract_epi64(value, 0) + _mm_extract_epi64(value, 1); 17| 40.9k| } _ZNK7simdutf8westmere14implementation13validate_utf8EPKcm: 299| 27.2k|implementation::validate_utf8(const char *buf, size_t len) const noexcept { 300| 27.2k| return westmere::utf8_validation::generic_validate_utf8(buf, len); 301| 27.2k|} _ZNK7simdutf8westmere14implementation25validate_utf8_with_errorsEPKcm: 306| 6.85k| const char *buf, size_t len) const noexcept { 307| 6.85k| return westmere::utf8_validation::generic_validate_utf8_with_errors(buf, len); 308| 6.85k|} _ZNK7simdutf8westmere14implementation16validate_utf16leEPKDsm: 345| 19.9k| size_t len) const noexcept { 346| 19.9k| if (simdutf_unlikely(len == 0)) { ------------------ | | 93| 19.9k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 17.6k, False: 2.33k] | | ------------------ ------------------ 347| | // empty input is valid UTF-16. protect the implementation from 348| | // handling nullptr 349| 17.6k| return true; 350| 17.6k| } 351| 2.33k| const auto res = 352| 2.33k| westmere::utf16::validate_utf16_with_errors(buf, len); 353| 2.33k| if (res.is_err()) { ------------------ | Branch (353:7): [True: 59, False: 2.27k] ------------------ 354| 59| return false; 355| 59| } 356| | 357| 2.27k| if (res.count == len) ------------------ | Branch (357:7): [True: 0, False: 2.27k] ------------------ 358| 0| return true; 359| | 360| 2.27k| return scalar::utf16::validate(buf + res.count, 361| 2.27k| len - res.count); 362| 2.27k|} _ZNK7simdutf8westmere14implementation16validate_utf16beEPKDsm: 368| 19.9k| size_t len) const noexcept { 369| 19.9k| if (simdutf_unlikely(len == 0)) { ------------------ | | 93| 19.9k| #define simdutf_unlikely(x) __builtin_expect(!!(x), 0) | | ------------------ | | | Branch (93:33): [True: 17.6k, False: 2.28k] | | ------------------ ------------------ 370| | // empty input is valid UTF-16. protect the implementation from 371| | // handling nullptr 372| 17.6k| return true; 373| 17.6k| } 374| 2.28k| const auto res = 375| 2.28k| westmere::utf16::validate_utf16_with_errors(buf, len); 376| 2.28k| if (res.is_err()) { ------------------ | Branch (376:7): [True: 61, False: 2.22k] ------------------ 377| 61| return false; 378| 61| } 379| | 380| 2.22k| if (res.count == len) ------------------ | Branch (380:7): [True: 0, False: 2.22k] ------------------ 381| 0| return true; 382| | 383| 2.22k| return scalar::utf16::validate(buf + res.count, 384| 2.22k| len - res.count); 385| 2.22k|} _ZNK7simdutf8westmere14implementation28validate_utf16le_with_errorsEPKDsm: 388| 6.85k| const char16_t *buf, size_t len) const noexcept { 389| 6.85k| const result res = 390| 6.85k| westmere::utf16::validate_utf16_with_errors(buf, len); 391| 6.85k| if (res.count != len) { ------------------ | Branch (391:7): [True: 385, False: 6.46k] ------------------ 392| 385| const result scalar_res = 393| 385| scalar::utf16::validate_with_errors( 394| 385| buf + res.count, len - res.count); 395| 385| return result(scalar_res.error, res.count + scalar_res.count); 396| 6.46k| } else { 397| 6.46k| return res; 398| 6.46k| } 399| 6.85k|} _ZNK7simdutf8westmere14implementation28validate_utf16be_with_errorsEPKDsm: 402| 6.85k| const char16_t *buf, size_t len) const noexcept { 403| 6.85k| const result res = 404| 6.85k| westmere::utf16::validate_utf16_with_errors(buf, len); 405| 6.85k| if (res.count != len) { ------------------ | Branch (405:7): [True: 333, False: 6.52k] ------------------ 406| 333| result scalar_res = scalar::utf16::validate_with_errors( 407| 333| buf + res.count, len - res.count); 408| 333| return result(scalar_res.error, res.count + scalar_res.count); 409| 6.52k| } else { 410| 6.52k| return res; 411| 6.52k| } 412| 6.85k|} _ZNK7simdutf8westmere14implementation14validate_utf32EPKDim: 427| 13.1k|implementation::validate_utf32(const char32_t *buf, size_t len) const noexcept { 428| 13.1k| return utf32::validate(buf, len); 429| 13.1k|} _ZNK7simdutf8westmere14implementation22convert_latin1_to_utf8EPKcmPc: 441| 7.19k| const char *buf, size_t len, char *utf8_output) const noexcept { 442| | 443| 7.19k| std::pair ret = 444| 7.19k| sse_convert_latin1_to_utf8(buf, len, utf8_output); 445| 7.19k| size_t converted_chars = ret.second - utf8_output; 446| | 447| 7.19k| if (ret.first != buf + len) { ------------------ | Branch (447:7): [True: 1.13k, False: 6.05k] ------------------ 448| 1.13k| const size_t scalar_converted_chars = scalar::latin1_to_utf8::convert( 449| 1.13k| ret.first, len - (ret.first - buf), ret.second); 450| 1.13k| converted_chars += scalar_converted_chars; 451| 1.13k| } 452| | 453| 7.19k| return converted_chars; 454| 7.19k|} _ZNK7simdutf8westmere14implementation25convert_latin1_to_utf16leEPKcmPDs: 459| 6.85k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 460| 6.85k| std::pair ret = 461| 6.85k| sse_convert_latin1_to_utf16(buf, len, utf16_output); 462| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (462:7): [True: 0, False: 6.85k] ------------------ 463| 0| return 0; 464| 0| } 465| 6.85k| size_t converted_chars = ret.second - utf16_output; 466| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (466:7): [True: 561, False: 6.29k] ------------------ 467| 561| const size_t scalar_converted_chars = 468| 561| scalar::latin1_to_utf16::convert( 469| 561| ret.first, len - (ret.first - buf), ret.second); 470| 561| if (scalar_converted_chars == 0) { ------------------ | Branch (470:9): [True: 0, False: 561] ------------------ 471| 0| return 0; 472| 0| } 473| 561| converted_chars += scalar_converted_chars; 474| 561| } 475| 6.85k| return converted_chars; 476| 6.85k|} _ZNK7simdutf8westmere14implementation25convert_latin1_to_utf16beEPKcmPDs: 479| 6.85k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 480| 6.85k| std::pair ret = 481| 6.85k| sse_convert_latin1_to_utf16(buf, len, utf16_output); 482| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (482:7): [True: 0, False: 6.85k] ------------------ 483| 0| return 0; 484| 0| } 485| 6.85k| size_t converted_chars = ret.second - utf16_output; 486| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (486:7): [True: 561, False: 6.29k] ------------------ 487| 561| const size_t scalar_converted_chars = 488| 561| scalar::latin1_to_utf16::convert( 489| 561| ret.first, len - (ret.first - buf), ret.second); 490| 561| if (scalar_converted_chars == 0) { ------------------ | Branch (490:9): [True: 0, False: 561] ------------------ 491| 0| return 0; 492| 0| } 493| 561| converted_chars += scalar_converted_chars; 494| 561| } 495| 6.85k| return converted_chars; 496| 6.85k|} _ZNK7simdutf8westmere14implementation23convert_latin1_to_utf32EPKcmPDi: 501| 6.85k| const char *buf, size_t len, char32_t *utf32_output) const noexcept { 502| 6.85k| std::pair ret = 503| 6.85k| sse_convert_latin1_to_utf32(buf, len, utf32_output); 504| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (504:7): [True: 0, False: 6.85k] ------------------ 505| 0| return 0; 506| 0| } 507| 6.85k| size_t converted_chars = ret.second - utf32_output; 508| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (508:7): [True: 561, False: 6.29k] ------------------ 509| 561| const size_t scalar_converted_chars = scalar::latin1_to_utf32::convert( 510| 561| ret.first, len - (ret.first - buf), ret.second); 511| 561| if (scalar_converted_chars == 0) { ------------------ | Branch (511:9): [True: 0, False: 561] ------------------ 512| 0| return 0; 513| 0| } 514| 561| converted_chars += scalar_converted_chars; 515| 561| } 516| 6.85k| return converted_chars; 517| 6.85k|} _ZNK7simdutf8westmere14implementation22convert_utf8_to_latin1EPKcmPc: 522| 13.7k| const char *buf, size_t len, char *latin1_output) const noexcept { 523| 13.7k| utf8_to_latin1::validating_transcoder converter; 524| 13.7k| return converter.convert(buf, len, latin1_output); 525| 13.7k|} _ZNK7simdutf8westmere14implementation23convert_utf8_to_utf16leEPKcmPDs: 541| 13.5k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 542| 13.5k| utf8_to_utf16::validating_transcoder converter; 543| 13.5k| return converter.convert(buf, len, utf16_output); 544| 13.5k|} _ZNK7simdutf8westmere14implementation23convert_utf8_to_utf16beEPKcmPDs: 547| 13.6k| const char *buf, size_t len, char16_t *utf16_output) const noexcept { 548| 13.6k| utf8_to_utf16::validating_transcoder converter; 549| 13.6k| return converter.convert(buf, len, utf16_output); 550| 13.6k|} _ZNK7simdutf8westmere14implementation21convert_utf8_to_utf32EPKcmPDi: 580| 6.85k| const char *buf, size_t len, char32_t *utf32_output) const noexcept { 581| 6.85k| utf8_to_utf32::validating_transcoder converter; 582| 6.85k| return converter.convert(buf, len, utf32_output); 583| 6.85k|} _ZNK7simdutf8westmere14implementation25convert_utf16le_to_latin1EPKDsmPc: 599| 6.85k| const char16_t *buf, size_t len, char *latin1_output) const noexcept { 600| 6.85k| std::pair ret = 601| 6.85k| sse_convert_utf16_to_latin1(buf, len, latin1_output); 602| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (602:7): [True: 0, False: 6.85k] ------------------ 603| 0| return 0; 604| 0| } 605| 6.85k| size_t saved_bytes = ret.second - latin1_output; 606| | 607| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (607:7): [True: 522, False: 6.33k] ------------------ 608| 522| const size_t scalar_saved_bytes = 609| 522| scalar::utf16_to_latin1::convert( 610| 522| ret.first, len - (ret.first - buf), ret.second); 611| 522| if (scalar_saved_bytes == 0) { ------------------ | Branch (611:9): [True: 0, False: 522] ------------------ 612| 0| return 0; 613| 0| } 614| 522| saved_bytes += scalar_saved_bytes; 615| 522| } 616| 6.85k| return saved_bytes; 617| 6.85k|} _ZNK7simdutf8westmere14implementation25convert_utf16be_to_latin1EPKDsmPc: 620| 6.85k| const char16_t *buf, size_t len, char *latin1_output) const noexcept { 621| 6.85k| std::pair ret = 622| 6.85k| sse_convert_utf16_to_latin1(buf, len, latin1_output); 623| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (623:7): [True: 0, False: 6.85k] ------------------ 624| 0| return 0; 625| 0| } 626| 6.85k| size_t saved_bytes = ret.second - latin1_output; 627| | 628| 6.85k| if (ret.first != buf + len) { ------------------ | Branch (628:7): [True: 522, False: 6.33k] ------------------ 629| 522| const size_t scalar_saved_bytes = 630| 522| scalar::utf16_to_latin1::convert( 631| 522| ret.first, len - (ret.first - buf), ret.second); 632| 522| if (scalar_saved_bytes == 0) { ------------------ | Branch (632:9): [True: 0, False: 522] ------------------ 633| 0| return 0; 634| 0| } 635| 522| saved_bytes += scalar_saved_bytes; 636| 522| } 637| 6.85k| return saved_bytes; 638| 6.85k|} _ZNK7simdutf8westmere14implementation23convert_utf16le_to_utf8EPKDsmPc: 709| 13.1k| const char16_t *buf, size_t len, char *utf8_output) const noexcept { 710| 13.1k| std::pair ret = 711| 13.1k| sse_convert_utf16_to_utf8(buf, len, utf8_output); 712| 13.1k| if (ret.first == nullptr) { ------------------ | Branch (712:7): [True: 64, False: 13.0k] ------------------ 713| 64| return 0; 714| 64| } 715| 13.0k| size_t saved_bytes = ret.second - utf8_output; 716| 13.0k| if (ret.first != buf + len) { ------------------ | Branch (716:7): [True: 1.08k, False: 11.9k] ------------------ 717| 1.08k| const size_t scalar_saved_bytes = 718| 1.08k| scalar::utf16_to_utf8::convert( 719| 1.08k| ret.first, len - (ret.first - buf), ret.second); 720| 1.08k| if (scalar_saved_bytes == 0) { ------------------ | Branch (720:9): [True: 45, False: 1.04k] ------------------ 721| 45| return 0; 722| 45| } 723| 1.04k| saved_bytes += scalar_saved_bytes; 724| 1.04k| } 725| 13.0k| return saved_bytes; 726| 13.0k|} _ZNK7simdutf8westmere14implementation23convert_utf16be_to_utf8EPKDsmPc: 729| 13.1k| const char16_t *buf, size_t len, char *utf8_output) const noexcept { 730| 13.1k| std::pair ret = 731| 13.1k| sse_convert_utf16_to_utf8(buf, len, utf8_output); 732| 13.1k| if (ret.first == nullptr) { ------------------ | Branch (732:7): [True: 67, False: 13.0k] ------------------ 733| 67| return 0; 734| 67| } 735| 13.0k| size_t saved_bytes = ret.second - utf8_output; 736| 13.0k| if (ret.first != buf + len) { ------------------ | Branch (736:7): [True: 1.03k, False: 12.0k] ------------------ 737| 1.03k| const size_t scalar_saved_bytes = 738| 1.03k| scalar::utf16_to_utf8::convert( 739| 1.03k| ret.first, len - (ret.first - buf), ret.second); 740| 1.03k| if (scalar_saved_bytes == 0) { ------------------ | Branch (740:9): [True: 35, False: 995] ------------------ 741| 35| return 0; 742| 35| } 743| 995| saved_bytes += scalar_saved_bytes; 744| 995| } 745| 13.0k| return saved_bytes; 746| 13.0k|} _ZNK7simdutf8westmere14implementation23convert_utf32_to_latin1EPKDimPc: 794| 6.85k| const char32_t *buf, size_t len, char *latin1_output) const noexcept { 795| 6.85k| std::pair ret = 796| 6.85k| sse_convert_utf32_to_latin1(buf, len, latin1_output); 797| 6.85k| if (ret.first == nullptr) { ------------------ | Branch (797:7): [True: 0, False: 6.85k] ------------------ 798| 0| return 0; 799| 0| } 800| 6.85k| size_t saved_bytes = ret.second - latin1_output; 801| | // if (ret.first != buf + len) { 802| 6.85k| if (ret.first < buf + len) { ------------------ | Branch (802:7): [True: 561, False: 6.29k] ------------------ 803| 561| const size_t scalar_saved_bytes = scalar::utf32_to_latin1::convert( 804| 561| ret.first, len - (ret.first - buf), ret.second); 805| 561| if (scalar_saved_bytes == 0) { ------------------ | Branch (805:9): [True: 0, False: 561] ------------------ 806| 0| return 0; 807| 0| } 808| 561| saved_bytes += scalar_saved_bytes; 809| 561| } 810| 6.85k| return saved_bytes; 811| 6.85k|} _ZNK7simdutf8westmere14implementation21convert_utf32_to_utf8EPKDimPc: 845| 6.26k| const char32_t *buf, size_t len, char *utf8_output) const noexcept { 846| 6.26k| std::pair ret = 847| 6.26k| sse_convert_utf32_to_utf8(buf, len, utf8_output); 848| 6.26k| if (ret.first == nullptr) { ------------------ | Branch (848:7): [True: 0, False: 6.26k] ------------------ 849| 0| return 0; 850| 0| } 851| 6.26k| size_t saved_bytes = ret.second - utf8_output; 852| 6.26k| if (ret.first != buf + len) { ------------------ | Branch (852:7): [True: 764, False: 5.50k] ------------------ 853| 764| const size_t scalar_saved_bytes = scalar::utf32_to_utf8::convert( 854| 764| ret.first, len - (ret.first - buf), ret.second); 855| 764| if (scalar_saved_bytes == 0) { ------------------ | Branch (855:9): [True: 0, False: 764] ------------------ 856| 0| return 0; 857| 0| } 858| 764| saved_bytes += scalar_saved_bytes; 859| 764| } 860| 6.26k| return saved_bytes; 861| 6.26k|} _ZNK7simdutf8westmere14implementation10count_utf8EPKcm: 1121| 13.7k|implementation::count_utf8(const char *input, size_t length) const noexcept { 1122| 13.7k| return utf8::count_code_points_bytemask(input, length); 1123| 13.7k|} _ZNK7simdutf8westmere14implementation23latin1_length_from_utf8EPKcm: 1133| 13.7k| const char *buf, size_t len) const noexcept { 1134| 13.7k| return count_utf8(buf, len); 1135| 13.7k|} _ZNK7simdutf8westmere14implementation24utf8_length_from_utf16leEPKDsm: 1140| 13.1k| const char16_t *input, size_t length) const noexcept { 1141| 13.1k| return utf16::utf8_length_from_utf16_bytemask(input, 1142| 13.1k| length); 1143| 13.1k|} _ZNK7simdutf8westmere14implementation24utf8_length_from_utf16beEPKDsm: 1146| 13.1k| const char16_t *input, size_t length) const noexcept { 1147| 13.1k| return utf16::utf8_length_from_utf16_bytemask(input, length); 1148| 13.1k|} _ZNK7simdutf8westmere14implementation23utf8_length_from_latin1EPKcm: 1153| 7.19k| const char *input, size_t len) const noexcept { 1154| 7.19k| const uint8_t *str = reinterpret_cast(input); 1155| 7.19k| size_t answer = len / sizeof(__m128i) * sizeof(__m128i); 1156| 7.19k| size_t i = 0; 1157| 7.19k| if (answer >= 2048) { // long strings optimization ------------------ | Branch (1157:7): [True: 0, False: 7.19k] ------------------ 1158| 0| __m128i two_64bits = _mm_setzero_si128(); 1159| 0| while (i + sizeof(__m128i) <= len) { ------------------ | Branch (1159:12): [True: 0, False: 0] ------------------ 1160| 0| __m128i runner = _mm_setzero_si128(); 1161| 0| size_t iterations = (len - i) / sizeof(__m128i); 1162| 0| if (iterations > 255) { ------------------ | Branch (1162:11): [True: 0, False: 0] ------------------ 1163| 0| iterations = 255; 1164| 0| } 1165| 0| size_t max_i = i + iterations * sizeof(__m128i) - sizeof(__m128i); 1166| 0| for (; i + 4 * sizeof(__m128i) <= max_i; i += 4 * sizeof(__m128i)) { ------------------ | Branch (1166:14): [True: 0, False: 0] ------------------ 1167| 0| __m128i input1 = _mm_loadu_si128((const __m128i *)(str + i)); 1168| 0| __m128i input2 = 1169| 0| _mm_loadu_si128((const __m128i *)(str + i + sizeof(__m128i))); 1170| 0| __m128i input3 = 1171| 0| _mm_loadu_si128((const __m128i *)(str + i + 2 * sizeof(__m128i))); 1172| 0| __m128i input4 = 1173| 0| _mm_loadu_si128((const __m128i *)(str + i + 3 * sizeof(__m128i))); 1174| 0| __m128i input12 = 1175| 0| _mm_add_epi8(_mm_cmpgt_epi8(_mm_setzero_si128(), input1), 1176| 0| _mm_cmpgt_epi8(_mm_setzero_si128(), input2)); 1177| 0| __m128i input34 = 1178| 0| _mm_add_epi8(_mm_cmpgt_epi8(_mm_setzero_si128(), input3), 1179| 0| _mm_cmpgt_epi8(_mm_setzero_si128(), input4)); 1180| 0| __m128i input1234 = _mm_add_epi8(input12, input34); 1181| 0| runner = _mm_sub_epi8(runner, input1234); 1182| 0| } 1183| 0| for (; i <= max_i; i += sizeof(__m128i)) { ------------------ | Branch (1183:14): [True: 0, False: 0] ------------------ 1184| 0| __m128i more_input = _mm_loadu_si128((const __m128i *)(str + i)); 1185| 0| runner = _mm_sub_epi8(runner, 1186| 0| _mm_cmpgt_epi8(_mm_setzero_si128(), more_input)); 1187| 0| } 1188| 0| two_64bits = 1189| 0| _mm_add_epi64(two_64bits, _mm_sad_epu8(runner, _mm_setzero_si128())); 1190| 0| } 1191| 0| answer += 1192| 0| _mm_extract_epi64(two_64bits, 0) + _mm_extract_epi64(two_64bits, 1); 1193| 7.19k| } else if (answer > 0) { // short string optimization ------------------ | Branch (1193:14): [True: 1.25k, False: 5.94k] ------------------ 1194| 12.5k| for (; i + 2 * sizeof(__m128i) <= len; i += 2 * sizeof(__m128i)) { ------------------ | Branch (1194:12): [True: 11.3k, False: 1.25k] ------------------ 1195| 11.3k| __m128i latin = _mm_loadu_si128((const __m128i *)(input + i)); 1196| 11.3k| uint16_t non_ascii = (uint16_t)_mm_movemask_epi8(latin); 1197| 11.3k| answer += count_ones(non_ascii); 1198| 11.3k| latin = _mm_loadu_si128((const __m128i *)(input + i) + 1); 1199| 11.3k| non_ascii = (uint16_t)_mm_movemask_epi8(latin); 1200| 11.3k| answer += count_ones(non_ascii); 1201| 11.3k| } 1202| 1.53k| for (; i + sizeof(__m128i) <= len; i += sizeof(__m128i)) { ------------------ | Branch (1202:12): [True: 281, False: 1.25k] ------------------ 1203| 281| __m128i latin = _mm_loadu_si128((const __m128i *)(input + i)); 1204| 281| uint16_t non_ascii = (uint16_t)_mm_movemask_epi8(latin); 1205| 281| answer += count_ones(non_ascii); 1206| 281| } 1207| 1.25k| } 1208| 7.19k| return answer + scalar::latin1::utf8_length_from_latin1( 1209| 7.19k| reinterpret_cast(str + i), len - i); 1210| 7.19k|} _ZNK7simdutf8westmere14implementation22utf16_length_from_utf8EPKcm: 1227| 27.2k| const char *input, size_t length) const noexcept { 1228| 27.2k| return utf8::utf16_length_from_utf8_bytemask(input, length); 1229| 27.2k|} _ZNK7simdutf8westmere14implementation22utf8_length_from_utf32EPKDim: 1268| 6.26k| const char32_t *input, size_t length) const noexcept { 1269| 6.26k| return utf32::utf8_length_from_utf32(input, length); 1270| 6.26k|} _ZNK7simdutf8westmere14implementation22utf32_length_from_utf8EPKcm: 1296| 6.85k| const char *input, size_t length) const noexcept { 1297| 6.85k| return utf8::count_code_points(input, length); 1298| 6.85k|} _ZNK7simdutf8westmere14implementation16base64_to_binaryEPKcmPcNS_14base64_optionsENS_27last_chunk_handling_optionsE: 1304| 19.6k| last_chunk_handling_options last_chunk_options) const noexcept { 1305| 19.6k| if (options & base64_default_or_url) { ------------------ | Branch (1305:7): [True: 0, False: 19.6k] ------------------ 1306| 0| if (options == base64_options::base64_default_or_url_accept_garbage) { ------------------ | Branch (1306:9): [True: 0, False: 0] ------------------ 1307| 0| return base64::compress_decode_base64( 1308| 0| output, input, length, options, last_chunk_options); 1309| 0| } else { 1310| 0| return base64::compress_decode_base64( 1311| 0| output, input, length, options, last_chunk_options); 1312| 0| } 1313| 19.6k| } else if (options & base64_url) { ------------------ | Branch (1313:14): [True: 0, False: 19.6k] ------------------ 1314| 0| if (options == base64_options::base64_url_accept_garbage) { ------------------ | Branch (1314:9): [True: 0, False: 0] ------------------ 1315| 0| return base64::compress_decode_base64( 1316| 0| output, input, length, options, last_chunk_options); 1317| 0| } else { 1318| 0| return base64::compress_decode_base64( 1319| 0| output, input, length, options, last_chunk_options); 1320| 0| } 1321| 19.6k| } else { 1322| 19.6k| if (options == base64_options::base64_default_accept_garbage) { ------------------ | Branch (1322:9): [True: 0, False: 19.6k] ------------------ 1323| 0| return base64::compress_decode_base64( 1324| 0| output, input, length, options, last_chunk_options); 1325| 19.6k| } else { 1326| 19.6k| return base64::compress_decode_base64( 1327| 19.6k| output, input, length, options, last_chunk_options); 1328| 19.6k| } 1329| 19.6k| } 1330| 19.6k|} _ZNK7simdutf8westmere14implementation16binary_to_base64EPKcmPcNS_14base64_optionsE: 1424| 18.7k| base64_options options) const noexcept { 1425| 18.7k| if (options & base64_url) { ------------------ | Branch (1425:7): [True: 0, False: 18.7k] ------------------ 1426| 0| return encode_base64(output, input, length, options); 1427| 18.7k| } else { 1428| 18.7k| return encode_base64(output, input, length, options); 1429| 18.7k| } 1430| 18.7k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_18is_asciiERKNS1_4simd8simd8x64IhEE: 13| 16.0k|simdutf_really_inline bool is_ascii(const simd8x64 &input) { 14| 16.0k| return input.reduce_or().is_ascii(); 15| 16.0k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_124must_be_2_3_continuationENS1_4simd5simd8IhEES4_: 22| 77.8k| const simd8 prev3) { 23| 77.8k| simd8 is_third_byte = 24| 77.8k| prev2.saturating_sub(0xe0u - 0x80); // Only 111_____ will be >= 0x80 25| 77.8k| simd8 is_fourth_byte = 26| 77.8k| prev3.saturating_sub(0xf0u - 0x80); // Only 1111____ will be >= 0x80 27| 77.8k| return simd8(is_third_byte | is_fourth_byte); 28| 77.8k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_18internal8westmere26write_v_u16_11bits_to_utf8EDv2_xRPcS4_S4_: 57| 18.0k| const __m128i v_ff80) { 58| | // no bits set above 7th bit 59| 18.0k| const __m128i one_byte_bytemask = 60| 18.0k| _mm_cmpeq_epi16(_mm_and_si128(v_u16, v_ff80), v_0000); 61| 18.0k| const uint16_t one_byte_bitmask = 62| 18.0k| static_cast(_mm_movemask_epi8(one_byte_bytemask)); 63| | 64| 18.0k| write_v_u16_11bits_to_utf8(v_u16, utf8_output, one_byte_bytemask, 65| 18.0k| one_byte_bitmask); 66| 18.0k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_18internal8westmere26write_v_u16_11bits_to_utf8EDv2_xRPcS4_t: 10| 21.3k| const uint16_t one_byte_bitmask) { 11| | // 0b1100_0000_1000_0000 12| 21.3k| const __m128i v_c080 = _mm_set1_epi16((int16_t)0xc080); 13| | // 0b0001_1111_0000_0000 14| 21.3k| const __m128i v_1f00 = _mm_set1_epi16((int16_t)0x1f00); 15| | // 0b0000_0000_0011_1111 16| 21.3k| const __m128i v_003f = _mm_set1_epi16((int16_t)0x003f); 17| | 18| | // 1. prepare 2-byte values 19| | // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 20| | // expected output : [110a|aaaa|10bb|bbbb] x 8 21| | 22| | // t0 = [000a|aaaa|bbbb|bb00] 23| 21.3k| const __m128i t0 = _mm_slli_epi16(v_u16, 2); 24| | // t1 = [000a|aaaa|0000|0000] 25| 21.3k| const __m128i t1 = _mm_and_si128(t0, v_1f00); 26| | // t2 = [0000|0000|00bb|bbbb] 27| 21.3k| const __m128i t2 = _mm_and_si128(v_u16, v_003f); 28| | // t3 = [000a|aaaa|00bb|bbbb] 29| 21.3k| const __m128i t3 = _mm_or_si128(t1, t2); 30| | // t4 = [110a|aaaa|10bb|bbbb] 31| 21.3k| const __m128i t4 = _mm_or_si128(t3, v_c080); 32| | 33| | // 2. merge ASCII and 2-byte codewords 34| 21.3k| const __m128i utf8_unpacked = _mm_blendv_epi8(t4, v_u16, one_byte_bytemask); 35| | 36| | // 3. prepare bitmask for 8-bit lookup 37| | // one_byte_bitmask = hhggffeeddccbbaa -- the bits are doubled (h - MSB, a 38| | // - LSB) 39| 21.3k| const uint16_t m0 = one_byte_bitmask & 0x5555; // m0 = 0h0g0f0e0d0c0b0a 40| 21.3k| const uint16_t m1 = static_cast(m0 >> 7); // m1 = 00000000h0g0f0e0 41| 21.3k| const uint8_t m2 = static_cast((m0 | m1) & 0xff); // m2 = hdgcfbea 42| | // 4. pack the bytes 43| 21.3k| const uint8_t *row = 44| 21.3k| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[m2][0]; 45| 21.3k| const __m128i shuffle = _mm_loadu_si128((__m128i *)(row + 1)); 46| 21.3k| const __m128i utf8_packed = _mm_shuffle_epi8(utf8_unpacked, shuffle); 47| | 48| | // 5. store bytes 49| 21.3k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 50| | 51| | // 6. adjust pointers 52| 21.3k| utf8_output += row[0]; 53| 21.3k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block64C2EPKc: 408| 5.20k| simdutf_really_inline block64(const char *src) { 409| 5.20k| chunks[0] = _mm_loadu_si128(reinterpret_cast(src)); 410| 5.20k| chunks[1] = _mm_loadu_si128(reinterpret_cast(src + 16)); 411| 5.20k| chunks[2] = _mm_loadu_si128(reinterpret_cast(src + 32)); 412| 5.20k| chunks[3] = _mm_loadu_si128(reinterpret_cast(src + 48)); 413| 5.20k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6419base64_decode_blockILb0EEEvPc: 665| 1.16k| simdutf_really_inline void base64_decode_block(char *out) { 666| 1.16k| base64_decode(out, chunks[0]); 667| 1.16k| base64_decode(out + 12, chunks[1]); 668| 1.16k| base64_decode(out + 24, chunks[2]); 669| 1.16k| base64_decode(out + 36, chunks[3]); 670| 1.16k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113base64_decodeILb0EEEvPcDv2_x: 369| 12.2k|static simdutf_really_inline void base64_decode(char *out, __m128i str) { 370| | // credit: aqrit 371| | 372| 12.2k| const __m128i pack_shuffle = 373| 12.2k| _mm_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1); 374| | 375| 12.2k| const __m128i t0 = _mm_maddubs_epi16(str, _mm_set1_epi32(0x01400140)); 376| 12.2k| const __m128i t1 = _mm_madd_epi16(t0, _mm_set1_epi32(0x00011000)); 377| 12.2k| const __m128i t2 = _mm_shuffle_epi8(t1, pack_shuffle); 378| | if constexpr (exact) { 379| | _mm_storel_epi64((__m128i *)out, t2); 380| | const int32_t last = _mm_extract_epi32(t2, 2); 381| | std::memcpy(out + 8, &last, sizeof(last)); 382| 12.2k| } else { 383| | // this writes 16 bytes, but we only need 12. 384| 12.2k| _mm_storeu_si128((__m128i *)out, t2); 385| 12.2k| } 386| 12.2k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6419base64_decode_blockILb1EEEvPc: 665| 1.83k| simdutf_really_inline void base64_decode_block(char *out) { 666| 1.83k| base64_decode(out, chunks[0]); 667| 1.83k| base64_decode(out + 12, chunks[1]); 668| 1.83k| base64_decode(out + 24, chunks[2]); 669| 1.83k| base64_decode(out + 36, chunks[3]); 670| 1.83k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113base64_decodeILb1EEEvPcDv2_x: 369| 1.93k|static simdutf_really_inline void base64_decode(char *out, __m128i str) { 370| | // credit: aqrit 371| | 372| 1.93k| const __m128i pack_shuffle = 373| 1.93k| _mm_setr_epi8(2, 1, 0, 6, 5, 4, 10, 9, 8, 14, 13, 12, -1, -1, -1, -1); 374| | 375| 1.93k| const __m128i t0 = _mm_maddubs_epi16(str, _mm_set1_epi32(0x01400140)); 376| 1.93k| const __m128i t1 = _mm_madd_epi16(t0, _mm_set1_epi32(0x00011000)); 377| 1.93k| const __m128i t2 = _mm_shuffle_epi8(t1, pack_shuffle); 378| 1.93k| if constexpr (exact) { 379| 1.93k| _mm_storel_epi64((__m128i *)out, t2); 380| 1.93k| const int32_t last = _mm_extract_epi32(t2, 2); 381| 1.93k| std::memcpy(out + 8, &last, sizeof(last)); 382| | } else { 383| | // this writes 16 bytes, but we only need 12. 384| | _mm_storeu_si128((__m128i *)out, t2); 385| | } 386| 1.93k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6414compress_blockEmPc: 462| 981| simdutf_really_inline uint64_t compress_block(uint64_t mask, char *output) { 463| 981| if (is_power_of_two(mask)) { ------------------ | Branch (463:9): [True: 318, False: 663] ------------------ 464| 318| return compress_block_single(mask, output); 465| 318| } 466| | 467| 663| uint64_t nmask = ~mask; 468| 663| compress(chunks[0], uint16_t(mask), output); 469| 663| compress(chunks[1], uint16_t(mask >> 16), 470| 663| output + count_ones(nmask & 0xFFFF)); 471| 663| compress(chunks[2], uint16_t(mask >> 32), 472| 663| output + count_ones(nmask & 0xFFFFFFFF)); 473| 663| compress(chunks[3], uint16_t(mask >> 48), 474| 663| output + count_ones(nmask & 0xFFFFFFFFFFFFULL)); 475| 663| return count_ones(nmask); 476| 981| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6421compress_block_singleEmPc: 480| 318| char *output) { 481| 318| const size_t pos64 = trailing_zeroes(mask); 482| 318| const int8_t pos = pos64 & 0xf; 483| 318| switch (pos64 >> 4) { ------------------ | Branch (483:13): [True: 318, False: 0] ------------------ 484| 70| case 0b00: { ------------------ | Branch (484:5): [True: 70, False: 248] ------------------ 485| 70| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 486| 70| const __m128i v1 = 487| 70| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 488| 70| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 489| 70| const __m128i sh = _mm_sub_epi8(v1, v2); 490| 70| const __m128i compressed = _mm_shuffle_epi8(chunks[0], sh); 491| | 492| 70| _mm_storeu_si128((__m128i *)(output + 0 * 16), compressed); 493| 70| _mm_storeu_si128((__m128i *)(output + 1 * 16 - 1), chunks[1]); 494| 70| _mm_storeu_si128((__m128i *)(output + 2 * 16 - 1), chunks[2]); 495| 70| _mm_storeu_si128((__m128i *)(output + 3 * 16 - 1), chunks[3]); 496| 70| } break; 497| 67| case 0b01: { ------------------ | Branch (497:5): [True: 67, False: 251] ------------------ 498| 67| _mm_storeu_si128((__m128i *)(output + 0 * 16), chunks[0]); 499| | 500| 67| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 501| 67| const __m128i v1 = 502| 67| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 503| 67| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 504| 67| const __m128i sh = _mm_sub_epi8(v1, v2); 505| 67| const __m128i compressed = _mm_shuffle_epi8(chunks[1], sh); 506| | 507| 67| _mm_storeu_si128((__m128i *)(output + 1 * 16), compressed); 508| 67| _mm_storeu_si128((__m128i *)(output + 2 * 16 - 1), chunks[2]); 509| 67| _mm_storeu_si128((__m128i *)(output + 3 * 16 - 1), chunks[3]); 510| 67| } break; 511| 92| case 0b10: { ------------------ | Branch (511:5): [True: 92, False: 226] ------------------ 512| 92| _mm_storeu_si128((__m128i *)(output + 0 * 16), chunks[0]); 513| 92| _mm_storeu_si128((__m128i *)(output + 1 * 16), chunks[1]); 514| | 515| 92| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 516| 92| const __m128i v1 = 517| 92| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 518| 92| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 519| 92| const __m128i sh = _mm_sub_epi8(v1, v2); 520| 92| const __m128i compressed = _mm_shuffle_epi8(chunks[2], sh); 521| | 522| 92| _mm_storeu_si128((__m128i *)(output + 2 * 16), compressed); 523| 92| _mm_storeu_si128((__m128i *)(output + 3 * 16 - 1), chunks[3]); 524| 92| } break; 525| 89| case 0b11: { ------------------ | Branch (525:5): [True: 89, False: 229] ------------------ 526| 89| _mm_storeu_si128((__m128i *)(output + 0 * 16), chunks[0]); 527| 89| _mm_storeu_si128((__m128i *)(output + 1 * 16), chunks[1]); 528| 89| _mm_storeu_si128((__m128i *)(output + 2 * 16), chunks[2]); 529| | 530| 89| const __m128i v0 = _mm_set1_epi8(char(pos - 1)); 531| 89| const __m128i v1 = 532| 89| _mm_setr_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15); 533| 89| const __m128i v2 = _mm_cmpgt_epi8(v1, v0); 534| 89| const __m128i sh = _mm_sub_epi8(v1, v2); 535| 89| const __m128i compressed = _mm_shuffle_epi8(chunks[3], sh); 536| | 537| 89| _mm_storeu_si128((__m128i *)(output + 3 * 16), compressed); 538| 89| } break; 539| 318| } 540| | 541| 318| return 63; 542| 318| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_18compressEDv2_xtPc: 332| 2.65k| char *output) { 333| 2.65k| if (mask == 0) { ------------------ | Branch (333:7): [True: 869, False: 1.78k] ------------------ 334| 869| _mm_storeu_si128(reinterpret_cast<__m128i *>(output), data); 335| 869| return; 336| 869| } 337| | 338| | // this particular implementation was inspired by work done by @animetosho 339| | // we do it in two steps, first 8 bytes and then second 8 bytes 340| 1.78k| uint8_t mask1 = uint8_t(mask); // least significant 8 bits 341| 1.78k| uint8_t mask2 = uint8_t(mask >> 8); // most significant 8 bits 342| | // next line just loads the 64-bit values thintable_epi8[mask1] and 343| | // thintable_epi8[mask2] into a 128-bit register, using only 344| | // two instructions on most compilers. 345| | 346| 1.78k| __m128i shufmask = _mm_set_epi64x(tables::base64::thintable_epi8[mask2], 347| 1.78k| tables::base64::thintable_epi8[mask1]); 348| | // we increment by 0x08 the second half of the mask 349| 1.78k| shufmask = 350| 1.78k| _mm_add_epi8(shufmask, _mm_set_epi32(0x08080808, 0x08080808, 0, 0)); 351| | // this is the version "nearly pruned" 352| 1.78k| __m128i pruned = _mm_shuffle_epi8(data, shufmask); 353| | // we still need to put the two halves together. 354| | // we compute the popcount of the first half: 355| 1.78k| int pop1 = tables::base64::BitsSetTable256mul2[mask1]; 356| | // then load the corresponding mask, what it does is to write 357| | // only the first pop1 bytes from the first 8 bytes, and then 358| | // it fills in with the bytes from the second 8 bytes + some filling 359| | // at the end. 360| 1.78k| __m128i compactmask = _mm_loadu_si128(reinterpret_cast( 361| 1.78k| tables::base64::pshufb_combine_table + pop1 * 8)); 362| 1.78k| __m128i answer = _mm_shuffle_epi8(pruned, compactmask); 363| 1.78k| _mm_storeu_si128(reinterpret_cast<__m128i *>(output), answer); 364| 1.78k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6410copy_blockEPc: 454| 88| simdutf_really_inline void copy_block(char *output) { 455| 88| _mm_storeu_si128(reinterpret_cast<__m128i *>(output), chunks[0]); 456| 88| _mm_storeu_si128(reinterpret_cast<__m128i *>(output + 16), chunks[1]); 457| 88| _mm_storeu_si128(reinterpret_cast<__m128i *>(output + 32), chunks[2]); 458| 88| _mm_storeu_si128(reinterpret_cast<__m128i *>(output + 48), chunks[3]); 459| 88| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_119base64_decode_blockILb0EEEvPcPKc: 390| 435|static inline void base64_decode_block(char *out, const char *src) { 391| 435| base64_decode(out, _mm_loadu_si128(reinterpret_cast(src))); 392| 435| base64_decode(out + 12, 393| 435| _mm_loadu_si128(reinterpret_cast(src + 16))); 394| 435| base64_decode(out + 24, 395| 435| _mm_loadu_si128(reinterpret_cast(src + 32))); 396| 435| base64_decode( 397| 435| out + 36, _mm_loadu_si128(reinterpret_cast(src + 48))); 398| 435|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_119base64_decode_blockILb1EEEvPcPKc: 390| 98|static inline void base64_decode_block(char *out, const char *src) { 391| 98| base64_decode(out, _mm_loadu_si128(reinterpret_cast(src))); 392| 98| base64_decode(out + 12, 393| 98| _mm_loadu_si128(reinterpret_cast(src + 16))); 394| 98| base64_decode(out + 24, 395| 98| _mm_loadu_si128(reinterpret_cast(src + 32))); 396| 98| base64_decode( 397| 98| out + 36, _mm_loadu_si128(reinterpret_cast(src + 48))); 398| 98|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6414to_base64_maskILb0ELb0ELb0EEEmPm: 546| 5.20k| simdutf_really_inline uint64_t to_base64_mask(uint64_t *error) { 547| 5.20k| uint32_t err0 = 0; 548| 5.20k| uint32_t err1 = 0; 549| 5.20k| uint32_t err2 = 0; 550| 5.20k| uint32_t err3 = 0; 551| 5.20k| uint64_t m0 = to_base64_mask( 552| 5.20k| &chunks[0], &err0); 553| 5.20k| uint64_t m1 = to_base64_mask( 554| 5.20k| &chunks[1], &err1); 555| 5.20k| uint64_t m2 = to_base64_mask( 556| 5.20k| &chunks[2], &err2); 557| 5.20k| uint64_t m3 = to_base64_mask( 558| 5.20k| &chunks[3], &err3); 559| 5.20k| if (!ignore_garbage) { ------------------ | Branch (559:9): [True: 5.20k, Folded] ------------------ 560| 5.20k| *error = (err0) | ((uint64_t)err1 << 16) | ((uint64_t)err2 << 32) | 561| 5.20k| ((uint64_t)err3 << 48); 562| 5.20k| } 563| 5.20k| return m0 | (m1 << 16) | (m2 << 32) | (m3 << 48); 564| 5.20k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_17block6414to_base64_maskILb0ELb0ELb0EEEtPDv2_xPj: 568| 20.8k| simdutf_really_inline uint16_t to_base64_mask(__m128i *src, uint32_t *error) { 569| 20.8k| const __m128i ascii_space_tbl = 570| 20.8k| _mm_setr_epi8(0x20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x9, 0xa, 571| 20.8k| 0x0, 0xc, 0xd, 0x0, 0x0); 572| | // credit: aqrit 573| 20.8k| __m128i delta_asso; 574| 20.8k| if (default_or_url) { ------------------ | Branch (574:9): [Folded, False: 20.8k] ------------------ 575| 0| delta_asso = 576| 0| _mm_setr_epi8(0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 577| 0| 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x16); 578| 20.8k| } else if (base64_url) { ------------------ | Branch (578:16): [Folded, False: 20.8k] ------------------ 579| 0| delta_asso = _mm_setr_epi8(0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x0, 580| 0| 0x0, 0x0, 0x0, 0x0, 0xF, 0x0, 0xF); 581| 20.8k| } else { 582| 20.8k| delta_asso = 583| 20.8k| _mm_setr_epi8(0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x00, 584| 20.8k| 0x00, 0x00, 0x00, 0x00, 0x0F, 0x00, 0x0F); 585| 20.8k| } 586| 20.8k| __m128i delta_values; 587| 20.8k| if (default_or_url) { ------------------ | Branch (587:9): [Folded, False: 20.8k] ------------------ 588| 0| delta_values = _mm_setr_epi8( 589| 0| uint8_t(0xBF), uint8_t(0xE0), uint8_t(0xB9), uint8_t(0x13), 590| 0| uint8_t(0x04), uint8_t(0xBF), uint8_t(0xBF), uint8_t(0xB9), 591| 0| uint8_t(0xB9), uint8_t(0x00), uint8_t(0xFF), uint8_t(0x11), 592| 0| uint8_t(0xFF), uint8_t(0xBF), uint8_t(0x10), uint8_t(0xB9)); 593| | 594| 20.8k| } else if (base64_url) { ------------------ | Branch (594:16): [Folded, False: 20.8k] ------------------ 595| 0| delta_values = _mm_setr_epi8(0x0, 0x0, 0x0, 0x13, 0x4, uint8_t(0xBF), 596| 0| uint8_t(0xBF), uint8_t(0xB9), uint8_t(0xB9), 597| 0| 0x0, 0x11, uint8_t(0xC3), uint8_t(0xBF), 598| 0| uint8_t(0xE0), uint8_t(0xB9), uint8_t(0xB9)); 599| 20.8k| } else { 600| 20.8k| delta_values = 601| 20.8k| _mm_setr_epi8(int8_t(0x00), int8_t(0x00), int8_t(0x00), int8_t(0x13), 602| 20.8k| int8_t(0x04), int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), 603| 20.8k| int8_t(0xB9), int8_t(0x00), int8_t(0x10), int8_t(0xC3), 604| 20.8k| int8_t(0xBF), int8_t(0xBF), int8_t(0xB9), int8_t(0xB9)); 605| 20.8k| } 606| 20.8k| __m128i check_asso; 607| 20.8k| if (default_or_url) { ------------------ | Branch (607:9): [Folded, False: 20.8k] ------------------ 608| 0| check_asso = 609| 0| _mm_setr_epi8(0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 610| 0| 0x01, 0x03, 0x07, 0x0B, 0x0E, 0x0B, 0x06); 611| 20.8k| } else if (base64_url) { ------------------ | Branch (611:16): [Folded, False: 20.8k] ------------------ 612| 0| check_asso = _mm_setr_epi8(0xD, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 613| 0| 0x1, 0x3, 0x7, 0xB, 0xE, 0xB, 0x6); 614| 20.8k| } else { 615| 20.8k| check_asso = 616| 20.8k| _mm_setr_epi8(0x0D, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 617| 20.8k| 0x01, 0x03, 0x07, 0x0B, 0x0B, 0x0B, 0x0F); 618| 20.8k| } 619| 20.8k| __m128i check_values; 620| 20.8k| if (default_or_url) { ------------------ | Branch (620:9): [Folded, False: 20.8k] ------------------ 621| 0| check_values = _mm_setr_epi8( 622| 0| uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), 623| 0| uint8_t(0xCF), uint8_t(0xBF), uint8_t(0xD5), uint8_t(0xA6), 624| 0| uint8_t(0xB5), uint8_t(0xA1), uint8_t(0x00), uint8_t(0x80), 625| 0| uint8_t(0x00), uint8_t(0x80), uint8_t(0x00), uint8_t(0x80)); 626| 20.8k| } else if (base64_url) { ------------------ | Branch (626:16): [Folded, False: 20.8k] ------------------ 627| 0| check_values = _mm_setr_epi8(uint8_t(0x80), uint8_t(0x80), uint8_t(0x80), 628| 0| uint8_t(0x80), uint8_t(0xCF), uint8_t(0xBF), 629| 0| uint8_t(0xB6), uint8_t(0xA6), uint8_t(0xB5), 630| 0| uint8_t(0xA1), 0x0, uint8_t(0x80), 0x0, 631| 0| uint8_t(0x80), 0x0, uint8_t(0x80)); 632| 20.8k| } else { 633| 20.8k| check_values = 634| 20.8k| _mm_setr_epi8(int8_t(0x80), int8_t(0x80), int8_t(0x80), int8_t(0x80), 635| 20.8k| int8_t(0xCF), int8_t(0xBF), int8_t(0xD5), int8_t(0xA6), 636| 20.8k| int8_t(0xB5), int8_t(0x86), int8_t(0xD1), int8_t(0x80), 637| 20.8k| int8_t(0xB1), int8_t(0x80), int8_t(0x91), int8_t(0x80)); 638| 20.8k| } 639| 20.8k| const __m128i shifted = _mm_srli_epi32(*src, 3); 640| | 641| 20.8k| __m128i delta_hash = 642| 20.8k| _mm_avg_epu8(_mm_shuffle_epi8(delta_asso, *src), shifted); 643| 20.8k| if (default_or_url) { ------------------ | Branch (643:9): [Folded, False: 20.8k] ------------------ 644| 0| delta_hash = _mm_and_si128(delta_hash, _mm_set1_epi8(0xf)); 645| 0| } 646| 20.8k| const __m128i check_hash = 647| 20.8k| _mm_avg_epu8(_mm_shuffle_epi8(check_asso, *src), shifted); 648| | 649| 20.8k| const __m128i out = 650| 20.8k| _mm_adds_epi8(_mm_shuffle_epi8(delta_values, delta_hash), *src); 651| 20.8k| const __m128i chk = 652| 20.8k| _mm_adds_epi8(_mm_shuffle_epi8(check_values, check_hash), *src); 653| 20.8k| const int mask = _mm_movemask_epi8(chk); 654| 20.8k| if (!ignore_garbage && mask) { ------------------ | Branch (654:9): [True: 20.8k, Folded] | Branch (654:28): [True: 5.40k, False: 15.4k] ------------------ 655| 5.40k| __m128i ascii_space = 656| 5.40k| _mm_cmpeq_epi8(_mm_shuffle_epi8(ascii_space_tbl, *src), *src); 657| 5.40k| *error = (mask ^ _mm_movemask_epi8(ascii_space)); 658| 5.40k| } 659| 20.8k| *src = out; 660| 20.8k| return (uint16_t)mask; 661| 20.8k| } simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_113encode_base64ILb0EEEmPcPKcmNS_14base64_optionsE: 325| 18.7k| base64_options options) { 326| 18.7k| return encode_base64_impl(dst, src, srclen, options); 327| 18.7k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_118encode_base64_implILb0ELb0EEEmPcPKcmNS_14base64_optionsEm: 95| 18.7k| size_t line_length = simdutf::default_line_length) { 96| 18.7k| size_t offset = 0; 97| 18.7k| if (line_length < 4) { ------------------ | Branch (97:7): [True: 0, False: 18.7k] ------------------ 98| 0| line_length = 4; // We do not support line_length less than 4 99| 0| } 100| | // credit: Wojciech Muła 101| | // SSE (lookup: pshufb improved unrolled) 102| 18.7k| const uint8_t *input = (const uint8_t *)src; 103| | 104| 18.7k| uint8_t *out = (uint8_t *)dst; 105| 18.7k| const __m128i shuf = 106| 18.7k| _mm_set_epi8(10, 11, 9, 10, 7, 8, 6, 7, 4, 5, 3, 4, 1, 2, 0, 1); 107| | 108| 18.7k| size_t i = 0; 109| 21.7k| for (; i + 52 <= srclen; i += 48) { ------------------ | Branch (109:10): [True: 3.01k, False: 18.7k] ------------------ 110| 3.01k| __m128i in0 = _mm_loadu_si128( 111| 3.01k| reinterpret_cast(input + i + 4 * 3 * 0)); 112| 3.01k| __m128i in1 = _mm_loadu_si128( 113| 3.01k| reinterpret_cast(input + i + 4 * 3 * 1)); 114| 3.01k| __m128i in2 = _mm_loadu_si128( 115| 3.01k| reinterpret_cast(input + i + 4 * 3 * 2)); 116| 3.01k| __m128i in3 = _mm_loadu_si128( 117| 3.01k| reinterpret_cast(input + i + 4 * 3 * 3)); 118| | 119| 3.01k| in0 = _mm_shuffle_epi8(in0, shuf); 120| 3.01k| in1 = _mm_shuffle_epi8(in1, shuf); 121| 3.01k| in2 = _mm_shuffle_epi8(in2, shuf); 122| 3.01k| in3 = _mm_shuffle_epi8(in3, shuf); 123| | 124| 3.01k| const __m128i t0_0 = _mm_and_si128(in0, _mm_set1_epi32(0x0fc0fc00)); 125| 3.01k| const __m128i t0_1 = _mm_and_si128(in1, _mm_set1_epi32(0x0fc0fc00)); 126| 3.01k| const __m128i t0_2 = _mm_and_si128(in2, _mm_set1_epi32(0x0fc0fc00)); 127| 3.01k| const __m128i t0_3 = _mm_and_si128(in3, _mm_set1_epi32(0x0fc0fc00)); 128| | 129| 3.01k| const __m128i t1_0 = _mm_mulhi_epu16(t0_0, _mm_set1_epi32(0x04000040)); 130| 3.01k| const __m128i t1_1 = _mm_mulhi_epu16(t0_1, _mm_set1_epi32(0x04000040)); 131| 3.01k| const __m128i t1_2 = _mm_mulhi_epu16(t0_2, _mm_set1_epi32(0x04000040)); 132| 3.01k| const __m128i t1_3 = _mm_mulhi_epu16(t0_3, _mm_set1_epi32(0x04000040)); 133| | 134| 3.01k| const __m128i t2_0 = _mm_and_si128(in0, _mm_set1_epi32(0x003f03f0)); 135| 3.01k| const __m128i t2_1 = _mm_and_si128(in1, _mm_set1_epi32(0x003f03f0)); 136| 3.01k| const __m128i t2_2 = _mm_and_si128(in2, _mm_set1_epi32(0x003f03f0)); 137| 3.01k| const __m128i t2_3 = _mm_and_si128(in3, _mm_set1_epi32(0x003f03f0)); 138| | 139| 3.01k| const __m128i t3_0 = _mm_mullo_epi16(t2_0, _mm_set1_epi32(0x01000010)); 140| 3.01k| const __m128i t3_1 = _mm_mullo_epi16(t2_1, _mm_set1_epi32(0x01000010)); 141| 3.01k| const __m128i t3_2 = _mm_mullo_epi16(t2_2, _mm_set1_epi32(0x01000010)); 142| 3.01k| const __m128i t3_3 = _mm_mullo_epi16(t2_3, _mm_set1_epi32(0x01000010)); 143| | 144| 3.01k| const __m128i input0 = _mm_or_si128(t1_0, t3_0); 145| 3.01k| const __m128i input1 = _mm_or_si128(t1_1, t3_1); 146| 3.01k| const __m128i input2 = _mm_or_si128(t1_2, t3_2); 147| 3.01k| const __m128i input3 = _mm_or_si128(t1_3, t3_3); 148| | 149| 3.01k| const __m128i t0 = lookup_pshufb_improved(input0); 150| 3.01k| const __m128i t1 = lookup_pshufb_improved(input1); 151| 3.01k| const __m128i t2 = lookup_pshufb_improved(input2); 152| 3.01k| const __m128i t3 = lookup_pshufb_improved(input3); 153| | 154| 3.01k| if (use_lines) { ------------------ | Branch (154:9): [Folded, False: 3.01k] ------------------ 155| 0| if (line_length >= 64) { // fast path ------------------ | Branch (155:11): [True: 0, False: 0] ------------------ 156| 0| if (offset + 64 > line_length) { ------------------ | Branch (156:13): [True: 0, False: 0] ------------------ 157| 0| size_t location_end = line_length - offset; 158| 0| size_t to_move = 64 - location_end; 159| 0| if (location_end < 16) { ------------------ | Branch (159:15): [True: 0, False: 0] ------------------ 160| | // We can store or extract store. See below. 161| | //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t0); 162| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), 163| 0| insert_line_feed16(t0, location_end)); 164| 0| out[16] = static_cast(_mm_extract_epi8(t0, 15)); 165| 0| out += 17; 166| 0| } else { 167| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t0); 168| 0| out += 16; 169| 0| } 170| 0| if (location_end >= 16 && location_end < 32) { ------------------ | Branch (170:15): [True: 0, False: 0] | Branch (170:37): [True: 0, False: 0] ------------------ 171| | // We can store or extract store. See below. 172| | //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t1); 173| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), 174| 0| insert_line_feed16(t1, location_end - 16)); 175| 0| out[16] = static_cast(_mm_extract_epi8(t1, 15)); 176| 0| out += 17; 177| 0| } else { 178| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t1); 179| 0| out += 16; 180| 0| } 181| 0| if (location_end >= 32 && location_end < 48) { ------------------ | Branch (181:15): [True: 0, False: 0] | Branch (181:37): [True: 0, False: 0] ------------------ 182| | // We can store or extract store. See below. 183| | //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t2); 184| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), 185| 0| insert_line_feed16(t2, location_end - 32)); 186| 0| out[16] = static_cast(_mm_extract_epi8(t2, 15)); 187| 0| out += 17; 188| 0| } else { 189| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t2); 190| 0| out += 16; 191| 0| } 192| 0| if (location_end >= 48) { ------------------ | Branch (192:15): [True: 0, False: 0] ------------------ 193| | // We can store or extract store. See below. 194| | //_mm_storeu_si128(reinterpret_cast<__m128i *>(out+1), t3); 195| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), 196| 0| insert_line_feed16(t3, location_end - 48)); 197| 0| out[16] = static_cast(_mm_extract_epi8(t3, 15)); 198| 0| out += 17; 199| 0| } else { 200| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t3); 201| 0| out += 16; 202| 0| } 203| 0| offset = to_move; 204| 0| } else { 205| | 206| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t0); 207| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 16), t1); 208| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 32), t2); 209| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 48), t3); 210| 0| offset += 64; 211| 0| out += 64; 212| 0| } 213| 0| } else { // slow path 214| | // could be optimized 215| 0| alignas(64) uint8_t buffer[64]; 216| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer), t0); 217| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 16), t1); 218| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 32), t2); 219| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer + 48), t3); 220| 0| std::memcpy(out, buffer, 64); 221| 0| size_t out_pos = 0; 222| 0| size_t local_offset = offset; 223| 0| for (size_t j = 0; j < 64;) { ------------------ | Branch (223:28): [True: 0, False: 0] ------------------ 224| 0| if (local_offset == line_length) { ------------------ | Branch (224:15): [True: 0, False: 0] ------------------ 225| 0| out[out_pos++] = '\n'; 226| 0| local_offset = 0; 227| 0| } 228| 0| out[out_pos++] = buffer[j++]; 229| 0| local_offset++; 230| 0| } 231| 0| offset = local_offset; 232| 0| out += out_pos; 233| 0| } 234| 3.01k| } else { 235| 3.01k| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), t0); 236| 3.01k| _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 16), t1); 237| 3.01k| _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 32), t2); 238| 3.01k| _mm_storeu_si128(reinterpret_cast<__m128i *>(out + 48), t3); 239| 3.01k| out += 64; 240| 3.01k| } 241| 3.01k| } 242| 19.7k| for (; i + 16 <= srclen; i += 12) { ------------------ | Branch (242:10): [True: 1.03k, False: 18.7k] ------------------ 243| | 244| 1.03k| __m128i in = _mm_loadu_si128(reinterpret_cast(input + i)); 245| | 246| | // bytes from groups A, B and C are needed in separate 32-bit lanes 247| | // in = [DDDD|CCCC|BBBB|AAAA] 248| | // 249| | // an input triplet has layout 250| | // [????????|ccdddddd|bbbbcccc|aaaaaabb] 251| | // byte 3 byte 2 byte 1 byte 0 -- byte 3 comes from the next 252| | // triplet 253| | // 254| | // shuffling changes the order of bytes: 1, 0, 2, 1 255| | // [bbbbcccc|ccdddddd|aaaaaabb|bbbbcccc] 256| | // ^^^^ ^^^^^^^^ ^^^^^^^^ ^^^^ 257| | // processed bits 258| 1.03k| in = _mm_shuffle_epi8(in, shuf); 259| | 260| | // unpacking 261| | 262| | // t0 = [0000cccc|cc000000|aaaaaa00|00000000] 263| 1.03k| const __m128i t0 = _mm_and_si128(in, _mm_set1_epi32(0x0fc0fc00)); 264| | // t1 = [00000000|00cccccc|00000000|00aaaaaa] 265| | // (c * (1 << 10), a * (1 << 6)) >> 16 (note: an unsigned 266| | // multiplication) 267| 1.03k| const __m128i t1 = _mm_mulhi_epu16(t0, _mm_set1_epi32(0x04000040)); 268| | 269| | // t2 = [00000000|00dddddd|000000bb|bbbb0000] 270| 1.03k| const __m128i t2 = _mm_and_si128(in, _mm_set1_epi32(0x003f03f0)); 271| | // t3 = [00dddddd|00000000|00bbbbbb|00000000]( 272| | // (d * (1 << 8), b * (1 << 4)) 273| 1.03k| const __m128i t3 = _mm_mullo_epi16(t2, _mm_set1_epi32(0x01000010)); 274| | 275| | // res = [00dddddd|00cccccc|00bbbbbb|00aaaaaa] = t1 | t3 276| 1.03k| const __m128i indices = _mm_or_si128(t1, t3); 277| | 278| 1.03k| const __m128i T0 = lookup_pshufb_improved(indices); 279| | 280| 1.03k| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), T0); 281| | 282| 1.03k| if (use_lines) { ------------------ | Branch (282:9): [Folded, False: 1.03k] ------------------ 283| 0| if (line_length >= 16) { // fast path ------------------ | Branch (283:11): [True: 0, False: 0] ------------------ 284| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), T0); 285| 0| if (offset + 16 > line_length) { ------------------ | Branch (285:13): [True: 0, False: 0] ------------------ 286| 0| size_t location_end = line_length - offset; 287| 0| size_t to_move = 16 - location_end; 288| 0| std::memmove(out + location_end + 1, out + location_end, to_move); 289| 0| out[location_end] = '\n'; 290| 0| offset = to_move; 291| 0| out += 16 + 1; 292| 0| } else { 293| 0| offset += 16; 294| 0| out += 16; 295| 0| } 296| 0| } else { // slow path 297| | // could be optimized 298| 0| uint8_t buffer[16]; 299| 0| _mm_storeu_si128(reinterpret_cast<__m128i *>(buffer), T0); 300| 0| size_t out_pos = 0; 301| 0| size_t local_offset = offset; 302| 0| for (size_t j = 0; j < 16;) { ------------------ | Branch (302:28): [True: 0, False: 0] ------------------ 303| 0| if (local_offset == line_length) { ------------------ | Branch (303:15): [True: 0, False: 0] ------------------ 304| 0| out[out_pos++] = '\n'; 305| 0| local_offset = 0; 306| 0| } 307| 0| out[out_pos++] = buffer[j++]; 308| 0| local_offset++; 309| 0| } 310| 0| offset = local_offset; 311| 0| out += out_pos; 312| 0| } 313| 1.03k| } else { 314| 1.03k| _mm_storeu_si128(reinterpret_cast<__m128i *>(out), T0); 315| 1.03k| out += 16; 316| 1.03k| } 317| 1.03k| } 318| 18.7k| return ((char *)out - (char *)dst) + 319| 18.7k| scalar::base64::tail_encode_base64_impl( 320| 18.7k| (char *)out, src + i, srclen - i, options, line_length, offset); 321| 18.7k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_122lookup_pshufb_improvedILb0EEEDv2_xS3_: 30| 13.1k|template __m128i lookup_pshufb_improved(const __m128i input) { 31| | // credit: Wojciech Muła 32| | // reduce 0..51 -> 0 33| | // 52..61 -> 1 .. 10 34| | // 62 -> 11 35| | // 63 -> 12 36| 13.1k| __m128i result = _mm_subs_epu8(input, _mm_set1_epi8(51)); 37| | 38| | // distinguish between ranges 0..25 and 26..51 by adding 1 (the 39| | // comparison yields -1) to every value greater than 25: 40| | // 0 .. 25 -> remains 0 41| | // 26 .. 51 -> becomes 1 42| | // 52 .. 61 -> 2 .. 11 43| | // 62 -> 12 44| | // 63 -> 13 45| 13.1k| result = _mm_sub_epi8(result, _mm_cmpgt_epi8(input, _mm_set1_epi8(25))); 46| | 47| 13.1k| __m128i shift_LUT; 48| 13.1k| if (base64_url) { ------------------ | Branch (48:7): [Folded, False: 13.1k] ------------------ 49| 0| shift_LUT = _mm_setr_epi8('A', 'a' - 26, '0' - 52, '0' - 52, '0' - 52, 50| 0| '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, 51| 0| '0' - 52, '0' - 52, '-' - 62, '_' - 63, 0, 0); 52| 13.1k| } else { 53| 13.1k| shift_LUT = _mm_setr_epi8('A', 'a' - 26, '0' - 52, '0' - 52, '0' - 52, 54| 13.1k| '0' - 52, '0' - 52, '0' - 52, '0' - 52, '0' - 52, 55| 13.1k| '0' - 52, '0' - 52, '+' - 62, '/' - 63, 0, 0); 56| 13.1k| } 57| | 58| | // read shift 59| 13.1k| result = _mm_shuffle_epi8(shift_LUT, result); 60| | 61| 13.1k| return _mm_add_epi8(result, input); 62| 13.1k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_127sse_convert_latin1_to_utf16ILNS_10endiannessE0EEENSt3__14pairIPKcPDsEES7_mS8_: 4| 6.85k| char16_t *utf16_output) { 5| 6.85k| size_t rounded_len = len & ~0xF; // Round down to nearest multiple of 16 6| 27.0k| for (size_t i = 0; i < rounded_len; i += 16) { ------------------ | Branch (6:22): [True: 20.2k, False: 6.85k] ------------------ 7| | // Load 16 Latin1 characters into a 128-bit register 8| 20.2k| __m128i in = 9| 20.2k| _mm_loadu_si128(reinterpret_cast(&latin1_input[i])); 10| 20.2k| __m128i out1 = big_endian ? _mm_unpacklo_epi8(_mm_setzero_si128(), in) ------------------ | Branch (10:20): [Folded, False: 20.2k] ------------------ 11| 20.2k| : _mm_unpacklo_epi8(in, _mm_setzero_si128()); 12| 20.2k| __m128i out2 = big_endian ? _mm_unpackhi_epi8(_mm_setzero_si128(), in) ------------------ | Branch (12:20): [Folded, False: 20.2k] ------------------ 13| 20.2k| : _mm_unpackhi_epi8(in, _mm_setzero_si128()); 14| | // Zero extend each Latin1 character to 16-bit integers and store the 15| | // results back to memory 16| 20.2k| _mm_storeu_si128(reinterpret_cast<__m128i *>(&utf16_output[i]), out1); 17| 20.2k| _mm_storeu_si128(reinterpret_cast<__m128i *>(&utf16_output[i + 8]), out2); 18| 20.2k| } 19| | // return pointers pointing to where we left off 20| 6.85k| return std::make_pair(latin1_input + rounded_len, utf16_output + rounded_len); 21| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_127sse_convert_latin1_to_utf16ILNS_10endiannessE1EEENSt3__14pairIPKcPDsEES7_mS8_: 4| 6.85k| char16_t *utf16_output) { 5| 6.85k| size_t rounded_len = len & ~0xF; // Round down to nearest multiple of 16 6| 27.0k| for (size_t i = 0; i < rounded_len; i += 16) { ------------------ | Branch (6:22): [True: 20.2k, False: 6.85k] ------------------ 7| | // Load 16 Latin1 characters into a 128-bit register 8| 20.2k| __m128i in = 9| 20.2k| _mm_loadu_si128(reinterpret_cast(&latin1_input[i])); 10| 20.2k| __m128i out1 = big_endian ? _mm_unpacklo_epi8(_mm_setzero_si128(), in) ------------------ | Branch (10:20): [True: 20.2k, Folded] ------------------ 11| 20.2k| : _mm_unpacklo_epi8(in, _mm_setzero_si128()); 12| 20.2k| __m128i out2 = big_endian ? _mm_unpackhi_epi8(_mm_setzero_si128(), in) ------------------ | Branch (12:20): [True: 20.2k, Folded] ------------------ 13| 20.2k| : _mm_unpackhi_epi8(in, _mm_setzero_si128()); 14| | // Zero extend each Latin1 character to 16-bit integers and store the 15| | // results back to memory 16| 20.2k| _mm_storeu_si128(reinterpret_cast<__m128i *>(&utf16_output[i]), out1); 17| 20.2k| _mm_storeu_si128(reinterpret_cast<__m128i *>(&utf16_output[i + 8]), out2); 18| 20.2k| } 19| | // return pointers pointing to where we left off 20| 6.85k| return std::make_pair(latin1_input + rounded_len, utf16_output + rounded_len); 21| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_127sse_convert_latin1_to_utf32EPKcmPDi: 3| 6.85k| char32_t *utf32_output) { 4| 6.85k| const char *end = buf + len; 5| | 6| 27.0k| while (end - buf >= 16) { ------------------ | Branch (6:10): [True: 20.2k, False: 6.85k] ------------------ 7| | // Load 16 Latin1 characters (16 bytes) into a 128-bit register 8| 20.2k| __m128i in = _mm_loadu_si128((__m128i *)buf); 9| | 10| | // Shift input to process next 4 bytes 11| 20.2k| __m128i in_shifted1 = _mm_srli_si128(in, 4); 12| 20.2k| __m128i in_shifted2 = _mm_srli_si128(in, 8); 13| 20.2k| __m128i in_shifted3 = _mm_srli_si128(in, 12); 14| | 15| | // expand 8-bit to 32-bit unit 16| 20.2k| __m128i out1 = _mm_cvtepu8_epi32(in); 17| 20.2k| __m128i out2 = _mm_cvtepu8_epi32(in_shifted1); 18| 20.2k| __m128i out3 = _mm_cvtepu8_epi32(in_shifted2); 19| 20.2k| __m128i out4 = _mm_cvtepu8_epi32(in_shifted3); 20| | 21| 20.2k| _mm_storeu_si128((__m128i *)utf32_output, out1); 22| 20.2k| _mm_storeu_si128((__m128i *)(utf32_output + 4), out2); 23| 20.2k| _mm_storeu_si128((__m128i *)(utf32_output + 8), out3); 24| 20.2k| _mm_storeu_si128((__m128i *)(utf32_output + 12), out4); 25| | 26| 20.2k| utf32_output += 16; 27| 20.2k| buf += 16; 28| 20.2k| } 29| | 30| 6.85k| return std::make_pair(buf, utf32_output); 31| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_126sse_convert_latin1_to_utf8EPKcmPc: 3| 7.19k| const size_t latin_input_length, char *utf8_output) { 4| 7.19k| const char *end = latin_input + latin_input_length; 5| | 6| 7.19k| const __m128i v_0000 = _mm_setzero_si128(); 7| | // 0b1000_0000 8| 7.19k| const __m128i v_80 = _mm_set1_epi8((uint8_t)0x80); 9| | // 0b1111_1111_1000_0000 10| 7.19k| const __m128i v_ff80 = _mm_set1_epi16((uint16_t)0xff80); 11| | 12| 7.19k| const __m128i latin_1_half_into_u16_byte_mask = 13| 7.19k| _mm_setr_epi8(0, '\x80', 1, '\x80', 2, '\x80', 3, '\x80', 4, '\x80', 5, 14| 7.19k| '\x80', 6, '\x80', 7, '\x80'); 15| | 16| 7.19k| const __m128i latin_2_half_into_u16_byte_mask = 17| 7.19k| _mm_setr_epi8(8, '\x80', 9, '\x80', 10, '\x80', 11, '\x80', 12, '\x80', 18| 7.19k| 13, '\x80', 14, '\x80', 15, '\x80'); 19| | 20| | // each latin1 takes 1-2 utf8 bytes 21| | // slow path writes useful 8-15 bytes twice (eagerly writes 16 bytes and then 22| | // adjust the pointer) so the last write can exceed the utf8_output size by 23| | // 8-1 bytes by reserving 8 extra input bytes, we expect the output to have 24| | // 8-16 bytes free 25| 29.1k| while (end - latin_input >= 16 + 8) { ------------------ | Branch (25:10): [True: 21.9k, False: 7.19k] ------------------ 26| | // Load 16 Latin1 characters (16 bytes) into a 128-bit register 27| 21.9k| __m128i v_latin = _mm_loadu_si128((__m128i *)latin_input); 28| | 29| 21.9k| if (_mm_testz_si128(v_latin, v_80)) { // ASCII fast path!!!! ------------------ | Branch (29:9): [True: 13.1k, False: 8.80k] ------------------ 30| 13.1k| _mm_storeu_si128((__m128i *)utf8_output, v_latin); 31| 13.1k| latin_input += 16; 32| 13.1k| utf8_output += 16; 33| 13.1k| continue; 34| 13.1k| } 35| | 36| | // assuming a/b are bytes and A/B are uint16 of the same value 37| | // aaaa_aaaa_bbbb_bbbb -> AAAA_AAAA 38| 8.80k| __m128i v_u16_latin_1_half = 39| 8.80k| _mm_shuffle_epi8(v_latin, latin_1_half_into_u16_byte_mask); 40| | // aaaa_aaaa_bbbb_bbbb -> BBBB_BBBB 41| 8.80k| __m128i v_u16_latin_2_half = 42| 8.80k| _mm_shuffle_epi8(v_latin, latin_2_half_into_u16_byte_mask); 43| | 44| 8.80k| internal::westmere::write_v_u16_11bits_to_utf8(v_u16_latin_1_half, 45| 8.80k| utf8_output, v_0000, v_ff80); 46| 8.80k| internal::westmere::write_v_u16_11bits_to_utf8(v_u16_latin_2_half, 47| 8.80k| utf8_output, v_0000, v_ff80); 48| 8.80k| latin_input += 16; 49| 8.80k| } 50| | 51| 7.19k| if (end - latin_input >= 16) { ------------------ | Branch (51:7): [True: 1.02k, False: 6.16k] ------------------ 52| | // Load 16 Latin1 characters (16 bytes) into a 128-bit register 53| 1.02k| __m128i v_latin = _mm_loadu_si128((__m128i *)latin_input); 54| | 55| 1.02k| if (_mm_testz_si128(v_latin, v_80)) { // ASCII fast path!!!! ------------------ | Branch (55:9): [True: 570, False: 452] ------------------ 56| 570| _mm_storeu_si128((__m128i *)utf8_output, v_latin); 57| 570| latin_input += 16; 58| 570| utf8_output += 16; 59| 570| } else { 60| | // assuming a/b are bytes and A/B are uint16 of the same value 61| | // aaaa_aaaa_bbbb_bbbb -> AAAA_AAAA 62| 452| __m128i v_u16_latin_1_half = 63| 452| _mm_shuffle_epi8(v_latin, latin_1_half_into_u16_byte_mask); 64| 452| internal::westmere::write_v_u16_11bits_to_utf8( 65| 452| v_u16_latin_1_half, utf8_output, v_0000, v_ff80); 66| 452| latin_input += 8; 67| 452| } 68| 1.02k| } 69| | 70| 7.19k| return std::make_pair(latin_input, utf8_output); 71| 7.19k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_127sse_convert_utf16_to_latin1ILNS_10endiannessE0EEENSt3__14pairIPKDsPcEES7_mS8_: 4| 6.85k| char *latin1_output) { 5| 6.85k| const char16_t *end = buf + len; 6| 47.4k| while (end - buf >= 8) { ------------------ | Branch (6:10): [True: 40.5k, False: 6.85k] ------------------ 7| | // Load 8 UTF-16 characters into 128-bit SSE register 8| 40.5k| __m128i in = _mm_loadu_si128(reinterpret_cast(buf)); 9| | 10| | if constexpr (!match_system(big_endian)) { 11| | const __m128i swap = 12| | _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 13| | in = _mm_shuffle_epi8(in, swap); 14| | } 15| | 16| 40.5k| __m128i high_byte_mask = _mm_set1_epi16((int16_t)0xFF00); 17| 40.5k| if (_mm_testz_si128(in, high_byte_mask)) { ------------------ | Branch (17:9): [True: 40.5k, False: 0] ------------------ 18| | // Pack 16-bit characters into 8-bit and store in latin1_output 19| 40.5k| __m128i latin1_packed = _mm_packus_epi16(in, in); 20| 40.5k| _mm_storel_epi64(reinterpret_cast<__m128i *>(latin1_output), 21| 40.5k| latin1_packed); 22| | // Adjust pointers for next iteration 23| 40.5k| buf += 8; 24| 40.5k| latin1_output += 8; 25| 40.5k| } else { 26| 0| return std::make_pair(nullptr, reinterpret_cast(latin1_output)); 27| 0| } 28| 40.5k| } // while 29| 6.85k| return std::make_pair(buf, latin1_output); 30| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_127sse_convert_utf16_to_latin1ILNS_10endiannessE1EEENSt3__14pairIPKDsPcEES7_mS8_: 4| 6.85k| char *latin1_output) { 5| 6.85k| const char16_t *end = buf + len; 6| 47.4k| while (end - buf >= 8) { ------------------ | Branch (6:10): [True: 40.5k, False: 6.85k] ------------------ 7| | // Load 8 UTF-16 characters into 128-bit SSE register 8| 40.5k| __m128i in = _mm_loadu_si128(reinterpret_cast(buf)); 9| | 10| 40.5k| if constexpr (!match_system(big_endian)) { 11| 40.5k| const __m128i swap = 12| 40.5k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 13| 40.5k| in = _mm_shuffle_epi8(in, swap); 14| 40.5k| } 15| | 16| 40.5k| __m128i high_byte_mask = _mm_set1_epi16((int16_t)0xFF00); 17| 40.5k| if (_mm_testz_si128(in, high_byte_mask)) { ------------------ | Branch (17:9): [True: 40.5k, False: 0] ------------------ 18| | // Pack 16-bit characters into 8-bit and store in latin1_output 19| 40.5k| __m128i latin1_packed = _mm_packus_epi16(in, in); 20| 40.5k| _mm_storel_epi64(reinterpret_cast<__m128i *>(latin1_output), 21| 40.5k| latin1_packed); 22| | // Adjust pointers for next iteration 23| 40.5k| buf += 8; 24| 40.5k| latin1_output += 8; 25| 40.5k| } else { 26| 0| return std::make_pair(nullptr, reinterpret_cast(latin1_output)); 27| 0| } 28| 40.5k| } // while 29| 6.85k| return std::make_pair(buf, latin1_output); 30| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_125sse_convert_utf16_to_utf8ILNS_10endiannessE0EEENSt3__14pairIPKDsPcEES7_mS8_: 56| 13.1k|sse_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { 57| | 58| 13.1k| const char16_t *end = buf + len; 59| | 60| 13.1k| const __m128i v_0000 = _mm_setzero_si128(); 61| 13.1k| const __m128i v_f800 = _mm_set1_epi16((int16_t)0xf800); 62| 13.1k| const __m128i v_d800 = _mm_set1_epi16((int16_t)0xd800); 63| 13.1k| const size_t safety_margin = 64| 13.1k| 12; // to avoid overruns, see issue 65| | // https://github.com/simdutf/simdutf/issues/92 66| | 67| 21.8k| while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { ------------------ | Branch (67:10): [True: 8.76k, False: 13.0k] ------------------ 68| 8.76k| __m128i in = _mm_loadu_si128((__m128i *)buf); 69| 8.76k| if (big_endian) { ------------------ | Branch (69:9): [Folded, False: 8.76k] ------------------ 70| 0| const __m128i swap = 71| 0| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 72| 0| in = _mm_shuffle_epi8(in, swap); 73| 0| } 74| | // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes 75| 8.76k| const __m128i v_ff80 = _mm_set1_epi16((int16_t)0xff80); 76| 8.76k| if (_mm_testz_si128(in, v_ff80)) { // ASCII fast path!!!! ------------------ | Branch (76:9): [True: 3.09k, False: 5.67k] ------------------ 77| 3.09k| __m128i nextin = _mm_loadu_si128((__m128i *)buf + 1); 78| 3.09k| if (big_endian) { ------------------ | Branch (78:11): [Folded, False: 3.09k] ------------------ 79| 0| const __m128i swap = 80| 0| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 81| 0| nextin = _mm_shuffle_epi8(nextin, swap); 82| 0| } 83| 3.09k| if (!_mm_testz_si128(nextin, v_ff80)) { ------------------ | Branch (83:11): [True: 336, False: 2.75k] ------------------ 84| | // 1. pack the bytes 85| | // obviously suboptimal. 86| 336| const __m128i utf8_packed = _mm_packus_epi16(in, in); 87| | // 2. store (16 bytes) 88| 336| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 89| | // 3. adjust pointers 90| 336| buf += 8; 91| 336| utf8_output += 8; 92| 336| in = nextin; 93| 2.75k| } else { 94| | // 1. pack the bytes 95| | // obviously suboptimal. 96| 2.75k| const __m128i utf8_packed = _mm_packus_epi16(in, nextin); 97| | // 2. store (16 bytes) 98| 2.75k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 99| | // 3. adjust pointers 100| 2.75k| buf += 16; 101| 2.75k| utf8_output += 16; 102| 2.75k| continue; // we are done for this round! 103| 2.75k| } 104| 3.09k| } 105| | 106| | // no bits set above 7th bit 107| 6.01k| const __m128i one_byte_bytemask = 108| 6.01k| _mm_cmpeq_epi16(_mm_and_si128(in, v_ff80), v_0000); 109| 6.01k| const uint16_t one_byte_bitmask = 110| 6.01k| static_cast(_mm_movemask_epi8(one_byte_bytemask)); 111| | 112| | // no bits set above 11th bit 113| 6.01k| const __m128i one_or_two_bytes_bytemask = 114| 6.01k| _mm_cmpeq_epi16(_mm_and_si128(in, v_f800), v_0000); 115| 6.01k| const uint16_t one_or_two_bytes_bitmask = 116| 6.01k| static_cast(_mm_movemask_epi8(one_or_two_bytes_bytemask)); 117| | 118| 6.01k| if (one_or_two_bytes_bitmask == 0xffff) { ------------------ | Branch (118:9): [True: 1.69k, False: 4.31k] ------------------ 119| 1.69k| internal::westmere::write_v_u16_11bits_to_utf8( 120| 1.69k| in, utf8_output, one_byte_bytemask, one_byte_bitmask); 121| 1.69k| buf += 8; 122| 1.69k| continue; 123| 1.69k| } 124| | 125| | // 1. Check if there are any surrogate word in the input chunk. 126| | // We have also deal with situation when there is a surrogate word 127| | // at the end of a chunk. 128| 4.31k| const __m128i surrogates_bytemask = 129| 4.31k| _mm_cmpeq_epi16(_mm_and_si128(in, v_f800), v_d800); 130| | 131| | // bitmask = 0x0000 if there are no surrogates 132| | // = 0xc000 if the last word is a surrogate 133| 4.31k| const uint16_t surrogates_bitmask = 134| 4.31k| static_cast(_mm_movemask_epi8(surrogates_bytemask)); 135| | // It might seem like checking for surrogates_bitmask == 0xc000 could help. 136| | // However, it is likely an uncommon occurrence. 137| 4.31k| if (surrogates_bitmask == 0x0000) { ------------------ | Branch (137:9): [True: 3.18k, False: 1.13k] ------------------ 138| | // case: code units from register produce either 1, 2 or 3 UTF-8 bytes 139| 3.18k| const __m128i dup_even = _mm_setr_epi16(0x0000, 0x0202, 0x0404, 0x0606, 140| 3.18k| 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); 141| | 142| | /* In this branch we handle three cases: 143| | 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - 144| | single UFT-8 byte 145| | 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two 146| | UTF-8 bytes 147| | 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - 148| | three UTF-8 bytes 149| | 150| | We expand the input word (16-bit) into two code units (32-bit), thus 151| | we have room for four bytes. However, we need five distinct bit 152| | layouts. Note that the last byte in cases #2 and #3 is the same. 153| | 154| | We precompute byte 1 for case #1 and the common byte for cases #2 & #3 155| | in register t2. 156| | 157| | We precompute byte 1 for case #3 and -- **conditionally** -- precompute 158| | either byte 1 for case #2 or byte 2 for case #3. Note that they 159| | differ by exactly one bit. 160| | 161| | Finally from these two code units we build proper UTF-8 sequence, taking 162| | into account the case (i.e, the number of bytes to write). 163| | */ 164| | /** 165| | * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: 166| | * t2 => [0ccc|cccc] [10cc|cccc] 167| | * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) 168| | */ 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) 170| | // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] 171| 3.18k| const __m128i t0 = _mm_shuffle_epi8(in, dup_even); 172| | // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] 173| 3.18k| const __m128i t1 = _mm_and_si128(t0, simdutf_vec(0b0011111101111111)); ------------------ | | 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 174| | // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] 175| 3.18k| const __m128i t2 = _mm_or_si128(t1, simdutf_vec(0b1000000000000000)); ------------------ | | 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 176| | 177| | // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] 178| 3.18k| const __m128i s0 = _mm_srli_epi16(in, 4); 179| | // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] 180| 3.18k| const __m128i s1 = _mm_and_si128(s0, simdutf_vec(0b0000111111111100)); ------------------ | | 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 181| | // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] 182| 3.18k| const __m128i s2 = _mm_maddubs_epi16(s1, simdutf_vec(0x0140)); ------------------ | | 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 183| | // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] 184| 3.18k| const __m128i s3 = _mm_or_si128(s2, simdutf_vec(0b1100000011100000)); ------------------ | | 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 185| 3.18k| const __m128i m0 = _mm_andnot_si128(one_or_two_bytes_bytemask, 186| 3.18k| simdutf_vec(0b0100000000000000)); ------------------ | | 169| 3.18k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 187| 3.18k| const __m128i s4 = _mm_xor_si128(s3, m0); 188| 3.18k|#undef simdutf_vec 189| | 190| | // 4. expand code units 16-bit => 32-bit 191| 3.18k| const __m128i out0 = _mm_unpacklo_epi16(t2, s4); 192| 3.18k| const __m128i out1 = _mm_unpackhi_epi16(t2, s4); 193| | 194| | // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle 195| 3.18k| const uint16_t mask = 196| 3.18k| (one_byte_bitmask & 0x5555) | (one_or_two_bytes_bitmask & 0xaaaa); 197| 3.18k| if (mask == 0) { ------------------ | Branch (197:11): [True: 2.39k, False: 790] ------------------ 198| | // We only have three-byte code units. Use fast path. 199| 2.39k| const __m128i shuffle = _mm_setr_epi8(2, 3, 1, 6, 7, 5, 10, 11, 9, 14, 200| 2.39k| 15, 13, -1, -1, -1, -1); 201| 2.39k| const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle); 202| 2.39k| const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle); 203| 2.39k| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 204| 2.39k| utf8_output += 12; 205| 2.39k| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 206| 2.39k| utf8_output += 12; 207| 2.39k| buf += 8; 208| 2.39k| continue; 209| 2.39k| } 210| 790| const uint8_t mask0 = uint8_t(mask); 211| | 212| 790| const uint8_t *row0 = 213| 790| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; 214| 790| const __m128i shuffle0 = _mm_loadu_si128((__m128i *)(row0 + 1)); 215| 790| const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle0); 216| | 217| 790| const uint8_t mask1 = static_cast(mask >> 8); 218| | 219| 790| const uint8_t *row1 = 220| 790| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; 221| 790| const __m128i shuffle1 = _mm_loadu_si128((__m128i *)(row1 + 1)); 222| 790| const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle1); 223| | 224| 790| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 225| 790| utf8_output += row0[0]; 226| 790| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 227| 790| utf8_output += row1[0]; 228| | 229| 790| buf += 8; 230| | // surrogate pair(s) in a register 231| 1.13k| } else { 232| | // Let us do a scalar fallback. 233| | // It may seem wasteful to use scalar code, but being efficient with SIMD 234| | // in the presence of surrogate pairs may require non-trivial tables. 235| 1.13k| size_t forward = 15; 236| 1.13k| size_t k = 0; 237| 1.13k| if (size_t(end - buf) < forward + 1) { ------------------ | Branch (237:11): [True: 0, False: 1.13k] ------------------ 238| 0| forward = size_t(end - buf - 1); 239| 0| } 240| 15.7k| for (; k < forward; k++) { ------------------ | Branch (240:14): [True: 14.7k, False: 1.06k] ------------------ 241| 14.7k| uint16_t word = scalar::utf16::swap_if_needed(buf[k]); 242| 14.7k| if ((word & 0xFF80) == 0) { ------------------ | Branch (242:13): [True: 9.22k, False: 5.47k] ------------------ 243| 9.22k| *utf8_output++ = char(word); 244| 9.22k| } else if ((word & 0xF800) == 0) { ------------------ | Branch (244:20): [True: 486, False: 4.98k] ------------------ 245| 486| *utf8_output++ = char((word >> 6) | 0b11000000); 246| 486| *utf8_output++ = char((word & 0b111111) | 0b10000000); 247| 4.98k| } else if ((word & 0xF800) != 0xD800) { ------------------ | Branch (247:20): [True: 3.23k, False: 1.75k] ------------------ 248| 3.23k| *utf8_output++ = char((word >> 12) | 0b11100000); 249| 3.23k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 250| 3.23k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 251| 3.23k| } else { 252| | // must be a surrogate pair 253| 1.75k| uint16_t diff = uint16_t(word - 0xD800); 254| 1.75k| uint16_t next_word = 255| 1.75k| scalar::utf16::swap_if_needed(buf[k + 1]); 256| 1.75k| k++; 257| 1.75k| uint16_t diff2 = uint16_t(next_word - 0xDC00); 258| 1.75k| if ((diff | diff2) > 0x3FF) { ------------------ | Branch (258:15): [True: 64, False: 1.69k] ------------------ 259| 64| return std::make_pair(nullptr, utf8_output); 260| 64| } 261| 1.69k| uint32_t value = (diff << 10) + diff2 + 0x10000; 262| 1.69k| *utf8_output++ = char((value >> 18) | 0b11110000); 263| 1.69k| *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); 264| 1.69k| *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); 265| 1.69k| *utf8_output++ = char((value & 0b111111) | 0b10000000); 266| 1.69k| } 267| 14.7k| } 268| 1.06k| buf += k; 269| 1.06k| } 270| 4.31k| } // while 271| | 272| 13.0k| return std::make_pair(buf, utf8_output); 273| 13.1k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_125sse_convert_utf16_to_utf8ILNS_10endiannessE1EEENSt3__14pairIPKDsPcEES7_mS8_: 56| 13.1k|sse_convert_utf16_to_utf8(const char16_t *buf, size_t len, char *utf8_output) { 57| | 58| 13.1k| const char16_t *end = buf + len; 59| | 60| 13.1k| const __m128i v_0000 = _mm_setzero_si128(); 61| 13.1k| const __m128i v_f800 = _mm_set1_epi16((int16_t)0xf800); 62| 13.1k| const __m128i v_d800 = _mm_set1_epi16((int16_t)0xd800); 63| 13.1k| const size_t safety_margin = 64| 13.1k| 12; // to avoid overruns, see issue 65| | // https://github.com/simdutf/simdutf/issues/92 66| | 67| 21.6k| while (end - buf >= std::ptrdiff_t(16 + safety_margin)) { ------------------ | Branch (67:10): [True: 8.62k, False: 13.0k] ------------------ 68| 8.62k| __m128i in = _mm_loadu_si128((__m128i *)buf); 69| 8.62k| if (big_endian) { ------------------ | Branch (69:9): [True: 8.62k, Folded] ------------------ 70| 8.62k| const __m128i swap = 71| 8.62k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 72| 8.62k| in = _mm_shuffle_epi8(in, swap); 73| 8.62k| } 74| | // a single 16-bit UTF-16 word can yield 1, 2 or 3 UTF-8 bytes 75| 8.62k| const __m128i v_ff80 = _mm_set1_epi16((int16_t)0xff80); 76| 8.62k| if (_mm_testz_si128(in, v_ff80)) { // ASCII fast path!!!! ------------------ | Branch (76:9): [True: 3.06k, False: 5.56k] ------------------ 77| 3.06k| __m128i nextin = _mm_loadu_si128((__m128i *)buf + 1); 78| 3.06k| if (big_endian) { ------------------ | Branch (78:11): [True: 3.06k, Folded] ------------------ 79| 3.06k| const __m128i swap = 80| 3.06k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 81| 3.06k| nextin = _mm_shuffle_epi8(nextin, swap); 82| 3.06k| } 83| 3.06k| if (!_mm_testz_si128(nextin, v_ff80)) { ------------------ | Branch (83:11): [True: 325, False: 2.73k] ------------------ 84| | // 1. pack the bytes 85| | // obviously suboptimal. 86| 325| const __m128i utf8_packed = _mm_packus_epi16(in, in); 87| | // 2. store (16 bytes) 88| 325| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 89| | // 3. adjust pointers 90| 325| buf += 8; 91| 325| utf8_output += 8; 92| 325| in = nextin; 93| 2.73k| } else { 94| | // 1. pack the bytes 95| | // obviously suboptimal. 96| 2.73k| const __m128i utf8_packed = _mm_packus_epi16(in, nextin); 97| | // 2. store (16 bytes) 98| 2.73k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 99| | // 3. adjust pointers 100| 2.73k| buf += 16; 101| 2.73k| utf8_output += 16; 102| 2.73k| continue; // we are done for this round! 103| 2.73k| } 104| 3.06k| } 105| | 106| | // no bits set above 7th bit 107| 5.88k| const __m128i one_byte_bytemask = 108| 5.88k| _mm_cmpeq_epi16(_mm_and_si128(in, v_ff80), v_0000); 109| 5.88k| const uint16_t one_byte_bitmask = 110| 5.88k| static_cast(_mm_movemask_epi8(one_byte_bytemask)); 111| | 112| | // no bits set above 11th bit 113| 5.88k| const __m128i one_or_two_bytes_bytemask = 114| 5.88k| _mm_cmpeq_epi16(_mm_and_si128(in, v_f800), v_0000); 115| 5.88k| const uint16_t one_or_two_bytes_bitmask = 116| 5.88k| static_cast(_mm_movemask_epi8(one_or_two_bytes_bytemask)); 117| | 118| 5.88k| if (one_or_two_bytes_bitmask == 0xffff) { ------------------ | Branch (118:9): [True: 1.57k, False: 4.31k] ------------------ 119| 1.57k| internal::westmere::write_v_u16_11bits_to_utf8( 120| 1.57k| in, utf8_output, one_byte_bytemask, one_byte_bitmask); 121| 1.57k| buf += 8; 122| 1.57k| continue; 123| 1.57k| } 124| | 125| | // 1. Check if there are any surrogate word in the input chunk. 126| | // We have also deal with situation when there is a surrogate word 127| | // at the end of a chunk. 128| 4.31k| const __m128i surrogates_bytemask = 129| 4.31k| _mm_cmpeq_epi16(_mm_and_si128(in, v_f800), v_d800); 130| | 131| | // bitmask = 0x0000 if there are no surrogates 132| | // = 0xc000 if the last word is a surrogate 133| 4.31k| const uint16_t surrogates_bitmask = 134| 4.31k| static_cast(_mm_movemask_epi8(surrogates_bytemask)); 135| | // It might seem like checking for surrogates_bitmask == 0xc000 could help. 136| | // However, it is likely an uncommon occurrence. 137| 4.31k| if (surrogates_bitmask == 0x0000) { ------------------ | Branch (137:9): [True: 3.20k, False: 1.11k] ------------------ 138| | // case: code units from register produce either 1, 2 or 3 UTF-8 bytes 139| 3.20k| const __m128i dup_even = _mm_setr_epi16(0x0000, 0x0202, 0x0404, 0x0606, 140| 3.20k| 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); 141| | 142| | /* In this branch we handle three cases: 143| | 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - 144| | single UFT-8 byte 145| | 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - two 146| | UTF-8 bytes 147| | 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - 148| | three UTF-8 bytes 149| | 150| | We expand the input word (16-bit) into two code units (32-bit), thus 151| | we have room for four bytes. However, we need five distinct bit 152| | layouts. Note that the last byte in cases #2 and #3 is the same. 153| | 154| | We precompute byte 1 for case #1 and the common byte for cases #2 & #3 155| | in register t2. 156| | 157| | We precompute byte 1 for case #3 and -- **conditionally** -- precompute 158| | either byte 1 for case #2 or byte 2 for case #3. Note that they 159| | differ by exactly one bit. 160| | 161| | Finally from these two code units we build proper UTF-8 sequence, taking 162| | into account the case (i.e, the number of bytes to write). 163| | */ 164| | /** 165| | * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: 166| | * t2 => [0ccc|cccc] [10cc|cccc] 167| | * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) 168| | */ 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) 170| | // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] 171| 3.20k| const __m128i t0 = _mm_shuffle_epi8(in, dup_even); 172| | // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] 173| 3.20k| const __m128i t1 = _mm_and_si128(t0, simdutf_vec(0b0011111101111111)); ------------------ | | 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 174| | // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] 175| 3.20k| const __m128i t2 = _mm_or_si128(t1, simdutf_vec(0b1000000000000000)); ------------------ | | 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 176| | 177| | // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] 178| 3.20k| const __m128i s0 = _mm_srli_epi16(in, 4); 179| | // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] 180| 3.20k| const __m128i s1 = _mm_and_si128(s0, simdutf_vec(0b0000111111111100)); ------------------ | | 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 181| | // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] 182| 3.20k| const __m128i s2 = _mm_maddubs_epi16(s1, simdutf_vec(0x0140)); ------------------ | | 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 183| | // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] 184| 3.20k| const __m128i s3 = _mm_or_si128(s2, simdutf_vec(0b1100000011100000)); ------------------ | | 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 185| 3.20k| const __m128i m0 = _mm_andnot_si128(one_or_two_bytes_bytemask, 186| 3.20k| simdutf_vec(0b0100000000000000)); ------------------ | | 169| 3.20k|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 187| 3.20k| const __m128i s4 = _mm_xor_si128(s3, m0); 188| 3.20k|#undef simdutf_vec 189| | 190| | // 4. expand code units 16-bit => 32-bit 191| 3.20k| const __m128i out0 = _mm_unpacklo_epi16(t2, s4); 192| 3.20k| const __m128i out1 = _mm_unpackhi_epi16(t2, s4); 193| | 194| | // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle 195| 3.20k| const uint16_t mask = 196| 3.20k| (one_byte_bitmask & 0x5555) | (one_or_two_bytes_bitmask & 0xaaaa); 197| 3.20k| if (mask == 0) { ------------------ | Branch (197:11): [True: 2.08k, False: 1.11k] ------------------ 198| | // We only have three-byte code units. Use fast path. 199| 2.08k| const __m128i shuffle = _mm_setr_epi8(2, 3, 1, 6, 7, 5, 10, 11, 9, 14, 200| 2.08k| 15, 13, -1, -1, -1, -1); 201| 2.08k| const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle); 202| 2.08k| const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle); 203| 2.08k| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 204| 2.08k| utf8_output += 12; 205| 2.08k| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 206| 2.08k| utf8_output += 12; 207| 2.08k| buf += 8; 208| 2.08k| continue; 209| 2.08k| } 210| 1.11k| const uint8_t mask0 = uint8_t(mask); 211| | 212| 1.11k| const uint8_t *row0 = 213| 1.11k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; 214| 1.11k| const __m128i shuffle0 = _mm_loadu_si128((__m128i *)(row0 + 1)); 215| 1.11k| const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle0); 216| | 217| 1.11k| const uint8_t mask1 = static_cast(mask >> 8); 218| | 219| 1.11k| const uint8_t *row1 = 220| 1.11k| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; 221| 1.11k| const __m128i shuffle1 = _mm_loadu_si128((__m128i *)(row1 + 1)); 222| 1.11k| const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle1); 223| | 224| 1.11k| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 225| 1.11k| utf8_output += row0[0]; 226| 1.11k| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 227| 1.11k| utf8_output += row1[0]; 228| | 229| 1.11k| buf += 8; 230| | // surrogate pair(s) in a register 231| 1.11k| } else { 232| | // Let us do a scalar fallback. 233| | // It may seem wasteful to use scalar code, but being efficient with SIMD 234| | // in the presence of surrogate pairs may require non-trivial tables. 235| 1.11k| size_t forward = 15; 236| 1.11k| size_t k = 0; 237| 1.11k| if (size_t(end - buf) < forward + 1) { ------------------ | Branch (237:11): [True: 0, False: 1.11k] ------------------ 238| 0| forward = size_t(end - buf - 1); 239| 0| } 240| 15.4k| for (; k < forward; k++) { ------------------ | Branch (240:14): [True: 14.4k, False: 1.04k] ------------------ 241| 14.4k| uint16_t word = scalar::utf16::swap_if_needed(buf[k]); 242| 14.4k| if ((word & 0xFF80) == 0) { ------------------ | Branch (242:13): [True: 9.21k, False: 5.21k] ------------------ 243| 9.21k| *utf8_output++ = char(word); 244| 9.21k| } else if ((word & 0xF800) == 0) { ------------------ | Branch (244:20): [True: 524, False: 4.68k] ------------------ 245| 524| *utf8_output++ = char((word >> 6) | 0b11000000); 246| 524| *utf8_output++ = char((word & 0b111111) | 0b10000000); 247| 4.68k| } else if ((word & 0xF800) != 0xD800) { ------------------ | Branch (247:20): [True: 3.05k, False: 1.63k] ------------------ 248| 3.05k| *utf8_output++ = char((word >> 12) | 0b11100000); 249| 3.05k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 250| 3.05k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 251| 3.05k| } else { 252| | // must be a surrogate pair 253| 1.63k| uint16_t diff = uint16_t(word - 0xD800); 254| 1.63k| uint16_t next_word = 255| 1.63k| scalar::utf16::swap_if_needed(buf[k + 1]); 256| 1.63k| k++; 257| 1.63k| uint16_t diff2 = uint16_t(next_word - 0xDC00); 258| 1.63k| if ((diff | diff2) > 0x3FF) { ------------------ | Branch (258:15): [True: 67, False: 1.56k] ------------------ 259| 67| return std::make_pair(nullptr, utf8_output); 260| 67| } 261| 1.56k| uint32_t value = (diff << 10) + diff2 + 0x10000; 262| 1.56k| *utf8_output++ = char((value >> 18) | 0b11110000); 263| 1.56k| *utf8_output++ = char(((value >> 12) & 0b111111) | 0b10000000); 264| 1.56k| *utf8_output++ = char(((value >> 6) & 0b111111) | 0b10000000); 265| 1.56k| *utf8_output++ = char((value & 0b111111) | 0b10000000); 266| 1.56k| } 267| 14.4k| } 268| 1.04k| buf += k; 269| 1.04k| } 270| 4.31k| } // while 271| | 272| 13.0k| return std::make_pair(buf, utf8_output); 273| 13.1k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_127sse_convert_utf32_to_latin1EPKDimPc: 3| 6.85k| char *latin1_output) { 4| 6.85k| const size_t rounded_len = len & ~0xF; // Round down to nearest multiple of 16 5| | 6| 6.85k| __m128i high_bytes_mask = _mm_set1_epi32(0xFFFFFF00); 7| 6.85k| __m128i shufmask = 8| 6.85k| _mm_set_epi8(-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 12, 8, 4, 0); 9| | 10| 27.0k| for (size_t i = 0; i < rounded_len; i += 16) { ------------------ | Branch (10:22): [True: 20.2k, False: 6.85k] ------------------ 11| 20.2k| __m128i in1 = _mm_loadu_si128((__m128i *)buf); 12| 20.2k| __m128i in2 = _mm_loadu_si128((__m128i *)(buf + 4)); 13| 20.2k| __m128i in3 = _mm_loadu_si128((__m128i *)(buf + 8)); 14| 20.2k| __m128i in4 = _mm_loadu_si128((__m128i *)(buf + 12)); 15| | 16| 20.2k| __m128i check_combined = _mm_or_si128(in1, in2); 17| 20.2k| check_combined = _mm_or_si128(check_combined, in3); 18| 20.2k| check_combined = _mm_or_si128(check_combined, in4); 19| | 20| 20.2k| if (!_mm_testz_si128(check_combined, high_bytes_mask)) { ------------------ | Branch (20:9): [True: 0, False: 20.2k] ------------------ 21| 0| return std::make_pair(nullptr, latin1_output); 22| 0| } 23| 20.2k| __m128i pack1 = _mm_unpacklo_epi32(_mm_shuffle_epi8(in1, shufmask), 24| 20.2k| _mm_shuffle_epi8(in2, shufmask)); 25| 20.2k| __m128i pack2 = _mm_unpacklo_epi32(_mm_shuffle_epi8(in3, shufmask), 26| 20.2k| _mm_shuffle_epi8(in4, shufmask)); 27| 20.2k| __m128i pack = _mm_unpacklo_epi64(pack1, pack2); 28| 20.2k| _mm_storeu_si128((__m128i *)latin1_output, pack); 29| 20.2k| latin1_output += 16; 30| 20.2k| buf += 16; 31| 20.2k| } 32| | 33| 6.85k| return std::make_pair(buf, latin1_output); 34| 6.85k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_125sse_convert_utf32_to_utf8EPKDimPc: 2| 6.26k|sse_convert_utf32_to_utf8(const char32_t *buf, size_t len, char *utf8_output) { 3| 6.26k| const char32_t *end = buf + len; 4| | 5| 6.26k| const __m128i v_0000 = _mm_setzero_si128(); //__m128 = 128 bits 6| 6.26k| const __m128i v_f800 = _mm_set1_epi16((uint16_t)0xf800); // 1111 1000 0000 7| | // 0000 8| 6.26k| const __m128i v_c080 = _mm_set1_epi16((uint16_t)0xc080); // 1100 0000 1000 9| | // 0000 10| 6.26k| const __m128i v_ff80 = _mm_set1_epi16((uint16_t)0xff80); // 1111 1111 1000 11| | // 0000 12| 6.26k| const __m128i v_ffff0000 = _mm_set1_epi32( 13| 6.26k| (uint32_t)0xffff0000); // 1111 1111 1111 1111 0000 0000 0000 0000 14| 6.26k| const __m128i v_7fffffff = _mm_set1_epi32( 15| 6.26k| (uint32_t)0x7fffffff); // 0111 1111 1111 1111 1111 1111 1111 1111 16| 6.26k| __m128i running_max = _mm_setzero_si128(); 17| 6.26k| __m128i forbidden_bytemask = _mm_setzero_si128(); 18| 6.26k| const size_t safety_margin = 19| 6.26k| 12; // to avoid overruns, see issue 20| | // https://github.com/simdutf/simdutf/issues/92 21| | 22| 11.3k| while (end - buf >= ------------------ | Branch (22:10): [True: 5.09k, False: 6.26k] ------------------ 23| 11.3k| std::ptrdiff_t( 24| 11.3k| 16 + safety_margin)) { // buf is a char32_t pointer, each char32_t 25| | // has 4 bytes or 32 bits, thus buf + 16 * 26| | // char_32t = 512 bits = 64 bytes 27| | // We load two 16 bytes registers for a total of 32 bytes or 16 characters. 28| 5.09k| __m128i in = _mm_loadu_si128((__m128i *)buf); 29| 5.09k| __m128i nextin = _mm_loadu_si128( 30| 5.09k| (__m128i *)buf + 1); // These two values can hold only 8 UTF32 chars 31| 5.09k| running_max = _mm_max_epu32( 32| 5.09k| _mm_max_epu32(in, running_max), // take element-wise max char32_t from 33| | // in and running_max vector 34| 5.09k| nextin); // and take element-wise max element from nextin and 35| | // running_max vector 36| | 37| | // Pack 32-bit UTF-32 code units to 16-bit UTF-16 code units with unsigned 38| | // saturation 39| 5.09k| __m128i in_16 = _mm_packus_epi32( 40| 5.09k| _mm_and_si128(in, v_7fffffff), 41| 5.09k| _mm_and_si128( 42| 5.09k| nextin, 43| 5.09k| v_7fffffff)); // in this context pack the two __m128 into a single 44| | // By ensuring the highest bit is set to 0(&v_7fffffff), we are making sure 45| | // all values are interpreted as non-negative, or specifically, the values 46| | // are within the range of valid Unicode code points. remember : having 47| | // leading byte 0 means a positive number by the two complements system. 48| | // Unicode is well beneath the range where you'll start getting issues so 49| | // that's OK. 50| | 51| | // Try to apply UTF-16 => UTF-8 from ./sse_convert_utf16_to_utf8.cpp 52| | 53| | // Check for ASCII fast path 54| | 55| | // ASCII fast path!!!! 56| | // We eagerly load another 32 bytes, hoping that they will be ASCII too. 57| | // The intuition is that we try to collect 16 ASCII characters which 58| | // requires a total of 64 bytes of input. If we fail, we just pass thirdin 59| | // and fourthin as our new inputs. 60| 5.09k| if (_mm_testz_si128(in_16, v_ff80)) { // if the first two blocks are ASCII ------------------ | Branch (60:9): [True: 2.89k, False: 2.19k] ------------------ 61| 2.89k| __m128i thirdin = _mm_loadu_si128((__m128i *)buf + 2); 62| 2.89k| __m128i fourthin = _mm_loadu_si128((__m128i *)buf + 3); 63| 2.89k| running_max = _mm_max_epu32( 64| 2.89k| _mm_max_epu32(thirdin, running_max), 65| 2.89k| fourthin); // take the running max of all 4 vectors thus far 66| 2.89k| __m128i nextin_16 = _mm_packus_epi32( 67| 2.89k| _mm_and_si128(thirdin, v_7fffffff), 68| 2.89k| _mm_and_si128(fourthin, 69| 2.89k| v_7fffffff)); // pack into 1 vector, now you have two 70| 2.89k| if (!_mm_testz_si128( ------------------ | Branch (70:11): [True: 285, False: 2.61k] ------------------ 71| 2.89k| nextin_16, 72| 2.89k| v_ff80)) { // checks if the second packed vector is ASCII, if not: 73| | // 1. pack the bytes 74| | // obviously suboptimal. 75| 285| const __m128i utf8_packed = _mm_packus_epi16( 76| 285| in_16, in_16); // creates two copy of in_16 in 1 vector 77| | // 2. store (16 bytes) 78| 285| _mm_storeu_si128((__m128i *)utf8_output, 79| 285| utf8_packed); // put them into the output 80| | // 3. adjust pointers 81| 285| buf += 8; // the char32_t buffer pointer goes up 8 char32_t chars* 32 82| | // bits = 256 bits 83| 285| utf8_output += 84| 285| 8; // same with output, e.g. lift the first two blocks alone. 85| | // Proceed with next input 86| 285| in_16 = nextin_16; 87| | // We need to update in and nextin because they are used later. 88| 285| in = thirdin; 89| 285| nextin = fourthin; 90| 2.61k| } else { 91| | // 1. pack the bytes 92| 2.61k| const __m128i utf8_packed = _mm_packus_epi16(in_16, nextin_16); 93| | // 2. store (16 bytes) 94| 2.61k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 95| | // 3. adjust pointers 96| 2.61k| buf += 16; 97| 2.61k| utf8_output += 16; 98| 2.61k| continue; // we are done for this round! 99| 2.61k| } 100| 2.89k| } 101| | 102| | // no bits set above 7th bit -- find out all the ASCII characters 103| 2.48k| const __m128i one_byte_bytemask = 104| 2.48k| _mm_cmpeq_epi16( // this takes four bytes at a time and compares: 105| 2.48k| _mm_and_si128(in_16, v_ff80), // the vector that get only the first 106| | // 9 bits of each 16-bit/2-byte units 107| 2.48k| v_0000 // 108| 2.48k| ); // they should be all zero if they are ASCII. E.g. ASCII in UTF32 is 109| | // of format 0000 0000 0000 0XXX XXXX 110| | // _mm_cmpeq_epi16 should now return a 1111 1111 1111 1111 for equals, and 111| | // 0000 0000 0000 0000 if not for each 16-bit/2-byte units 112| 2.48k| const uint16_t one_byte_bitmask = static_cast(_mm_movemask_epi8( 113| 2.48k| one_byte_bytemask)); // collect the MSB from previous vector and put 114| | // them into uint16_t mas 115| | 116| | // no bits set above 11th bit 117| 2.48k| const __m128i one_or_two_bytes_bytemask = 118| 2.48k| _mm_cmpeq_epi16(_mm_and_si128(in_16, v_f800), v_0000); 119| 2.48k| const uint16_t one_or_two_bytes_bitmask = 120| 2.48k| static_cast(_mm_movemask_epi8(one_or_two_bytes_bytemask)); 121| | 122| 2.48k| if (one_or_two_bytes_bitmask == 0xffff) { ------------------ | Branch (122:9): [True: 1.38k, False: 1.10k] ------------------ 123| | // case: all code units either produce 1 or 2 UTF-8 bytes (at least one 124| | // produces 2 bytes) 125| | // 1. prepare 2-byte values 126| | // input 16-bit word : [0000|0aaa|aabb|bbbb] x 8 127| | // expected output : [110a|aaaa|10bb|bbbb] x 8 128| 1.38k| const __m128i v_1f00 = 129| 1.38k| _mm_set1_epi16((int16_t)0x1f00); // 0001 1111 0000 0000 130| 1.38k| const __m128i v_003f = 131| 1.38k| _mm_set1_epi16((int16_t)0x003f); // 0000 0000 0011 1111 132| | 133| | // t0 = [000a|aaaa|bbbb|bb00] 134| 1.38k| const __m128i t0 = _mm_slli_epi16(in_16, 2); // shift packed vector by two 135| | // t1 = [000a|aaaa|0000|0000] 136| 1.38k| const __m128i t1 = _mm_and_si128(t0, v_1f00); // potential first utf8 byte 137| | // t2 = [0000|0000|00bb|bbbb] 138| 1.38k| const __m128i t2 = 139| 1.38k| _mm_and_si128(in_16, v_003f); // potential second utf8 byte 140| | // t3 = [000a|aaaa|00bb|bbbb] 141| 1.38k| const __m128i t3 = 142| 1.38k| _mm_or_si128(t1, t2); // first and second potential utf8 byte together 143| | // t4 = [110a|aaaa|10bb|bbbb] 144| 1.38k| const __m128i t4 = _mm_or_si128( 145| 1.38k| t3, 146| 1.38k| v_c080); // t3 | 1100 0000 1000 0000 = full potential 2-byte utf8 unit 147| | 148| | // 2. merge ASCII and 2-byte codewords 149| 1.38k| const __m128i utf8_unpacked = 150| 1.38k| _mm_blendv_epi8(t4, in_16, one_byte_bytemask); 151| | 152| | // 3. prepare bitmask for 8-bit lookup 153| | // one_byte_bitmask = hhggffeeddccbbaa -- the bits are doubled (h - 154| | // MSB, a - LSB) 155| 1.38k| const uint16_t m0 = one_byte_bitmask & 0x5555; // m0 = 0h0g0f0e0d0c0b0a 156| 1.38k| const uint16_t m1 = 157| 1.38k| static_cast(m0 >> 7); // m1 = 00000000h0g0f0e0 158| 1.38k| const uint8_t m2 = 159| 1.38k| static_cast((m0 | m1) & 0xff); // m2 = hdgcfbea 160| | // 4. pack the bytes 161| 1.38k| const uint8_t *row = 162| 1.38k| &simdutf::tables::utf16_to_utf8::pack_1_2_utf8_bytes[m2][0]; 163| 1.38k| const __m128i shuffle = _mm_loadu_si128((__m128i *)(row + 1)); 164| 1.38k| const __m128i utf8_packed = _mm_shuffle_epi8(utf8_unpacked, shuffle); 165| | 166| | // 5. store bytes 167| 1.38k| _mm_storeu_si128((__m128i *)utf8_output, utf8_packed); 168| | 169| | // 6. adjust pointers 170| 1.38k| buf += 8; 171| 1.38k| utf8_output += row[0]; 172| 1.38k| continue; 173| 1.38k| } 174| | 175| | // Check for overflow in packing 176| | 177| 1.10k| const __m128i saturation_bytemask = _mm_cmpeq_epi32( 178| 1.10k| _mm_and_si128(_mm_or_si128(in, nextin), v_ffff0000), v_0000); 179| 1.10k| const uint32_t saturation_bitmask = 180| 1.10k| static_cast(_mm_movemask_epi8(saturation_bytemask)); 181| 1.10k| if (saturation_bitmask == 0xffff) { ------------------ | Branch (181:9): [True: 377, False: 723] ------------------ 182| | // case: code units from register produce either 1, 2 or 3 UTF-8 bytes 183| 377| const __m128i v_d800 = _mm_set1_epi16((uint16_t)0xd800); 184| 377| forbidden_bytemask = 185| 377| _mm_or_si128(forbidden_bytemask, 186| 377| _mm_cmpeq_epi16(_mm_and_si128(in_16, v_f800), v_d800)); 187| | 188| 377| const __m128i dup_even = _mm_setr_epi16(0x0000, 0x0202, 0x0404, 0x0606, 189| 377| 0x0808, 0x0a0a, 0x0c0c, 0x0e0e); 190| | 191| | /* In this branch we handle three cases: 192| | 1. [0000|0000|0ccc|cccc] => [0ccc|cccc] - 193| | single UFT-8 byte 194| | 2. [0000|0bbb|bbcc|cccc] => [110b|bbbb], [10cc|cccc] - 195| | two UTF-8 bytes 196| | 3. [aaaa|bbbb|bbcc|cccc] => [1110|aaaa], [10bb|bbbb], [10cc|cccc] - 197| | three UTF-8 bytes 198| | 199| | We expand the input word (16-bit) into two code units (32-bit), thus 200| | we have room for four bytes. However, we need five distinct bit 201| | layouts. Note that the last byte in cases #2 and #3 is the same. 202| | 203| | We precompute byte 1 for case #1 and the common byte for cases #2 & #3 204| | in register t2. 205| | 206| | We precompute byte 1 for case #3 and -- **conditionally** -- precompute 207| | either byte 1 for case #2 or byte 2 for case #3. Note that they 208| | differ by exactly one bit. 209| | 210| | Finally from these two code units we build proper UTF-8 sequence, taking 211| | into account the case (i.e, the number of bytes to write). 212| | */ 213| | /** 214| | * Given [aaaa|bbbb|bbcc|cccc] our goal is to produce: 215| | * t2 => [0ccc|cccc] [10cc|cccc] 216| | * s4 => [1110|aaaa] ([110b|bbbb] OR [10bb|bbbb]) 217| | */ 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) 219| | // [aaaa|bbbb|bbcc|cccc] => [bbcc|cccc|bbcc|cccc] 220| 377| const __m128i t0 = _mm_shuffle_epi8(in_16, dup_even); 221| | // [bbcc|cccc|bbcc|cccc] => [00cc|cccc|0bcc|cccc] 222| 377| const __m128i t1 = _mm_and_si128(t0, simdutf_vec(0b0011111101111111)); ------------------ | | 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 223| | // [00cc|cccc|0bcc|cccc] => [10cc|cccc|0bcc|cccc] 224| 377| const __m128i t2 = _mm_or_si128(t1, simdutf_vec(0b1000000000000000)); ------------------ | | 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 225| | 226| | // [aaaa|bbbb|bbcc|cccc] => [0000|aaaa|bbbb|bbcc] 227| 377| const __m128i s0 = _mm_srli_epi16(in_16, 4); 228| | // [0000|aaaa|bbbb|bbcc] => [0000|aaaa|bbbb|bb00] 229| 377| const __m128i s1 = _mm_and_si128(s0, simdutf_vec(0b0000111111111100)); ------------------ | | 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 230| | // [0000|aaaa|bbbb|bb00] => [00bb|bbbb|0000|aaaa] 231| 377| const __m128i s2 = _mm_maddubs_epi16(s1, simdutf_vec(0x0140)); ------------------ | | 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 232| | // [00bb|bbbb|0000|aaaa] => [11bb|bbbb|1110|aaaa] 233| 377| const __m128i s3 = _mm_or_si128(s2, simdutf_vec(0b1100000011100000)); ------------------ | | 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 234| 377| const __m128i m0 = _mm_andnot_si128(one_or_two_bytes_bytemask, 235| 377| simdutf_vec(0b0100000000000000)); ------------------ | | 218| 377|#define simdutf_vec(x) _mm_set1_epi16(static_cast(x)) ------------------ 236| 377| const __m128i s4 = _mm_xor_si128(s3, m0); 237| 377|#undef simdutf_vec 238| | 239| | // 4. expand code units 16-bit => 32-bit 240| 377| const __m128i out0 = _mm_unpacklo_epi16(t2, s4); 241| 377| const __m128i out1 = _mm_unpackhi_epi16(t2, s4); 242| | 243| | // 5. compress 32-bit code units into 1, 2 or 3 bytes -- 2 x shuffle 244| 377| const uint16_t mask = 245| 377| (one_byte_bitmask & 0x5555) | (one_or_two_bytes_bitmask & 0xaaaa); 246| 377| if (mask == 0) { ------------------ | Branch (246:11): [True: 85, False: 292] ------------------ 247| | // We only have three-byte code units. Use fast path. 248| 85| const __m128i shuffle = _mm_setr_epi8(2, 3, 1, 6, 7, 5, 10, 11, 9, 14, 249| 85| 15, 13, -1, -1, -1, -1); 250| 85| const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle); 251| 85| const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle); 252| 85| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 253| 85| utf8_output += 12; 254| 85| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 255| 85| utf8_output += 12; 256| 85| buf += 8; 257| 85| continue; 258| 85| } 259| 292| const uint8_t mask0 = uint8_t(mask); 260| | 261| 292| const uint8_t *row0 = 262| 292| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask0][0]; 263| 292| const __m128i shuffle0 = _mm_loadu_si128((__m128i *)(row0 + 1)); 264| 292| const __m128i utf8_0 = _mm_shuffle_epi8(out0, shuffle0); 265| | 266| 292| const uint8_t mask1 = static_cast(mask >> 8); 267| | 268| 292| const uint8_t *row1 = 269| 292| &simdutf::tables::utf16_to_utf8::pack_1_2_3_utf8_bytes[mask1][0]; 270| 292| const __m128i shuffle1 = _mm_loadu_si128((__m128i *)(row1 + 1)); 271| 292| const __m128i utf8_1 = _mm_shuffle_epi8(out1, shuffle1); 272| | 273| 292| _mm_storeu_si128((__m128i *)utf8_output, utf8_0); 274| 292| utf8_output += row0[0]; 275| 292| _mm_storeu_si128((__m128i *)utf8_output, utf8_1); 276| 292| utf8_output += row1[0]; 277| | 278| 292| buf += 8; 279| 723| } else { 280| | // case: at least one 32-bit word produce a surrogate pair in UTF-16 <=> 281| | // will produce four UTF-8 bytes Let us do a scalar fallback. It may seem 282| | // wasteful to use scalar code, but being efficient with SIMD in the 283| | // presence of surrogate pairs may require non-trivial tables. 284| 723| size_t forward = 15; 285| 723| size_t k = 0; 286| 723| if (size_t(end - buf) < forward + 1) { ------------------ | Branch (286:11): [True: 0, False: 723] ------------------ 287| 0| forward = size_t(end - buf - 1); 288| 0| } 289| 11.5k| for (; k < forward; k++) { ------------------ | Branch (289:14): [True: 10.8k, False: 723] ------------------ 290| 10.8k| uint32_t word = buf[k]; 291| 10.8k| if ((word & 0xFFFFFF80) == 0) { ------------------ | Branch (291:13): [True: 8.86k, False: 1.98k] ------------------ 292| 8.86k| *utf8_output++ = char(word); 293| 8.86k| } else if ((word & 0xFFFFF800) == 0) { ------------------ | Branch (293:20): [True: 377, False: 1.60k] ------------------ 294| 377| *utf8_output++ = char((word >> 6) | 0b11000000); 295| 377| *utf8_output++ = char((word & 0b111111) | 0b10000000); 296| 1.60k| } else if ((word & 0xFFFF0000) == 0) { ------------------ | Branch (296:20): [True: 486, False: 1.12k] ------------------ 297| 486| if (word >= 0xD800 && word <= 0xDFFF) { ------------------ | Branch (297:15): [True: 250, False: 236] | Branch (297:33): [True: 0, False: 250] ------------------ 298| 0| return std::make_pair(nullptr, utf8_output); 299| 0| } 300| 486| *utf8_output++ = char((word >> 12) | 0b11100000); 301| 486| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 302| 486| *utf8_output++ = char((word & 0b111111) | 0b10000000); 303| 1.12k| } else { 304| 1.12k| if (word > 0x10FFFF) { ------------------ | Branch (304:15): [True: 0, False: 1.12k] ------------------ 305| 0| return std::make_pair(nullptr, utf8_output); 306| 0| } 307| 1.12k| *utf8_output++ = char((word >> 18) | 0b11110000); 308| 1.12k| *utf8_output++ = char(((word >> 12) & 0b111111) | 0b10000000); 309| 1.12k| *utf8_output++ = char(((word >> 6) & 0b111111) | 0b10000000); 310| 1.12k| *utf8_output++ = char((word & 0b111111) | 0b10000000); 311| 1.12k| } 312| 10.8k| } 313| 723| buf += k; 314| 723| } 315| 1.10k| } // while 316| | 317| | // check for invalid input 318| 6.26k| const __m128i v_10ffff = _mm_set1_epi32((uint32_t)0x10ffff); 319| 6.26k| if (static_cast(_mm_movemask_epi8(_mm_cmpeq_epi32( ------------------ | Branch (319:7): [True: 0, False: 6.26k] ------------------ 320| 6.26k| _mm_max_epu32(running_max, v_10ffff), v_10ffff))) != 0xffff) { 321| 0| return std::make_pair(nullptr, utf8_output); 322| 0| } 323| | 324| 6.26k| if (static_cast(_mm_movemask_epi8(forbidden_bytemask)) != 0) { ------------------ | Branch (324:7): [True: 0, False: 6.26k] ------------------ 325| 0| return std::make_pair(nullptr, utf8_output); 326| 0| } 327| | 328| 6.26k| return std::make_pair(buf, utf8_output); 329| 6.26k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_129convert_masked_utf8_to_latin1EPKcmRPc: 9| 39.1k| char *&latin1_output) { 10| | // we use an approach where we try to process up to 12 input bytes. 11| | // Why 12 input bytes and not 16? Because we are concerned with the size of 12| | // the lookup tables. Also 12 is nicely divisible by two and three. 13| | // 14| | // 15| | // Optimization note: our main path below is load-latency dependent. Thus it 16| | // is maybe beneficial to have fast paths that depend on branch prediction but 17| | // have less latency. This results in more instructions but, potentially, also 18| | // higher speeds. 19| | // 20| 39.1k| const __m128i in = _mm_loadu_si128((__m128i *)input); 21| 39.1k| const uint16_t input_utf8_end_of_code_point_mask = 22| 39.1k| utf8_end_of_code_point_mask & 23| 39.1k| 0xfff; // we are only processing 12 bytes in case it is not all ASCII 24| 39.1k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (24:7): [True: 260, False: 38.9k] ------------------ 25| | // We process the data in chunks of 12 bytes. 26| 260| _mm_storeu_si128(reinterpret_cast<__m128i *>(latin1_output), in); 27| 260| latin1_output += 12; // We wrote 12 characters. 28| 260| return 12; // We consumed 12 bytes. 29| 260| } 30| | /// We do not have a fast path available, so we fallback. 31| 38.9k| const uint8_t idx = 32| 38.9k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; 33| 38.9k| const uint8_t consumed = 34| 38.9k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; 35| | // this indicates an invalid input: 36| 38.9k| if (idx >= 64) { ------------------ | Branch (36:7): [True: 5.76k, False: 33.1k] ------------------ 37| 5.76k| return consumed; 38| 5.76k| } 39| | // Here we should have (idx < 64), if not, there is a bug in the validation or 40| | // elsewhere. SIX (6) input code-code units this is a relatively easy scenario 41| | // we process SIX (6) input code-code units. The max length in bytes of six 42| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 43| | // processors where pdep/pext is fast, we might be able to use a small lookup 44| | // table. 45| 33.1k| const __m128i sh = 46| 33.1k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 47| 33.1k| const __m128i perm = _mm_shuffle_epi8(in, sh); 48| 33.1k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 49| 33.1k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 50| 33.1k| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 51| 33.1k| const __m128i latin1_packed = _mm_packus_epi16(composed, composed); 52| | // writing 8 bytes even though we only care about the first 6 bytes. 53| | // performance note: it would be faster to use _mm_storeu_si128, we should 54| | // investigate. 55| 33.1k| _mm_storel_epi64((__m128i *)latin1_output, latin1_packed); 56| 33.1k| latin1_output += 6; // We wrote 6 bytes. 57| 33.1k| return consumed; 58| 38.9k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_128convert_masked_utf8_to_utf16ILNS_10endiannessE0EEEmPKcmRPDs: 10| 21.1k| char16_t *&utf16_output) { 11| | // we use an approach where we try to process up to 12 input bytes. 12| | // Why 12 input bytes and not 16? Because we are concerned with the size of 13| | // the lookup tables. Also 12 is nicely divisible by two and three. 14| | // 15| | // 16| | // Optimization note: our main path below is load-latency dependent. Thus it 17| | // is maybe beneficial to have fast paths that depend on branch prediction but 18| | // have less latency. This results in more instructions but, potentially, also 19| | // higher speeds. 20| | // 21| | // We first try a few fast paths. 22| 21.1k| const __m128i swap = 23| 21.1k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 24| 21.1k| const __m128i in = _mm_loadu_si128((__m128i *)input); 25| 21.1k| const uint16_t input_utf8_end_of_code_point_mask = 26| 21.1k| utf8_end_of_code_point_mask & 0xfff; 27| 21.1k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (27:7): [True: 106, False: 21.0k] ------------------ 28| | // We process the data in chunks of 12 bytes. 29| | // Note: using 16 bytes is unsafe, see issue_ossfuzz_71218 30| 106| __m128i ascii_first = _mm_cvtepu8_epi16(in); 31| 106| __m128i ascii_second = _mm_cvtepu8_epi16(_mm_srli_si128(in, 8)); 32| 106| if (big_endian) { ------------------ | Branch (32:9): [Folded, False: 106] ------------------ 33| 0| ascii_first = _mm_shuffle_epi8(ascii_first, swap); 34| 0| ascii_second = _mm_shuffle_epi8(ascii_second, swap); 35| 0| } 36| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf16_output), ascii_first); 37| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf16_output + 8), 38| 106| ascii_second); 39| 106| utf16_output += 12; // We wrote 12 16-bit characters. 40| 106| return 12; // We consumed 12 bytes. 41| 106| } 42| 21.0k| if (((utf8_end_of_code_point_mask & 0xFFFF) == 0xaaaa)) { ------------------ | Branch (42:7): [True: 449, False: 20.6k] ------------------ 43| | // We want to take 8 2-byte UTF-8 code units and turn them into 8 2-byte 44| | // UTF-16 code units. There is probably a more efficient sequence, but the 45| | // following might do. 46| 449| const __m128i sh = 47| 449| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 48| 449| const __m128i perm = _mm_shuffle_epi8(in, sh); 49| 449| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 50| 449| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 51| 449| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 52| 449| if (big_endian) ------------------ | Branch (52:9): [Folded, False: 449] ------------------ 53| 0| composed = _mm_shuffle_epi8(composed, swap); 54| 449| _mm_storeu_si128((__m128i *)utf16_output, composed); 55| 449| utf16_output += 8; // We wrote 16 bytes, 8 code points. 56| 449| return 16; 57| 449| } 58| 20.6k| if (input_utf8_end_of_code_point_mask == 0x924) { ------------------ | Branch (58:7): [True: 5.10k, False: 15.5k] ------------------ 59| | // We want to take 4 3-byte UTF-8 code units and turn them into 4 2-byte 60| | // UTF-16 code units. There is probably a more efficient sequence, but the 61| | // following might do. 62| 5.10k| const __m128i sh = 63| 5.10k| _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); 64| 5.10k| const __m128i perm = _mm_shuffle_epi8(in, sh); 65| 5.10k| const __m128i ascii = 66| 5.10k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 67| 5.10k| const __m128i middlebyte = 68| 5.10k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 69| 5.10k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 70| 5.10k| const __m128i highbyte = 71| 5.10k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 72| 5.10k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 73| 5.10k| const __m128i composed = 74| 5.10k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 75| 5.10k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 76| 5.10k| if (big_endian) ------------------ | Branch (76:9): [Folded, False: 5.10k] ------------------ 77| 0| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 78| 5.10k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 79| 5.10k| utf16_output += 4; 80| 5.10k| return 12; 81| 5.10k| } 82| | /// We do not have a fast path available, so we fallback. 83| | 84| 15.5k| const uint8_t idx = 85| 15.5k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; 86| 15.5k| const uint8_t consumed = 87| 15.5k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; 88| 15.5k| if (idx < 64) { ------------------ | Branch (88:7): [True: 8.36k, False: 7.15k] ------------------ 89| | // SIX (6) input code-code units 90| | // this is a relatively easy scenario 91| | // we process SIX (6) input code-code units. The max length in bytes of six 92| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 93| | // processors where pdep/pext is fast, we might be able to use a small 94| | // lookup table. 95| 8.36k| const __m128i sh = 96| 8.36k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 97| 8.36k| const __m128i perm = _mm_shuffle_epi8(in, sh); 98| 8.36k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 99| 8.36k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 100| 8.36k| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 101| 8.36k| if (big_endian) ------------------ | Branch (101:9): [Folded, False: 8.36k] ------------------ 102| 0| composed = _mm_shuffle_epi8(composed, swap); 103| 8.36k| _mm_storeu_si128((__m128i *)utf16_output, composed); 104| 8.36k| utf16_output += 6; // We wrote 12 bytes, 6 code points. 105| 8.36k| } else if (idx < 145) { ------------------ | Branch (105:14): [True: 2.23k, False: 4.92k] ------------------ 106| | // FOUR (4) input code-code units 107| 2.23k| const __m128i sh = 108| 2.23k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 109| 2.23k| const __m128i perm = _mm_shuffle_epi8(in, sh); 110| 2.23k| const __m128i ascii = 111| 2.23k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 112| 2.23k| const __m128i middlebyte = 113| 2.23k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 114| 2.23k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 115| 2.23k| const __m128i highbyte = 116| 2.23k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 117| 2.23k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 118| 2.23k| const __m128i composed = 119| 2.23k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 120| 2.23k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 121| 2.23k| if (big_endian) ------------------ | Branch (121:9): [Folded, False: 2.23k] ------------------ 122| 0| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 123| 2.23k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 124| 2.23k| utf16_output += 4; 125| 4.92k| } else if (idx < 209) { ------------------ | Branch (125:14): [True: 4.70k, False: 221] ------------------ 126| | // TWO (2) input code-code units 127| | ////////////// 128| | // There might be garbage inputs where a leading byte mascarades as a 129| | // four-byte leading byte (by being followed by 3 continuation byte), but is 130| | // not greater than 0xf0. This could trigger a buffer overflow if we only 131| | // counted leading bytes of the form 0xf0 as generating surrogate pairs, 132| | // without further UTF-8 validation. Thus we must be careful to ensure that 133| | // only leading bytes at least as large as 0xf0 generate surrogate pairs. We 134| | // do as at the cost of an extra mask. 135| | ///////////// 136| 4.70k| const __m128i sh = 137| 4.70k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 138| 4.70k| const __m128i perm = _mm_shuffle_epi8(in, sh); 139| 4.70k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); 140| 4.70k| const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); 141| 4.70k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 142| 4.70k| __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); 143| | // correct for spurious high bit 144| 4.70k| const __m128i correct = 145| 4.70k| _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); 146| 4.70k| middlehighbyte = _mm_xor_si128(correct, middlehighbyte); 147| 4.70k| const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); 148| | // We deliberately carry the leading four bits in highbyte if they are 149| | // present, we remove them later when computing hightenbits. 150| 4.70k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0xff000000)); 151| 4.70k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); 152| | // When we need to generate a surrogate pair (leading byte > 0xF0), then 153| | // the corresponding 32-bit value in 'composed' will be greater than 154| | // > (0xff00000>>6) or > 0x3c00000. This can be used later to identify the 155| | // location of the surrogate pairs. 156| 4.70k| const __m128i composed = 157| 4.70k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), 158| 4.70k| _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); 159| 4.70k| const __m128i composedminus = 160| 4.70k| _mm_sub_epi32(composed, _mm_set1_epi32(0x10000)); 161| 4.70k| const __m128i lowtenbits = 162| 4.70k| _mm_and_si128(composedminus, _mm_set1_epi32(0x3ff)); 163| | // Notice the 0x3ff mask: 164| 4.70k| const __m128i hightenbits = 165| 4.70k| _mm_and_si128(_mm_srli_epi32(composedminus, 10), _mm_set1_epi32(0x3ff)); 166| 4.70k| const __m128i lowtenbitsadd = 167| 4.70k| _mm_add_epi32(lowtenbits, _mm_set1_epi32(0xDC00)); 168| 4.70k| const __m128i hightenbitsadd = 169| 4.70k| _mm_add_epi32(hightenbits, _mm_set1_epi32(0xD800)); 170| 4.70k| const __m128i lowtenbitsaddshifted = _mm_slli_epi32(lowtenbitsadd, 16); 171| 4.70k| __m128i surrogates = _mm_or_si128(hightenbitsadd, lowtenbitsaddshifted); 172| 4.70k| uint32_t basic_buffer[4]; 173| 4.70k| uint32_t basic_buffer_swap[4]; 174| 4.70k| if (big_endian) { ------------------ | Branch (174:9): [Folded, False: 4.70k] ------------------ 175| 0| _mm_storeu_si128((__m128i *)basic_buffer_swap, 176| 0| _mm_shuffle_epi8(composed, swap)); 177| 0| surrogates = _mm_shuffle_epi8(surrogates, swap); 178| 0| } 179| 4.70k| _mm_storeu_si128((__m128i *)basic_buffer, composed); 180| 4.70k| uint32_t surrogate_buffer[4]; 181| 4.70k| _mm_storeu_si128((__m128i *)surrogate_buffer, surrogates); 182| 18.8k| for (size_t i = 0; i < 3; i++) { ------------------ | Branch (182:24): [True: 14.1k, False: 4.70k] ------------------ 183| 14.1k| if (basic_buffer[i] > 0x3c00000) { ------------------ | Branch (183:11): [True: 3.05k, False: 11.0k] ------------------ 184| 3.05k| utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xffff); 185| 3.05k| utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); 186| 3.05k| utf16_output += 2; 187| 11.0k| } else { 188| 11.0k| utf16_output[0] = big_endian ? uint16_t(basic_buffer_swap[i]) ------------------ | Branch (188:27): [Folded, False: 11.0k] ------------------ 189| 11.0k| : uint16_t(basic_buffer[i]); 190| 11.0k| utf16_output++; 191| 11.0k| } 192| 14.1k| } 193| 4.70k| } else { 194| | // here we know that there is an error but we do not handle errors 195| 221| } 196| 15.5k| return consumed; 197| 20.6k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_128convert_masked_utf8_to_utf16ILNS_10endiannessE1EEEmPKcmRPDs: 10| 21.2k| char16_t *&utf16_output) { 11| | // we use an approach where we try to process up to 12 input bytes. 12| | // Why 12 input bytes and not 16? Because we are concerned with the size of 13| | // the lookup tables. Also 12 is nicely divisible by two and three. 14| | // 15| | // 16| | // Optimization note: our main path below is load-latency dependent. Thus it 17| | // is maybe beneficial to have fast paths that depend on branch prediction but 18| | // have less latency. This results in more instructions but, potentially, also 19| | // higher speeds. 20| | // 21| | // We first try a few fast paths. 22| 21.2k| const __m128i swap = 23| 21.2k| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 24| 21.2k| const __m128i in = _mm_loadu_si128((__m128i *)input); 25| 21.2k| const uint16_t input_utf8_end_of_code_point_mask = 26| 21.2k| utf8_end_of_code_point_mask & 0xfff; 27| 21.2k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (27:7): [True: 106, False: 21.1k] ------------------ 28| | // We process the data in chunks of 12 bytes. 29| | // Note: using 16 bytes is unsafe, see issue_ossfuzz_71218 30| 106| __m128i ascii_first = _mm_cvtepu8_epi16(in); 31| 106| __m128i ascii_second = _mm_cvtepu8_epi16(_mm_srli_si128(in, 8)); 32| 106| if (big_endian) { ------------------ | Branch (32:9): [True: 106, Folded] ------------------ 33| 106| ascii_first = _mm_shuffle_epi8(ascii_first, swap); 34| 106| ascii_second = _mm_shuffle_epi8(ascii_second, swap); 35| 106| } 36| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf16_output), ascii_first); 37| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf16_output + 8), 38| 106| ascii_second); 39| 106| utf16_output += 12; // We wrote 12 16-bit characters. 40| 106| return 12; // We consumed 12 bytes. 41| 106| } 42| 21.1k| if (((utf8_end_of_code_point_mask & 0xFFFF) == 0xaaaa)) { ------------------ | Branch (42:7): [True: 309, False: 20.8k] ------------------ 43| | // We want to take 8 2-byte UTF-8 code units and turn them into 8 2-byte 44| | // UTF-16 code units. There is probably a more efficient sequence, but the 45| | // following might do. 46| 309| const __m128i sh = 47| 309| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 48| 309| const __m128i perm = _mm_shuffle_epi8(in, sh); 49| 309| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 50| 309| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 51| 309| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 52| 309| if (big_endian) ------------------ | Branch (52:9): [True: 309, Folded] ------------------ 53| 309| composed = _mm_shuffle_epi8(composed, swap); 54| 309| _mm_storeu_si128((__m128i *)utf16_output, composed); 55| 309| utf16_output += 8; // We wrote 16 bytes, 8 code points. 56| 309| return 16; 57| 309| } 58| 20.8k| if (input_utf8_end_of_code_point_mask == 0x924) { ------------------ | Branch (58:7): [True: 4.74k, False: 16.0k] ------------------ 59| | // We want to take 4 3-byte UTF-8 code units and turn them into 4 2-byte 60| | // UTF-16 code units. There is probably a more efficient sequence, but the 61| | // following might do. 62| 4.74k| const __m128i sh = 63| 4.74k| _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); 64| 4.74k| const __m128i perm = _mm_shuffle_epi8(in, sh); 65| 4.74k| const __m128i ascii = 66| 4.74k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 67| 4.74k| const __m128i middlebyte = 68| 4.74k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 69| 4.74k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 70| 4.74k| const __m128i highbyte = 71| 4.74k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 72| 4.74k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 73| 4.74k| const __m128i composed = 74| 4.74k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 75| 4.74k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 76| 4.74k| if (big_endian) ------------------ | Branch (76:9): [True: 4.74k, Folded] ------------------ 77| 4.74k| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 78| 4.74k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 79| 4.74k| utf16_output += 4; 80| 4.74k| return 12; 81| 4.74k| } 82| | /// We do not have a fast path available, so we fallback. 83| | 84| 16.0k| const uint8_t idx = 85| 16.0k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; 86| 16.0k| const uint8_t consumed = 87| 16.0k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; 88| 16.0k| if (idx < 64) { ------------------ | Branch (88:7): [True: 8.35k, False: 7.71k] ------------------ 89| | // SIX (6) input code-code units 90| | // this is a relatively easy scenario 91| | // we process SIX (6) input code-code units. The max length in bytes of six 92| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 93| | // processors where pdep/pext is fast, we might be able to use a small 94| | // lookup table. 95| 8.35k| const __m128i sh = 96| 8.35k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 97| 8.35k| const __m128i perm = _mm_shuffle_epi8(in, sh); 98| 8.35k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 99| 8.35k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 100| 8.35k| __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 101| 8.35k| if (big_endian) ------------------ | Branch (101:9): [True: 8.35k, Folded] ------------------ 102| 8.35k| composed = _mm_shuffle_epi8(composed, swap); 103| 8.35k| _mm_storeu_si128((__m128i *)utf16_output, composed); 104| 8.35k| utf16_output += 6; // We wrote 12 bytes, 6 code points. 105| 8.35k| } else if (idx < 145) { ------------------ | Branch (105:14): [True: 2.76k, False: 4.94k] ------------------ 106| | // FOUR (4) input code-code units 107| 2.76k| const __m128i sh = 108| 2.76k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 109| 2.76k| const __m128i perm = _mm_shuffle_epi8(in, sh); 110| 2.76k| const __m128i ascii = 111| 2.76k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 112| 2.76k| const __m128i middlebyte = 113| 2.76k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 114| 2.76k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 115| 2.76k| const __m128i highbyte = 116| 2.76k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 117| 2.76k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 118| 2.76k| const __m128i composed = 119| 2.76k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 120| 2.76k| __m128i composed_repacked = _mm_packus_epi32(composed, composed); 121| 2.76k| if (big_endian) ------------------ | Branch (121:9): [True: 2.76k, Folded] ------------------ 122| 2.76k| composed_repacked = _mm_shuffle_epi8(composed_repacked, swap); 123| 2.76k| _mm_storeu_si128((__m128i *)utf16_output, composed_repacked); 124| 2.76k| utf16_output += 4; 125| 4.94k| } else if (idx < 209) { ------------------ | Branch (125:14): [True: 4.72k, False: 221] ------------------ 126| | // TWO (2) input code-code units 127| | ////////////// 128| | // There might be garbage inputs where a leading byte mascarades as a 129| | // four-byte leading byte (by being followed by 3 continuation byte), but is 130| | // not greater than 0xf0. This could trigger a buffer overflow if we only 131| | // counted leading bytes of the form 0xf0 as generating surrogate pairs, 132| | // without further UTF-8 validation. Thus we must be careful to ensure that 133| | // only leading bytes at least as large as 0xf0 generate surrogate pairs. We 134| | // do as at the cost of an extra mask. 135| | ///////////// 136| 4.72k| const __m128i sh = 137| 4.72k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 138| 4.72k| const __m128i perm = _mm_shuffle_epi8(in, sh); 139| 4.72k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); 140| 4.72k| const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); 141| 4.72k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 142| 4.72k| __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); 143| | // correct for spurious high bit 144| 4.72k| const __m128i correct = 145| 4.72k| _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); 146| 4.72k| middlehighbyte = _mm_xor_si128(correct, middlehighbyte); 147| 4.72k| const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); 148| | // We deliberately carry the leading four bits in highbyte if they are 149| | // present, we remove them later when computing hightenbits. 150| 4.72k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0xff000000)); 151| 4.72k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); 152| | // When we need to generate a surrogate pair (leading byte > 0xF0), then 153| | // the corresponding 32-bit value in 'composed' will be greater than 154| | // > (0xff00000>>6) or > 0x3c00000. This can be used later to identify the 155| | // location of the surrogate pairs. 156| 4.72k| const __m128i composed = 157| 4.72k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), 158| 4.72k| _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); 159| 4.72k| const __m128i composedminus = 160| 4.72k| _mm_sub_epi32(composed, _mm_set1_epi32(0x10000)); 161| 4.72k| const __m128i lowtenbits = 162| 4.72k| _mm_and_si128(composedminus, _mm_set1_epi32(0x3ff)); 163| | // Notice the 0x3ff mask: 164| 4.72k| const __m128i hightenbits = 165| 4.72k| _mm_and_si128(_mm_srli_epi32(composedminus, 10), _mm_set1_epi32(0x3ff)); 166| 4.72k| const __m128i lowtenbitsadd = 167| 4.72k| _mm_add_epi32(lowtenbits, _mm_set1_epi32(0xDC00)); 168| 4.72k| const __m128i hightenbitsadd = 169| 4.72k| _mm_add_epi32(hightenbits, _mm_set1_epi32(0xD800)); 170| 4.72k| const __m128i lowtenbitsaddshifted = _mm_slli_epi32(lowtenbitsadd, 16); 171| 4.72k| __m128i surrogates = _mm_or_si128(hightenbitsadd, lowtenbitsaddshifted); 172| 4.72k| uint32_t basic_buffer[4]; 173| 4.72k| uint32_t basic_buffer_swap[4]; 174| 4.72k| if (big_endian) { ------------------ | Branch (174:9): [True: 4.72k, Folded] ------------------ 175| 4.72k| _mm_storeu_si128((__m128i *)basic_buffer_swap, 176| 4.72k| _mm_shuffle_epi8(composed, swap)); 177| 4.72k| surrogates = _mm_shuffle_epi8(surrogates, swap); 178| 4.72k| } 179| 4.72k| _mm_storeu_si128((__m128i *)basic_buffer, composed); 180| 4.72k| uint32_t surrogate_buffer[4]; 181| 4.72k| _mm_storeu_si128((__m128i *)surrogate_buffer, surrogates); 182| 18.9k| for (size_t i = 0; i < 3; i++) { ------------------ | Branch (182:24): [True: 14.1k, False: 4.72k] ------------------ 183| 14.1k| if (basic_buffer[i] > 0x3c00000) { ------------------ | Branch (183:11): [True: 3.06k, False: 11.1k] ------------------ 184| 3.06k| utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xffff); 185| 3.06k| utf16_output[1] = uint16_t(surrogate_buffer[i] >> 16); 186| 3.06k| utf16_output += 2; 187| 11.1k| } else { 188| 11.1k| utf16_output[0] = big_endian ? uint16_t(basic_buffer_swap[i]) ------------------ | Branch (188:27): [True: 11.1k, Folded] ------------------ 189| 11.1k| : uint16_t(basic_buffer[i]); 190| 11.1k| utf16_output++; 191| 11.1k| } 192| 14.1k| } 193| 4.72k| } else { 194| | // here we know that there is an error but we do not handle errors 195| 221| } 196| 16.0k| return consumed; 197| 20.8k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_128convert_masked_utf8_to_utf32EPKcmRPDi: 9| 14.9k| char32_t *&utf32_output) { 10| | // we use an approach where we try to process up to 12 input bytes. 11| | // Why 12 input bytes and not 16? Because we are concerned with the size of 12| | // the lookup tables. Also 12 is nicely divisible by two and three. 13| | // 14| | // 15| | // Optimization note: our main path below is load-latency dependent. Thus it 16| | // is maybe beneficial to have fast paths that depend on branch prediction but 17| | // have less latency. This results in more instructions but, potentially, also 18| | // higher speeds. 19| | // 20| | // We first try a few fast paths. 21| 14.9k| const __m128i in = _mm_loadu_si128((__m128i *)input); 22| 14.9k| const uint16_t input_utf8_end_of_code_point_mask = 23| 14.9k| utf8_end_of_code_point_mask & 0xfff; 24| 14.9k| if (utf8_end_of_code_point_mask == 0xfff) { ------------------ | Branch (24:7): [True: 106, False: 14.7k] ------------------ 25| | // We process the data in chunks of 12 bytes. 26| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output), 27| 106| _mm_cvtepu8_epi32(in)); 28| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output + 4), 29| 106| _mm_cvtepu8_epi32(_mm_srli_si128(in, 4))); 30| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output + 8), 31| 106| _mm_cvtepu8_epi32(_mm_srli_si128(in, 8))); 32| 106| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output + 12), 33| 106| _mm_cvtepu8_epi32(_mm_srli_si128(in, 12))); 34| 106| utf32_output += 12; // We wrote 12 32-bit characters. 35| 106| return 12; // We consumed 12 bytes. 36| 106| } 37| 14.7k| if (((utf8_end_of_code_point_mask & 0xffff) == 0xaaaa)) { ------------------ | Branch (37:7): [True: 227, False: 14.5k] ------------------ 38| | // We want to take 8 2-byte UTF-8 code units and turn them into 8 4-byte 39| | // UTF-32 code units. There is probably a more efficient sequence, but the 40| | // following might do. 41| 227| const __m128i sh = 42| 227| _mm_setr_epi8(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); 43| 227| const __m128i perm = _mm_shuffle_epi8(in, sh); 44| 227| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 45| 227| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 46| 227| const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 47| 227| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output), 48| 227| _mm_cvtepu16_epi32(composed)); 49| 227| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output + 4), 50| 227| _mm_cvtepu16_epi32(_mm_srli_si128(composed, 8))); 51| 227| utf32_output += 8; // We wrote 32 bytes, 8 code points. 52| 227| return 16; 53| 227| } 54| 14.5k| if (input_utf8_end_of_code_point_mask == 0x924) { ------------------ | Branch (54:7): [True: 400, False: 14.1k] ------------------ 55| | // We want to take 4 3-byte UTF-8 code units and turn them into 4 4-byte 56| | // UTF-32 code units. There is probably a more efficient sequence, but the 57| | // following might do. 58| 400| const __m128i sh = 59| 400| _mm_setr_epi8(2, 1, 0, -1, 5, 4, 3, -1, 8, 7, 6, -1, 11, 10, 9, -1); 60| 400| const __m128i perm = _mm_shuffle_epi8(in, sh); 61| 400| const __m128i ascii = 62| 400| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 63| 400| const __m128i middlebyte = 64| 400| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 65| 400| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 66| 400| const __m128i highbyte = 67| 400| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 68| 400| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 69| 400| const __m128i composed = 70| 400| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 71| 400| _mm_storeu_si128((__m128i *)utf32_output, composed); 72| 400| utf32_output += 4; 73| 400| return 12; 74| 400| } 75| | /// We do not have a fast path available, so we fallback. 76| | 77| 14.1k| const uint8_t idx = 78| 14.1k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][0]; 79| 14.1k| const uint8_t consumed = 80| 14.1k| tables::utf8_to_utf16::utf8bigindex[input_utf8_end_of_code_point_mask][1]; 81| 14.1k| if (idx < 64) { ------------------ | Branch (81:7): [True: 8.09k, False: 6.07k] ------------------ 82| | // SIX (6) input code-code units 83| | // this is a relatively easy scenario 84| | // we process SIX (6) input code-code units. The max length in bytes of six 85| | // code code units spanning between 1 and 2 bytes each is 12 bytes. On 86| | // processors where pdep/pext is fast, we might be able to use a small 87| | // lookup table. 88| 8.09k| const __m128i sh = 89| 8.09k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 90| 8.09k| const __m128i perm = _mm_shuffle_epi8(in, sh); 91| 8.09k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi16(0x7f)); 92| 8.09k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi16(0x1f00)); 93| 8.09k| const __m128i composed = _mm_or_si128(ascii, _mm_srli_epi16(highbyte, 2)); 94| 8.09k| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output), 95| 8.09k| _mm_cvtepu16_epi32(composed)); 96| 8.09k| _mm_storeu_si128(reinterpret_cast<__m128i *>(utf32_output + 4), 97| 8.09k| _mm_cvtepu16_epi32(_mm_srli_si128(composed, 8))); 98| 8.09k| utf32_output += 6; // We wrote 12 bytes, 6 code points. 99| 8.09k| } else if (idx < 145) { ------------------ | Branch (99:14): [True: 1.56k, False: 4.51k] ------------------ 100| | // FOUR (4) input code-code units 101| 1.56k| const __m128i sh = 102| 1.56k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 103| 1.56k| const __m128i perm = _mm_shuffle_epi8(in, sh); 104| 1.56k| const __m128i ascii = 105| 1.56k| _mm_and_si128(perm, _mm_set1_epi32(0x7f)); // 7 or 6 bits 106| 1.56k| const __m128i middlebyte = 107| 1.56k| _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); // 5 or 6 bits 108| 1.56k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 109| 1.56k| const __m128i highbyte = 110| 1.56k| _mm_and_si128(perm, _mm_set1_epi32(0x0f0000)); // 4 bits 111| 1.56k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 4); 112| 1.56k| const __m128i composed = 113| 1.56k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), highbyte_shifted); 114| 1.56k| _mm_storeu_si128((__m128i *)utf32_output, composed); 115| 1.56k| utf32_output += 4; 116| 4.51k| } else if (idx < 209) { ------------------ | Branch (116:14): [True: 4.29k, False: 221] ------------------ 117| | // TWO (2) input code-code units 118| 4.29k| const __m128i sh = 119| 4.29k| _mm_loadu_si128((const __m128i *)tables::utf8_to_utf16::shufutf8[idx]); 120| 4.29k| const __m128i perm = _mm_shuffle_epi8(in, sh); 121| 4.29k| const __m128i ascii = _mm_and_si128(perm, _mm_set1_epi32(0x7f)); 122| 4.29k| const __m128i middlebyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f00)); 123| 4.29k| const __m128i middlebyte_shifted = _mm_srli_epi32(middlebyte, 2); 124| 4.29k| __m128i middlehighbyte = _mm_and_si128(perm, _mm_set1_epi32(0x3f0000)); 125| | // correct for spurious high bit 126| 4.29k| const __m128i correct = 127| 4.29k| _mm_srli_epi32(_mm_and_si128(perm, _mm_set1_epi32(0x400000)), 1); 128| 4.29k| middlehighbyte = _mm_xor_si128(correct, middlehighbyte); 129| 4.29k| const __m128i middlehighbyte_shifted = _mm_srli_epi32(middlehighbyte, 4); 130| 4.29k| const __m128i highbyte = _mm_and_si128(perm, _mm_set1_epi32(0x07000000)); 131| 4.29k| const __m128i highbyte_shifted = _mm_srli_epi32(highbyte, 6); 132| 4.29k| const __m128i composed = 133| 4.29k| _mm_or_si128(_mm_or_si128(ascii, middlebyte_shifted), 134| 4.29k| _mm_or_si128(highbyte_shifted, middlehighbyte_shifted)); 135| 4.29k| _mm_storeu_si128((__m128i *)utf32_output, composed); 136| 4.29k| utf32_output += 3; 137| 4.29k| } else { 138| | // here we know that there is an error but we do not handle errors 139| 221| } 140| 14.1k| return consumed; 141| 14.5k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_123utf16_gather_high_bytesILNS_10endiannessE0EEENS1_4simd5simd8IhEENS4_6simd16ItEES8_: 3| 26.5k| const simd16 in1) { 4| 26.5k| if (big_endian) { ------------------ | Branch (4:7): [Folded, False: 26.5k] ------------------ 5| | // we want lower bytes 6| 0| const auto mask = simd16(0x00ff); 7| 0| const auto t0 = in0 & mask; 8| 0| const auto t1 = in1 & mask; 9| | 10| 0| return simd16::pack(t0, t1); 11| 26.5k| } else { 12| 26.5k| const auto t0 = in0.shr<8>(); 13| 26.5k| const auto t1 = in1.shr<8>(); 14| | 15| 26.5k| return simd16::pack(t0, t1); 16| 26.5k| } 17| 26.5k|} simdutf.cpp:_ZN7simdutf8westmere12_GLOBAL__N_123utf16_gather_high_bytesILNS_10endiannessE1EEENS1_4simd5simd8IhEENS4_6simd16ItEES8_: 3| 26.3k| const simd16 in1) { 4| 26.3k| if (big_endian) { ------------------ | Branch (4:7): [True: 26.3k, Folded] ------------------ 5| | // we want lower bytes 6| 26.3k| const auto mask = simd16(0x00ff); 7| 26.3k| const auto t0 = in0 & mask; 8| 26.3k| const auto t1 = in1 & mask; 9| | 10| 26.3k| return simd16::pack(t0, t1); 11| 26.3k| } else { 12| 0| const auto t0 = in0.shr<8>(); 13| 0| const auto t1 = in1.shr<8>(); 14| | 15| 0| return simd16::pack(t0, t1); 16| 0| } 17| 26.3k|}