95 | 6.88M | } Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long) void jxl::N_SSE4::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 5.07k | pixel_type* out2, size_t w) { | 33 | 5.07k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 5.07k | "Invalid transform type"); | 35 | 5.07k | int second = transform_type >> 1; | 36 | 5.07k | int third = transform_type & 1; | 37 | | | 38 | 5.07k | size_t x = 0; | 39 | 5.07k | const HWY_FULL(pixel_type) d; | 40 | 5.07k | const size_t N = Lanes(d); | 41 | 99.8k | for (; x + N - 1 < w; x += N) { | 42 | 94.7k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 94.7k | } else { | 54 | 94.7k | auto First = Load(d, in0 + x); | 55 | 94.7k | auto Second = Load(d, in1 + x); | 56 | 94.7k | auto Third = Load(d, in2 + x); | 57 | 94.8k | if (third) Third = Add(Third, First); | 58 | 94.7k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 94.7k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 94.7k | Store(First, d, out0 + x); | 64 | 94.7k | Store(Second, d, out1 + x); | 65 | 94.7k | Store(Third, d, out2 + x); | 66 | 94.7k | } | 67 | 94.7k | } | 68 | 15.5k | for (; x < w; x++) { | 69 | 10.4k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 10.4k | } else { | 81 | 10.4k | pixel_type First = in0[x]; | 82 | 10.4k | pixel_type Second = in1[x]; | 83 | 10.4k | pixel_type Third = in2[x]; | 84 | 10.4k | if (third) Third = PixelAdd(Third, First); | 85 | 10.4k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 10.4k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 10.4k | out0[x] = First; | 91 | 10.4k | out1[x] = Second; | 92 | 10.4k | out2[x] = Third; | 93 | 10.4k | } | 94 | 10.4k | } | 95 | 5.07k | } |
void jxl::N_SSE4::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 3.52k | pixel_type* out2, size_t w) { | 33 | 3.52k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 3.52k | "Invalid transform type"); | 35 | 3.52k | int second = transform_type >> 1; | 36 | 3.52k | int third = transform_type & 1; | 37 | | | 38 | 3.52k | size_t x = 0; | 39 | 3.52k | const HWY_FULL(pixel_type) d; | 40 | 3.52k | const size_t N = Lanes(d); | 41 | 85.6k | for (; x + N - 1 < w; x += N) { | 42 | 82.1k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 82.1k | } else { | 54 | 82.1k | auto First = Load(d, in0 + x); | 55 | 82.1k | auto Second = Load(d, in1 + x); | 56 | 82.1k | auto Third = Load(d, in2 + x); | 57 | 82.1k | if (third) Third = Add(Third, First); | 58 | 82.2k | if (second == 1) { | 59 | 82.2k | Second = Add(Second, First); | 60 | 18.4E | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 82.1k | Store(First, d, out0 + x); | 64 | 82.1k | Store(Second, d, out1 + x); | 65 | 82.1k | Store(Third, d, out2 + x); | 66 | 82.1k | } | 67 | 82.1k | } | 68 | 6.58k | for (; x < w; x++) { | 69 | 3.05k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 3.05k | } else { | 81 | 3.05k | pixel_type First = in0[x]; | 82 | 3.05k | pixel_type Second = in1[x]; | 83 | 3.05k | pixel_type Third = in2[x]; | 84 | 3.05k | if (third) Third = PixelAdd(Third, First); | 85 | 3.05k | if (second == 1) { | 86 | 3.05k | Second = PixelAdd(Second, First); | 87 | 3.05k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 3.05k | out0[x] = First; | 91 | 3.05k | out1[x] = Second; | 92 | 3.05k | out2[x] = Third; | 93 | 3.05k | } | 94 | 3.05k | } | 95 | 3.52k | } |
void jxl::N_SSE4::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 3.17k | pixel_type* out2, size_t w) { | 33 | 3.17k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 3.17k | "Invalid transform type"); | 35 | 3.17k | int second = transform_type >> 1; | 36 | 3.17k | int third = transform_type & 1; | 37 | | | 38 | 3.17k | size_t x = 0; | 39 | 3.17k | const HWY_FULL(pixel_type) d; | 40 | 3.17k | const size_t N = Lanes(d); | 41 | 75.5k | for (; x + N - 1 < w; x += N) { | 42 | 72.3k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 72.3k | } else { | 54 | 72.3k | auto First = Load(d, in0 + x); | 55 | 72.3k | auto Second = Load(d, in1 + x); | 56 | 72.3k | auto Third = Load(d, in2 + x); | 57 | 72.3k | if (third) Third = Add(Third, First); | 58 | 72.3k | if (second == 1) { | 59 | 72.3k | Second = Add(Second, First); | 60 | 72.3k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 72.3k | Store(First, d, out0 + x); | 64 | 72.3k | Store(Second, d, out1 + x); | 65 | 72.3k | Store(Third, d, out2 + x); | 66 | 72.3k | } | 67 | 72.3k | } | 68 | 5.50k | for (; x < w; x++) { | 69 | 2.32k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 2.32k | } else { | 81 | 2.32k | pixel_type First = in0[x]; | 82 | 2.32k | pixel_type Second = in1[x]; | 83 | 2.32k | pixel_type Third = in2[x]; | 84 | 2.32k | if (third) Third = PixelAdd(Third, First); | 85 | 2.32k | if (second == 1) { | 86 | 2.32k | Second = PixelAdd(Second, First); | 87 | 2.32k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 2.32k | out0[x] = First; | 91 | 2.32k | out1[x] = Second; | 92 | 2.32k | out2[x] = Third; | 93 | 2.32k | } | 94 | 2.32k | } | 95 | 3.17k | } |
void jxl::N_SSE4::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 12.2k | pixel_type* out2, size_t w) { | 33 | 12.2k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 12.2k | "Invalid transform type"); | 35 | 12.2k | int second = transform_type >> 1; | 36 | 12.2k | int third = transform_type & 1; | 37 | | | 38 | 12.2k | size_t x = 0; | 39 | 12.2k | const HWY_FULL(pixel_type) d; | 40 | 12.2k | const size_t N = Lanes(d); | 41 | 340k | for (; x + N - 1 < w; x += N) { | 42 | 328k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 328k | } else { | 54 | 328k | auto First = Load(d, in0 + x); | 55 | 328k | auto Second = Load(d, in1 + x); | 56 | 328k | auto Third = Load(d, in2 + x); | 57 | 328k | if (third) Third = Add(Third, First); | 58 | 328k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 328k | } else if (second == 2) { | 61 | 328k | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 328k | } | 63 | 328k | Store(First, d, out0 + x); | 64 | 328k | Store(Second, d, out1 + x); | 65 | 328k | Store(Third, d, out2 + x); | 66 | 328k | } | 67 | 328k | } | 68 | 19.4k | for (; x < w; x++) { | 69 | 7.27k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 7.27k | } else { | 81 | 7.27k | pixel_type First = in0[x]; | 82 | 7.27k | pixel_type Second = in1[x]; | 83 | 7.27k | pixel_type Third = in2[x]; | 84 | 7.27k | if (third) Third = PixelAdd(Third, First); | 85 | 7.27k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 7.27k | } else if (second == 2) { | 88 | 7.27k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 7.27k | } | 90 | 7.27k | out0[x] = First; | 91 | 7.27k | out1[x] = Second; | 92 | 7.27k | out2[x] = Third; | 93 | 7.27k | } | 94 | 7.27k | } | 95 | 12.2k | } |
void jxl::N_SSE4::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 94.7k | pixel_type* out2, size_t w) { | 33 | 94.7k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 94.7k | "Invalid transform type"); | 35 | 94.7k | int second = transform_type >> 1; | 36 | 94.7k | int third = transform_type & 1; | 37 | | | 38 | 94.7k | size_t x = 0; | 39 | 94.7k | const HWY_FULL(pixel_type) d; | 40 | 94.7k | const size_t N = Lanes(d); | 41 | 4.10M | for (; x + N - 1 < w; x += N) { | 42 | 4.01M | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 4.01M | } else { | 54 | 4.01M | auto First = Load(d, in0 + x); | 55 | 4.01M | auto Second = Load(d, in1 + x); | 56 | 4.01M | auto Third = Load(d, in2 + x); | 57 | 4.02M | if (third) Third = Add(Third, First); | 58 | 4.01M | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 4.01M | } else if (second == 2) { | 61 | 4.00M | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 4.00M | } | 63 | 4.01M | Store(First, d, out0 + x); | 64 | 4.01M | Store(Second, d, out1 + x); | 65 | 4.01M | Store(Third, d, out2 + x); | 66 | 4.01M | } | 67 | 4.01M | } | 68 | 96.3k | for (; x < w; x++) { | 69 | 1.60k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 1.60k | } else { | 81 | 1.60k | pixel_type First = in0[x]; | 82 | 1.60k | pixel_type Second = in1[x]; | 83 | 1.60k | pixel_type Third = in2[x]; | 84 | 1.60k | if (third) Third = PixelAdd(Third, First); | 85 | 1.60k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 1.60k | } else if (second == 2) { | 88 | 1.60k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 1.60k | } | 90 | 1.60k | out0[x] = First; | 91 | 1.60k | out1[x] = Second; | 92 | 1.60k | out2[x] = Third; | 93 | 1.60k | } | 94 | 1.60k | } | 95 | 94.7k | } |
void jxl::N_SSE4::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 24.5k | pixel_type* out2, size_t w) { | 33 | 24.5k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 24.5k | "Invalid transform type"); | 35 | 24.5k | int second = transform_type >> 1; | 36 | 24.5k | int third = transform_type & 1; | 37 | | | 38 | 24.5k | size_t x = 0; | 39 | 24.5k | const HWY_FULL(pixel_type) d; | 40 | 24.5k | const size_t N = Lanes(d); | 41 | 749k | for (; x + N - 1 < w; x += N) { | 42 | 724k | if (transform_type == 6) { | 43 | 724k | auto Y = Load(d, in0 + x); | 44 | 724k | auto Co = Load(d, in1 + x); | 45 | 724k | auto Cg = Load(d, in2 + x); | 46 | 724k | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 724k | auto G = Add(Cg, Y); | 48 | 724k | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 724k | auto R = Add(Y, Co); | 50 | 724k | Store(R, d, out0 + x); | 51 | 724k | Store(G, d, out1 + x); | 52 | 724k | Store(Y, d, out2 + x); | 53 | 18.4E | } else { | 54 | 18.4E | auto First = Load(d, in0 + x); | 55 | 18.4E | auto Second = Load(d, in1 + x); | 56 | 18.4E | auto Third = Load(d, in2 + x); | 57 | 18.4E | if (third) Third = Add(Third, First); | 58 | 18.4E | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 18.4E | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 18.4E | Store(First, d, out0 + x); | 64 | 18.4E | Store(Second, d, out1 + x); | 65 | 18.4E | Store(Third, d, out2 + x); | 66 | 18.4E | } | 67 | 724k | } | 68 | 44.7k | for (; x < w; x++) { | 69 | 20.1k | if (transform_type == 6) { | 70 | 20.1k | pixel_type Y = in0[x]; | 71 | 20.1k | pixel_type Co = in1[x]; | 72 | 20.1k | pixel_type Cg = in2[x]; | 73 | 20.1k | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 20.1k | pixel_type G = PixelAdd(Cg, tmp); | 75 | 20.1k | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 20.1k | pixel_type R = PixelAdd(B, Co); | 77 | 20.1k | out0[x] = R; | 78 | 20.1k | out1[x] = G; | 79 | 20.1k | out2[x] = B; | 80 | 20.1k | } else { | 81 | 0 | pixel_type First = in0[x]; | 82 | 0 | pixel_type Second = in1[x]; | 83 | 0 | pixel_type Third = in2[x]; | 84 | 0 | if (third) Third = PixelAdd(Third, First); | 85 | 0 | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 0 | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 0 | out0[x] = First; | 91 | 0 | out1[x] = Second; | 92 | 0 | out2[x] = Third; | 93 | 0 | } | 94 | 20.1k | } | 95 | 24.5k | } |
Unexecuted instantiation: void jxl::N_AVX2::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long) void jxl::N_AVX2::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 246k | pixel_type* out2, size_t w) { | 33 | 246k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 246k | "Invalid transform type"); | 35 | 246k | int second = transform_type >> 1; | 36 | 246k | int third = transform_type & 1; | 37 | | | 38 | 246k | size_t x = 0; | 39 | 246k | const HWY_FULL(pixel_type) d; | 40 | 246k | const size_t N = Lanes(d); | 41 | 822k | for (; x + N - 1 < w; x += N) { | 42 | 575k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 575k | } else { | 54 | 575k | auto First = Load(d, in0 + x); | 55 | 575k | auto Second = Load(d, in1 + x); | 56 | 575k | auto Third = Load(d, in2 + x); | 57 | 575k | if (third) Third = Add(Third, First); | 58 | 575k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 575k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 575k | Store(First, d, out0 + x); | 64 | 575k | Store(Second, d, out1 + x); | 65 | 575k | Store(Third, d, out2 + x); | 66 | 575k | } | 67 | 575k | } | 68 | 854k | for (; x < w; x++) { | 69 | 608k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 608k | } else { | 81 | 608k | pixel_type First = in0[x]; | 82 | 608k | pixel_type Second = in1[x]; | 83 | 608k | pixel_type Third = in2[x]; | 84 | 608k | if (third) Third = PixelAdd(Third, First); | 85 | 608k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 608k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 608k | out0[x] = First; | 91 | 608k | out1[x] = Second; | 92 | 608k | out2[x] = Third; | 93 | 608k | } | 94 | 608k | } | 95 | 246k | } |
void jxl::N_AVX2::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 173k | pixel_type* out2, size_t w) { | 33 | 173k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 173k | "Invalid transform type"); | 35 | 173k | int second = transform_type >> 1; | 36 | 173k | int third = transform_type & 1; | 37 | | | 38 | 173k | size_t x = 0; | 39 | 173k | const HWY_FULL(pixel_type) d; | 40 | 173k | const size_t N = Lanes(d); | 41 | 1.39M | for (; x + N - 1 < w; x += N) { | 42 | 1.22M | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 1.22M | } else { | 54 | 1.22M | auto First = Load(d, in0 + x); | 55 | 1.22M | auto Second = Load(d, in1 + x); | 56 | 1.22M | auto Third = Load(d, in2 + x); | 57 | 1.22M | if (third) Third = Add(Third, First); | 58 | 1.22M | if (second == 1) { | 59 | 1.22M | Second = Add(Second, First); | 60 | 18.4E | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 1.22M | Store(First, d, out0 + x); | 64 | 1.22M | Store(Second, d, out1 + x); | 65 | 1.22M | Store(Third, d, out2 + x); | 66 | 1.22M | } | 67 | 1.22M | } | 68 | 634k | for (; x < w; x++) { | 69 | 460k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 460k | } else { | 81 | 460k | pixel_type First = in0[x]; | 82 | 460k | pixel_type Second = in1[x]; | 83 | 460k | pixel_type Third = in2[x]; | 84 | 460k | if (third) Third = PixelAdd(Third, First); | 85 | 460k | if (second == 1) { | 86 | 460k | Second = PixelAdd(Second, First); | 87 | 460k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 460k | out0[x] = First; | 91 | 460k | out1[x] = Second; | 92 | 460k | out2[x] = Third; | 93 | 460k | } | 94 | 460k | } | 95 | 173k | } |
void jxl::N_AVX2::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 102k | pixel_type* out2, size_t w) { | 33 | 102k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 102k | "Invalid transform type"); | 35 | 102k | int second = transform_type >> 1; | 36 | 102k | int third = transform_type & 1; | 37 | | | 38 | 102k | size_t x = 0; | 39 | 102k | const HWY_FULL(pixel_type) d; | 40 | 102k | const size_t N = Lanes(d); | 41 | 1.14M | for (; x + N - 1 < w; x += N) { | 42 | 1.04M | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 1.04M | } else { | 54 | 1.04M | auto First = Load(d, in0 + x); | 55 | 1.04M | auto Second = Load(d, in1 + x); | 56 | 1.04M | auto Third = Load(d, in2 + x); | 57 | 1.04M | if (third) Third = Add(Third, First); | 58 | 1.04M | if (second == 1) { | 59 | 1.04M | Second = Add(Second, First); | 60 | 18.4E | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 1.04M | Store(First, d, out0 + x); | 64 | 1.04M | Store(Second, d, out1 + x); | 65 | 1.04M | Store(Third, d, out2 + x); | 66 | 1.04M | } | 67 | 1.04M | } | 68 | 539k | for (; x < w; x++) { | 69 | 437k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 437k | } else { | 81 | 437k | pixel_type First = in0[x]; | 82 | 437k | pixel_type Second = in1[x]; | 83 | 437k | pixel_type Third = in2[x]; | 84 | 437k | if (third) Third = PixelAdd(Third, First); | 85 | 437k | if (second == 1) { | 86 | 437k | Second = PixelAdd(Second, First); | 87 | 437k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 437k | out0[x] = First; | 91 | 437k | out1[x] = Second; | 92 | 437k | out2[x] = Third; | 93 | 437k | } | 94 | 437k | } | 95 | 102k | } |
void jxl::N_AVX2::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 86.3k | pixel_type* out2, size_t w) { | 33 | 86.3k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 86.3k | "Invalid transform type"); | 35 | 86.3k | int second = transform_type >> 1; | 36 | 86.3k | int third = transform_type & 1; | 37 | | | 38 | 86.3k | size_t x = 0; | 39 | 86.3k | const HWY_FULL(pixel_type) d; | 40 | 86.3k | const size_t N = Lanes(d); | 41 | 645k | for (; x + N - 1 < w; x += N) { | 42 | 559k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 559k | } else { | 54 | 559k | auto First = Load(d, in0 + x); | 55 | 559k | auto Second = Load(d, in1 + x); | 56 | 559k | auto Third = Load(d, in2 + x); | 57 | 559k | if (third) Third = Add(Third, First); | 58 | 559k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 559k | } else if (second == 2) { | 61 | 559k | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 559k | } | 63 | 559k | Store(First, d, out0 + x); | 64 | 559k | Store(Second, d, out1 + x); | 65 | 559k | Store(Third, d, out2 + x); | 66 | 559k | } | 67 | 559k | } | 68 | 204k | for (; x < w; x++) { | 69 | 118k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 118k | } else { | 81 | 118k | pixel_type First = in0[x]; | 82 | 118k | pixel_type Second = in1[x]; | 83 | 118k | pixel_type Third = in2[x]; | 84 | 118k | if (third) Third = PixelAdd(Third, First); | 85 | 118k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 118k | } else if (second == 2) { | 88 | 118k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 118k | } | 90 | 118k | out0[x] = First; | 91 | 118k | out1[x] = Second; | 92 | 118k | out2[x] = Third; | 93 | 118k | } | 94 | 118k | } | 95 | 86.3k | } |
void jxl::N_AVX2::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 221k | pixel_type* out2, size_t w) { | 33 | 221k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 221k | "Invalid transform type"); | 35 | 221k | int second = transform_type >> 1; | 36 | 221k | int third = transform_type & 1; | 37 | | | 38 | 221k | size_t x = 0; | 39 | 221k | const HWY_FULL(pixel_type) d; | 40 | 221k | const size_t N = Lanes(d); | 41 | 3.40M | for (; x + N - 1 < w; x += N) { | 42 | 3.18M | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 3.18M | } else { | 54 | 3.18M | auto First = Load(d, in0 + x); | 55 | 3.18M | auto Second = Load(d, in1 + x); | 56 | 3.18M | auto Third = Load(d, in2 + x); | 57 | 3.18M | if (third) Third = Add(Third, First); | 58 | 3.18M | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 3.18M | } else if (second == 2) { | 61 | 3.18M | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 3.18M | } | 63 | 3.18M | Store(First, d, out0 + x); | 64 | 3.18M | Store(Second, d, out1 + x); | 65 | 3.18M | Store(Third, d, out2 + x); | 66 | 3.18M | } | 67 | 3.18M | } | 68 | 696k | for (; x < w; x++) { | 69 | 474k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 474k | } else { | 81 | 474k | pixel_type First = in0[x]; | 82 | 474k | pixel_type Second = in1[x]; | 83 | 474k | pixel_type Third = in2[x]; | 84 | 474k | if (third) Third = PixelAdd(Third, First); | 85 | 474k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 474k | } else if (second == 2) { | 88 | 474k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 474k | } | 90 | 474k | out0[x] = First; | 91 | 474k | out1[x] = Second; | 92 | 474k | out2[x] = Third; | 93 | 474k | } | 94 | 474k | } | 95 | 221k | } |
void jxl::N_AVX2::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 5.76M | pixel_type* out2, size_t w) { | 33 | 5.76M | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 5.76M | "Invalid transform type"); | 35 | 5.76M | int second = transform_type >> 1; | 36 | 5.76M | int third = transform_type & 1; | 37 | | | 38 | 5.76M | size_t x = 0; | 39 | 5.76M | const HWY_FULL(pixel_type) d; | 40 | 5.76M | const size_t N = Lanes(d); | 41 | 23.3M | for (; x + N - 1 < w; x += N) { | 42 | 17.5M | if (transform_type == 6) { | 43 | 17.5M | auto Y = Load(d, in0 + x); | 44 | 17.5M | auto Co = Load(d, in1 + x); | 45 | 17.5M | auto Cg = Load(d, in2 + x); | 46 | 17.5M | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 17.5M | auto G = Add(Cg, Y); | 48 | 17.5M | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 17.5M | auto R = Add(Y, Co); | 50 | 17.5M | Store(R, d, out0 + x); | 51 | 17.5M | Store(G, d, out1 + x); | 52 | 17.5M | Store(Y, d, out2 + x); | 53 | 17.5M | } else { | 54 | 9 | auto First = Load(d, in0 + x); | 55 | 9 | auto Second = Load(d, in1 + x); | 56 | 9 | auto Third = Load(d, in2 + x); | 57 | 9 | if (third) Third = Add(Third, First); | 58 | 9 | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 9 | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 9 | Store(First, d, out0 + x); | 64 | 9 | Store(Second, d, out1 + x); | 65 | 9 | Store(Third, d, out2 + x); | 66 | 9 | } | 67 | 17.5M | } | 68 | 43.9M | for (; x < w; x++) { | 69 | 38.2M | if (transform_type == 6) { | 70 | 38.2M | pixel_type Y = in0[x]; | 71 | 38.2M | pixel_type Co = in1[x]; | 72 | 38.2M | pixel_type Cg = in2[x]; | 73 | 38.2M | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 38.2M | pixel_type G = PixelAdd(Cg, tmp); | 75 | 38.2M | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 38.2M | pixel_type R = PixelAdd(B, Co); | 77 | 38.2M | out0[x] = R; | 78 | 38.2M | out1[x] = G; | 79 | 38.2M | out2[x] = B; | 80 | 38.2M | } else { | 81 | 0 | pixel_type First = in0[x]; | 82 | 0 | pixel_type Second = in1[x]; | 83 | 0 | pixel_type Third = in2[x]; | 84 | 0 | if (third) Third = PixelAdd(Third, First); | 85 | 0 | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 0 | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 0 | out0[x] = First; | 91 | 0 | out1[x] = Second; | 92 | 0 | out2[x] = Third; | 93 | 0 | } | 94 | 38.2M | } | 95 | 5.76M | } |
Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long) void jxl::N_SSE2::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 4.62k | pixel_type* out2, size_t w) { | 33 | 4.62k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 4.62k | "Invalid transform type"); | 35 | 4.62k | int second = transform_type >> 1; | 36 | 4.62k | int third = transform_type & 1; | 37 | | | 38 | 4.62k | size_t x = 0; | 39 | 4.62k | const HWY_FULL(pixel_type) d; | 40 | 4.62k | const size_t N = Lanes(d); | 41 | 106k | for (; x + N - 1 < w; x += N) { | 42 | 101k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 101k | } else { | 54 | 101k | auto First = Load(d, in0 + x); | 55 | 101k | auto Second = Load(d, in1 + x); | 56 | 101k | auto Third = Load(d, in2 + x); | 57 | 102k | if (third) Third = Add(Third, First); | 58 | 101k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 101k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 101k | Store(First, d, out0 + x); | 64 | 101k | Store(Second, d, out1 + x); | 65 | 101k | Store(Third, d, out2 + x); | 66 | 101k | } | 67 | 101k | } | 68 | 13.6k | for (; x < w; x++) { | 69 | 8.99k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 8.99k | } else { | 81 | 8.99k | pixel_type First = in0[x]; | 82 | 8.99k | pixel_type Second = in1[x]; | 83 | 8.99k | pixel_type Third = in2[x]; | 84 | 8.99k | if (third) Third = PixelAdd(Third, First); | 85 | 8.99k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 8.99k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 8.99k | out0[x] = First; | 91 | 8.99k | out1[x] = Second; | 92 | 8.99k | out2[x] = Third; | 93 | 8.99k | } | 94 | 8.99k | } | 95 | 4.62k | } |
void jxl::N_SSE2::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 5.98k | pixel_type* out2, size_t w) { | 33 | 5.98k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 5.98k | "Invalid transform type"); | 35 | 5.98k | int second = transform_type >> 1; | 36 | 5.98k | int third = transform_type & 1; | 37 | | | 38 | 5.98k | size_t x = 0; | 39 | 5.98k | const HWY_FULL(pixel_type) d; | 40 | 5.98k | const size_t N = Lanes(d); | 41 | 200k | for (; x + N - 1 < w; x += N) { | 42 | 194k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 194k | } else { | 54 | 194k | auto First = Load(d, in0 + x); | 55 | 194k | auto Second = Load(d, in1 + x); | 56 | 194k | auto Third = Load(d, in2 + x); | 57 | 194k | if (third) Third = Add(Third, First); | 58 | 194k | if (second == 1) { | 59 | 194k | Second = Add(Second, First); | 60 | 18.4E | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 194k | Store(First, d, out0 + x); | 64 | 194k | Store(Second, d, out1 + x); | 65 | 194k | Store(Third, d, out2 + x); | 66 | 194k | } | 67 | 194k | } | 68 | 15.7k | for (; x < w; x++) { | 69 | 9.74k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 9.74k | } else { | 81 | 9.74k | pixel_type First = in0[x]; | 82 | 9.74k | pixel_type Second = in1[x]; | 83 | 9.74k | pixel_type Third = in2[x]; | 84 | 9.74k | if (third) Third = PixelAdd(Third, First); | 85 | 9.74k | if (second == 1) { | 86 | 9.74k | Second = PixelAdd(Second, First); | 87 | 18.4E | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 9.74k | out0[x] = First; | 91 | 9.74k | out1[x] = Second; | 92 | 9.74k | out2[x] = Third; | 93 | 9.74k | } | 94 | 9.74k | } | 95 | 5.98k | } |
void jxl::N_SSE2::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 2.33k | pixel_type* out2, size_t w) { | 33 | 2.33k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 2.33k | "Invalid transform type"); | 35 | 2.33k | int second = transform_type >> 1; | 36 | 2.33k | int third = transform_type & 1; | 37 | | | 38 | 2.33k | size_t x = 0; | 39 | 2.33k | const HWY_FULL(pixel_type) d; | 40 | 2.33k | const size_t N = Lanes(d); | 41 | 53.6k | for (; x + N - 1 < w; x += N) { | 42 | 51.3k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 51.3k | } else { | 54 | 51.3k | auto First = Load(d, in0 + x); | 55 | 51.3k | auto Second = Load(d, in1 + x); | 56 | 51.3k | auto Third = Load(d, in2 + x); | 57 | 51.3k | if (third) Third = Add(Third, First); | 58 | 51.3k | if (second == 1) { | 59 | 51.3k | Second = Add(Second, First); | 60 | 51.3k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 51.3k | Store(First, d, out0 + x); | 64 | 51.3k | Store(Second, d, out1 + x); | 65 | 51.3k | Store(Third, d, out2 + x); | 66 | 51.3k | } | 67 | 51.3k | } | 68 | 4.57k | for (; x < w; x++) { | 69 | 2.24k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 2.24k | } else { | 81 | 2.24k | pixel_type First = in0[x]; | 82 | 2.24k | pixel_type Second = in1[x]; | 83 | 2.24k | pixel_type Third = in2[x]; | 84 | 2.24k | if (third) Third = PixelAdd(Third, First); | 85 | 2.24k | if (second == 1) { | 86 | 2.24k | Second = PixelAdd(Second, First); | 87 | 2.24k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 2.24k | out0[x] = First; | 91 | 2.24k | out1[x] = Second; | 92 | 2.24k | out2[x] = Third; | 93 | 2.24k | } | 94 | 2.24k | } | 95 | 2.33k | } |
void jxl::N_SSE2::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 26.3k | pixel_type* out2, size_t w) { | 33 | 26.3k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 26.3k | "Invalid transform type"); | 35 | 26.3k | int second = transform_type >> 1; | 36 | 26.3k | int third = transform_type & 1; | 37 | | | 38 | 26.3k | size_t x = 0; | 39 | 26.3k | const HWY_FULL(pixel_type) d; | 40 | 26.3k | const size_t N = Lanes(d); | 41 | 577k | for (; x + N - 1 < w; x += N) { | 42 | 551k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 551k | } else { | 54 | 551k | auto First = Load(d, in0 + x); | 55 | 551k | auto Second = Load(d, in1 + x); | 56 | 551k | auto Third = Load(d, in2 + x); | 57 | 551k | if (third) Third = Add(Third, First); | 58 | 551k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 551k | } else if (second == 2) { | 61 | 551k | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 551k | } | 63 | 551k | Store(First, d, out0 + x); | 64 | 551k | Store(Second, d, out1 + x); | 65 | 551k | Store(Third, d, out2 + x); | 66 | 551k | } | 67 | 551k | } | 68 | 43.3k | for (; x < w; x++) { | 69 | 17.0k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 17.0k | } else { | 81 | 17.0k | pixel_type First = in0[x]; | 82 | 17.0k | pixel_type Second = in1[x]; | 83 | 17.0k | pixel_type Third = in2[x]; | 84 | 17.0k | if (third) Third = PixelAdd(Third, First); | 85 | 17.0k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 17.0k | } else if (second == 2) { | 88 | 17.0k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 17.0k | } | 90 | 17.0k | out0[x] = First; | 91 | 17.0k | out1[x] = Second; | 92 | 17.0k | out2[x] = Third; | 93 | 17.0k | } | 94 | 17.0k | } | 95 | 26.3k | } |
void jxl::N_SSE2::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 72.0k | pixel_type* out2, size_t w) { | 33 | 72.0k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 72.0k | "Invalid transform type"); | 35 | 72.0k | int second = transform_type >> 1; | 36 | 72.0k | int third = transform_type & 1; | 37 | | | 38 | 72.0k | size_t x = 0; | 39 | 72.0k | const HWY_FULL(pixel_type) d; | 40 | 72.0k | const size_t N = Lanes(d); | 41 | 3.09M | for (; x + N - 1 < w; x += N) { | 42 | 3.02M | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 3.02M | } else { | 54 | 3.02M | auto First = Load(d, in0 + x); | 55 | 3.02M | auto Second = Load(d, in1 + x); | 56 | 3.02M | auto Third = Load(d, in2 + x); | 57 | 3.07M | if (third) Third = Add(Third, First); | 58 | 3.02M | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 3.06M | } else if (second == 2) { | 61 | 3.06M | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 3.06M | } | 63 | 3.02M | Store(First, d, out0 + x); | 64 | 3.02M | Store(Second, d, out1 + x); | 65 | 3.02M | Store(Third, d, out2 + x); | 66 | 3.02M | } | 67 | 3.02M | } | 68 | 75.3k | for (; x < w; x++) { | 69 | 3.32k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 3.32k | } else { | 81 | 3.32k | pixel_type First = in0[x]; | 82 | 3.32k | pixel_type Second = in1[x]; | 83 | 3.32k | pixel_type Third = in2[x]; | 84 | 3.32k | if (third) Third = PixelAdd(Third, First); | 85 | 3.32k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 3.32k | } else if (second == 2) { | 88 | 3.32k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 3.32k | } | 90 | 3.32k | out0[x] = First; | 91 | 3.32k | out1[x] = Second; | 92 | 3.32k | out2[x] = Third; | 93 | 3.32k | } | 94 | 3.32k | } | 95 | 72.0k | } |
void jxl::N_SSE2::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 36.8k | pixel_type* out2, size_t w) { | 33 | 36.8k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 36.8k | "Invalid transform type"); | 35 | 36.8k | int second = transform_type >> 1; | 36 | 36.8k | int third = transform_type & 1; | 37 | | | 38 | 36.8k | size_t x = 0; | 39 | 36.8k | const HWY_FULL(pixel_type) d; | 40 | 36.8k | const size_t N = Lanes(d); | 41 | 958k | for (; x + N - 1 < w; x += N) { | 42 | 921k | if (transform_type == 6) { | 43 | 921k | auto Y = Load(d, in0 + x); | 44 | 921k | auto Co = Load(d, in1 + x); | 45 | 921k | auto Cg = Load(d, in2 + x); | 46 | 921k | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 921k | auto G = Add(Cg, Y); | 48 | 921k | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 921k | auto R = Add(Y, Co); | 50 | 921k | Store(R, d, out0 + x); | 51 | 921k | Store(G, d, out1 + x); | 52 | 921k | Store(Y, d, out2 + x); | 53 | 18.4E | } else { | 54 | 18.4E | auto First = Load(d, in0 + x); | 55 | 18.4E | auto Second = Load(d, in1 + x); | 56 | 18.4E | auto Third = Load(d, in2 + x); | 57 | 18.4E | if (third) Third = Add(Third, First); | 58 | 18.4E | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 18.4E | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 18.4E | Store(First, d, out0 + x); | 64 | 18.4E | Store(Second, d, out1 + x); | 65 | 18.4E | Store(Third, d, out2 + x); | 66 | 18.4E | } | 67 | 921k | } | 68 | 60.9k | for (; x < w; x++) { | 69 | 24.1k | if (transform_type == 6) { | 70 | 24.1k | pixel_type Y = in0[x]; | 71 | 24.1k | pixel_type Co = in1[x]; | 72 | 24.1k | pixel_type Cg = in2[x]; | 73 | 24.1k | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 24.1k | pixel_type G = PixelAdd(Cg, tmp); | 75 | 24.1k | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 24.1k | pixel_type R = PixelAdd(B, Co); | 77 | 24.1k | out0[x] = R; | 78 | 24.1k | out1[x] = G; | 79 | 24.1k | out2[x] = B; | 80 | 24.1k | } else { | 81 | 0 | pixel_type First = in0[x]; | 82 | 0 | pixel_type Second = in1[x]; | 83 | 0 | pixel_type Third = in2[x]; | 84 | 0 | if (third) Third = PixelAdd(Third, First); | 85 | 0 | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 0 | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 0 | out0[x] = First; | 91 | 0 | out1[x] = Second; | 92 | 0 | out2[x] = Third; | 93 | 0 | } | 94 | 24.1k | } | 95 | 36.8k | } |
|