/src/libjxl/lib/jxl/modular/transform/rct.cc
Line | Count | Source |
1 | | // Copyright (c) the JPEG XL Project Authors. All rights reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style |
4 | | // license that can be found in the LICENSE file. |
5 | | |
6 | | #include "lib/jxl/modular/transform/rct.h" |
7 | | |
8 | | #include <cstddef> |
9 | | #include <cstdint> |
10 | | #include <utility> |
11 | | |
12 | | #include "lib/jxl/base/data_parallel.h" |
13 | | #include "lib/jxl/base/status.h" |
14 | | #include "lib/jxl/modular/modular_image.h" |
15 | | #include "lib/jxl/modular/transform/transform.h" |
16 | | #undef HWY_TARGET_INCLUDE |
17 | | #define HWY_TARGET_INCLUDE "lib/jxl/modular/transform/rct.cc" |
18 | | #include <hwy/foreach_target.h> |
19 | | #include <hwy/highway.h> |
20 | | HWY_BEFORE_NAMESPACE(); |
21 | | namespace jxl { |
22 | | namespace HWY_NAMESPACE { |
23 | | |
24 | | // These templates are not found via ADL. |
25 | | using hwy::HWY_NAMESPACE::Add; |
26 | | using hwy::HWY_NAMESPACE::ShiftRight; |
27 | | using hwy::HWY_NAMESPACE::Sub; |
28 | | |
29 | | template <int transform_type> |
30 | | void InvRCTRow(const pixel_type* in0, const pixel_type* in1, |
31 | | const pixel_type* in2, pixel_type* out0, pixel_type* out1, |
32 | 85.9k | pixel_type* out2, size_t w) { |
33 | 85.9k | static_assert(transform_type >= 0 && transform_type < 7, |
34 | 85.9k | "Invalid transform type"); |
35 | 85.9k | int second = transform_type >> 1; |
36 | 85.9k | int third = transform_type & 1; |
37 | | |
38 | 85.9k | size_t x = 0; |
39 | 85.9k | const HWY_FULL(pixel_type) d; |
40 | 85.9k | const size_t N = Lanes(d); |
41 | 912k | for (; x + N - 1 < w; x += N) { |
42 | 826k | if (transform_type == 6) { |
43 | 502k | auto Y = Load(d, in0 + x); |
44 | 502k | auto Co = Load(d, in1 + x); |
45 | 502k | auto Cg = Load(d, in2 + x); |
46 | 502k | Y = Sub(Y, ShiftRight<1>(Cg)); |
47 | 502k | auto G = Add(Cg, Y); |
48 | 502k | Y = Sub(Y, ShiftRight<1>(Co)); |
49 | 502k | auto R = Add(Y, Co); |
50 | 502k | Store(R, d, out0 + x); |
51 | 502k | Store(G, d, out1 + x); |
52 | 502k | Store(Y, d, out2 + x); |
53 | 502k | } else { |
54 | 323k | auto First = Load(d, in0 + x); |
55 | 323k | auto Second = Load(d, in1 + x); |
56 | 323k | auto Third = Load(d, in2 + x); |
57 | 323k | if (third) Third = Add(Third, First); |
58 | 323k | if (second == 1) { |
59 | 268k | Second = Add(Second, First); |
60 | 268k | } else if (second == 2) { |
61 | 25.5k | Second = Add(Second, ShiftRight<1>(Add(First, Third))); |
62 | 25.5k | } |
63 | 323k | Store(First, d, out0 + x); |
64 | 323k | Store(Second, d, out1 + x); |
65 | 323k | Store(Third, d, out2 + x); |
66 | 323k | } |
67 | 826k | } |
68 | 161k | for (; x < w; x++) { |
69 | 75.1k | if (transform_type == 6) { |
70 | 34.5k | pixel_type Y = in0[x]; |
71 | 34.5k | pixel_type Co = in1[x]; |
72 | 34.5k | pixel_type Cg = in2[x]; |
73 | 34.5k | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); |
74 | 34.5k | pixel_type G = PixelAdd(Cg, tmp); |
75 | 34.5k | pixel_type B = PixelAdd(tmp, -(Co >> 1)); |
76 | 34.5k | pixel_type R = PixelAdd(B, Co); |
77 | 34.5k | out0[x] = R; |
78 | 34.5k | out1[x] = G; |
79 | 34.5k | out2[x] = B; |
80 | 40.6k | } else { |
81 | 40.6k | pixel_type First = in0[x]; |
82 | 40.6k | pixel_type Second = in1[x]; |
83 | 40.6k | pixel_type Third = in2[x]; |
84 | 40.6k | if (third) Third = PixelAdd(Third, First); |
85 | 40.6k | if (second == 1) { |
86 | 24.2k | Second = PixelAdd(Second, First); |
87 | 24.2k | } else if (second == 2) { |
88 | 10.6k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); |
89 | 10.6k | } |
90 | 40.6k | out0[x] = First; |
91 | 40.6k | out1[x] = Second; |
92 | 40.6k | out2[x] = Third; |
93 | 40.6k | } |
94 | 75.1k | } |
95 | 85.9k | } Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_AVX2::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long) void jxl::N_AVX2::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 2.85k | pixel_type* out2, size_t w) { | 33 | 2.85k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 2.85k | "Invalid transform type"); | 35 | 2.85k | int second = transform_type >> 1; | 36 | 2.85k | int third = transform_type & 1; | 37 | | | 38 | 2.85k | size_t x = 0; | 39 | 2.85k | const HWY_FULL(pixel_type) d; | 40 | 2.85k | const size_t N = Lanes(d); | 41 | 33.3k | for (; x + N - 1 < w; x += N) { | 42 | 30.4k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 30.4k | } else { | 54 | 30.4k | auto First = Load(d, in0 + x); | 55 | 30.4k | auto Second = Load(d, in1 + x); | 56 | 30.4k | auto Third = Load(d, in2 + x); | 57 | 30.4k | if (third) Third = Add(Third, First); | 58 | 30.4k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 30.4k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 30.4k | Store(First, d, out0 + x); | 64 | 30.4k | Store(Second, d, out1 + x); | 65 | 30.4k | Store(Third, d, out2 + x); | 66 | 30.4k | } | 67 | 30.4k | } | 68 | 8.65k | for (; x < w; x++) { | 69 | 5.79k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 5.79k | } else { | 81 | 5.79k | pixel_type First = in0[x]; | 82 | 5.79k | pixel_type Second = in1[x]; | 83 | 5.79k | pixel_type Third = in2[x]; | 84 | 5.79k | if (third) Third = PixelAdd(Third, First); | 85 | 5.79k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 5.79k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 5.79k | out0[x] = First; | 91 | 5.79k | out1[x] = Second; | 92 | 5.79k | out2[x] = Third; | 93 | 5.79k | } | 94 | 5.79k | } | 95 | 2.85k | } |
void jxl::N_AVX2::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 32.4k | pixel_type* out2, size_t w) { | 33 | 32.4k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 32.4k | "Invalid transform type"); | 35 | 32.4k | int second = transform_type >> 1; | 36 | 32.4k | int third = transform_type & 1; | 37 | | | 38 | 32.4k | size_t x = 0; | 39 | 32.4k | const HWY_FULL(pixel_type) d; | 40 | 32.4k | const size_t N = Lanes(d); | 41 | 268k | for (; x + N - 1 < w; x += N) { | 42 | 236k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 236k | } else { | 54 | 236k | auto First = Load(d, in0 + x); | 55 | 236k | auto Second = Load(d, in1 + x); | 56 | 236k | auto Third = Load(d, in2 + x); | 57 | 236k | if (third) Third = Add(Third, First); | 58 | 236k | if (second == 1) { | 59 | 236k | Second = Add(Second, First); | 60 | 236k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 236k | Store(First, d, out0 + x); | 64 | 236k | Store(Second, d, out1 + x); | 65 | 236k | Store(Third, d, out2 + x); | 66 | 236k | } | 67 | 236k | } | 68 | 49.9k | for (; x < w; x++) { | 69 | 17.5k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 17.5k | } else { | 81 | 17.5k | pixel_type First = in0[x]; | 82 | 17.5k | pixel_type Second = in1[x]; | 83 | 17.5k | pixel_type Third = in2[x]; | 84 | 17.5k | if (third) Third = PixelAdd(Third, First); | 85 | 17.5k | if (second == 1) { | 86 | 17.5k | Second = PixelAdd(Second, First); | 87 | 17.5k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 17.5k | out0[x] = First; | 91 | 17.5k | out1[x] = Second; | 92 | 17.5k | out2[x] = Third; | 93 | 17.5k | } | 94 | 17.5k | } | 95 | 32.4k | } |
void jxl::N_AVX2::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 3.38k | pixel_type* out2, size_t w) { | 33 | 3.38k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 3.38k | "Invalid transform type"); | 35 | 3.38k | int second = transform_type >> 1; | 36 | 3.38k | int third = transform_type & 1; | 37 | | | 38 | 3.38k | size_t x = 0; | 39 | 3.38k | const HWY_FULL(pixel_type) d; | 40 | 3.38k | const size_t N = Lanes(d); | 41 | 35.1k | for (; x + N - 1 < w; x += N) { | 42 | 31.8k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 31.8k | } else { | 54 | 31.8k | auto First = Load(d, in0 + x); | 55 | 31.8k | auto Second = Load(d, in1 + x); | 56 | 31.8k | auto Third = Load(d, in2 + x); | 57 | 31.8k | if (third) Third = Add(Third, First); | 58 | 31.8k | if (second == 1) { | 59 | 31.8k | Second = Add(Second, First); | 60 | 31.8k | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 31.8k | Store(First, d, out0 + x); | 64 | 31.8k | Store(Second, d, out1 + x); | 65 | 31.8k | Store(Third, d, out2 + x); | 66 | 31.8k | } | 67 | 31.8k | } | 68 | 10.0k | for (; x < w; x++) { | 69 | 6.64k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 6.64k | } else { | 81 | 6.64k | pixel_type First = in0[x]; | 82 | 6.64k | pixel_type Second = in1[x]; | 83 | 6.64k | pixel_type Third = in2[x]; | 84 | 6.64k | if (third) Third = PixelAdd(Third, First); | 85 | 6.64k | if (second == 1) { | 86 | 6.64k | Second = PixelAdd(Second, First); | 87 | 6.64k | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 6.64k | out0[x] = First; | 91 | 6.64k | out1[x] = Second; | 92 | 6.64k | out2[x] = Third; | 93 | 6.64k | } | 94 | 6.64k | } | 95 | 3.38k | } |
void jxl::N_AVX2::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 9.21k | pixel_type* out2, size_t w) { | 33 | 9.21k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 9.21k | "Invalid transform type"); | 35 | 9.21k | int second = transform_type >> 1; | 36 | 9.21k | int third = transform_type & 1; | 37 | | | 38 | 9.21k | size_t x = 0; | 39 | 9.21k | const HWY_FULL(pixel_type) d; | 40 | 9.21k | const size_t N = Lanes(d); | 41 | 29.2k | for (; x + N - 1 < w; x += N) { | 42 | 20.0k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 20.0k | } else { | 54 | 20.0k | auto First = Load(d, in0 + x); | 55 | 20.0k | auto Second = Load(d, in1 + x); | 56 | 20.0k | auto Third = Load(d, in2 + x); | 57 | 20.0k | if (third) Third = Add(Third, First); | 58 | 20.0k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 20.0k | } else if (second == 2) { | 61 | 20.0k | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 20.0k | } | 63 | 20.0k | Store(First, d, out0 + x); | 64 | 20.0k | Store(Second, d, out1 + x); | 65 | 20.0k | Store(Third, d, out2 + x); | 66 | 20.0k | } | 67 | 20.0k | } | 68 | 18.8k | for (; x < w; x++) { | 69 | 9.60k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 9.60k | } else { | 81 | 9.60k | pixel_type First = in0[x]; | 82 | 9.60k | pixel_type Second = in1[x]; | 83 | 9.60k | pixel_type Third = in2[x]; | 84 | 9.60k | if (third) Third = PixelAdd(Third, First); | 85 | 9.60k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 9.60k | } else if (second == 2) { | 88 | 9.60k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 9.60k | } | 90 | 9.60k | out0[x] = First; | 91 | 9.60k | out1[x] = Second; | 92 | 9.60k | out2[x] = Third; | 93 | 9.60k | } | 94 | 9.60k | } | 95 | 9.21k | } |
void jxl::N_AVX2::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 686 | pixel_type* out2, size_t w) { | 33 | 686 | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 686 | "Invalid transform type"); | 35 | 686 | int second = transform_type >> 1; | 36 | 686 | int third = transform_type & 1; | 37 | | | 38 | 686 | size_t x = 0; | 39 | 686 | const HWY_FULL(pixel_type) d; | 40 | 686 | const size_t N = Lanes(d); | 41 | 6.11k | for (; x + N - 1 < w; x += N) { | 42 | 5.42k | if (transform_type == 6) { | 43 | 0 | auto Y = Load(d, in0 + x); | 44 | 0 | auto Co = Load(d, in1 + x); | 45 | 0 | auto Cg = Load(d, in2 + x); | 46 | 0 | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 0 | auto G = Add(Cg, Y); | 48 | 0 | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 0 | auto R = Add(Y, Co); | 50 | 0 | Store(R, d, out0 + x); | 51 | 0 | Store(G, d, out1 + x); | 52 | 0 | Store(Y, d, out2 + x); | 53 | 5.42k | } else { | 54 | 5.42k | auto First = Load(d, in0 + x); | 55 | 5.42k | auto Second = Load(d, in1 + x); | 56 | 5.42k | auto Third = Load(d, in2 + x); | 57 | 5.42k | if (third) Third = Add(Third, First); | 58 | 5.42k | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 5.42k | } else if (second == 2) { | 61 | 5.42k | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 5.42k | } | 63 | 5.42k | Store(First, d, out0 + x); | 64 | 5.42k | Store(Second, d, out1 + x); | 65 | 5.42k | Store(Third, d, out2 + x); | 66 | 5.42k | } | 67 | 5.42k | } | 68 | 1.70k | for (; x < w; x++) { | 69 | 1.01k | if (transform_type == 6) { | 70 | 0 | pixel_type Y = in0[x]; | 71 | 0 | pixel_type Co = in1[x]; | 72 | 0 | pixel_type Cg = in2[x]; | 73 | 0 | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 0 | pixel_type G = PixelAdd(Cg, tmp); | 75 | 0 | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 0 | pixel_type R = PixelAdd(B, Co); | 77 | 0 | out0[x] = R; | 78 | 0 | out1[x] = G; | 79 | 0 | out2[x] = B; | 80 | 1.01k | } else { | 81 | 1.01k | pixel_type First = in0[x]; | 82 | 1.01k | pixel_type Second = in1[x]; | 83 | 1.01k | pixel_type Third = in2[x]; | 84 | 1.01k | if (third) Third = PixelAdd(Third, First); | 85 | 1.01k | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 1.01k | } else if (second == 2) { | 88 | 1.01k | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 1.01k | } | 90 | 1.01k | out0[x] = First; | 91 | 1.01k | out1[x] = Second; | 92 | 1.01k | out2[x] = Third; | 93 | 1.01k | } | 94 | 1.01k | } | 95 | 686 | } |
void jxl::N_AVX2::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Line | Count | Source | 32 | 37.3k | pixel_type* out2, size_t w) { | 33 | 37.3k | static_assert(transform_type >= 0 && transform_type < 7, | 34 | 37.3k | "Invalid transform type"); | 35 | 37.3k | int second = transform_type >> 1; | 36 | 37.3k | int third = transform_type & 1; | 37 | | | 38 | 37.3k | size_t x = 0; | 39 | 37.3k | const HWY_FULL(pixel_type) d; | 40 | 37.3k | const size_t N = Lanes(d); | 41 | 539k | for (; x + N - 1 < w; x += N) { | 42 | 502k | if (transform_type == 6) { | 43 | 502k | auto Y = Load(d, in0 + x); | 44 | 502k | auto Co = Load(d, in1 + x); | 45 | 502k | auto Cg = Load(d, in2 + x); | 46 | 502k | Y = Sub(Y, ShiftRight<1>(Cg)); | 47 | 502k | auto G = Add(Cg, Y); | 48 | 502k | Y = Sub(Y, ShiftRight<1>(Co)); | 49 | 502k | auto R = Add(Y, Co); | 50 | 502k | Store(R, d, out0 + x); | 51 | 502k | Store(G, d, out1 + x); | 52 | 502k | Store(Y, d, out2 + x); | 53 | 502k | } else { | 54 | 0 | auto First = Load(d, in0 + x); | 55 | 0 | auto Second = Load(d, in1 + x); | 56 | 0 | auto Third = Load(d, in2 + x); | 57 | 0 | if (third) Third = Add(Third, First); | 58 | 0 | if (second == 1) { | 59 | 0 | Second = Add(Second, First); | 60 | 0 | } else if (second == 2) { | 61 | 0 | Second = Add(Second, ShiftRight<1>(Add(First, Third))); | 62 | 0 | } | 63 | 0 | Store(First, d, out0 + x); | 64 | 0 | Store(Second, d, out1 + x); | 65 | 0 | Store(Third, d, out2 + x); | 66 | 0 | } | 67 | 502k | } | 68 | 71.9k | for (; x < w; x++) { | 69 | 34.5k | if (transform_type == 6) { | 70 | 34.5k | pixel_type Y = in0[x]; | 71 | 34.5k | pixel_type Co = in1[x]; | 72 | 34.5k | pixel_type Cg = in2[x]; | 73 | 34.5k | pixel_type tmp = PixelAdd(Y, -(Cg >> 1)); | 74 | 34.5k | pixel_type G = PixelAdd(Cg, tmp); | 75 | 34.5k | pixel_type B = PixelAdd(tmp, -(Co >> 1)); | 76 | 34.5k | pixel_type R = PixelAdd(B, Co); | 77 | 34.5k | out0[x] = R; | 78 | 34.5k | out1[x] = G; | 79 | 34.5k | out2[x] = B; | 80 | 34.5k | } else { | 81 | 0 | pixel_type First = in0[x]; | 82 | 0 | pixel_type Second = in1[x]; | 83 | 0 | pixel_type Third = in2[x]; | 84 | 0 | if (third) Third = PixelAdd(Third, First); | 85 | 0 | if (second == 1) { | 86 | 0 | Second = PixelAdd(Second, First); | 87 | 0 | } else if (second == 2) { | 88 | 0 | Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1)); | 89 | 0 | } | 90 | 0 | out0[x] = First; | 91 | 0 | out1[x] = Second; | 92 | 0 | out2[x] = Third; | 93 | 0 | } | 94 | 34.5k | } | 95 | 37.3k | } |
Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long) Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long) |
96 | | |
97 | 4.67k | Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) { |
98 | 4.67k | JXL_RETURN_IF_ERROR(CheckEqualChannels(input, begin_c, begin_c + 2)); |
99 | 4.67k | size_t m = begin_c; |
100 | 4.67k | Channel& c0 = input.channel[m + 0]; |
101 | 4.67k | size_t w = c0.w; |
102 | 4.67k | size_t h = c0.h; |
103 | 4.67k | if (rct_type == 0) { // noop |
104 | 1.42k | return true; |
105 | 1.42k | } |
106 | | // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR |
107 | 3.25k | int permutation = rct_type / 7; |
108 | 3.25k | JXL_ENSURE(permutation < 6); |
109 | | // 0-5 values have the low bit corresponding to Third and the high bits |
110 | | // corresponding to Second. 6 corresponds to YCoCg. |
111 | | // |
112 | | // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird |
113 | | // |
114 | | // Third: 0=nop, 1=SubtractFirst |
115 | 3.25k | int custom = rct_type % 7; |
116 | | // Special case: permute-only. Swap channels around. |
117 | 3.25k | if (custom == 0) { |
118 | 18 | Channel ch0 = std::move(input.channel[m]); |
119 | 18 | Channel ch1 = std::move(input.channel[m + 1]); |
120 | 18 | Channel ch2 = std::move(input.channel[m + 2]); |
121 | 18 | input.channel[m + (permutation % 3)] = std::move(ch0); |
122 | 18 | input.channel[m + ((permutation + 1 + permutation / 3) % 3)] = |
123 | 18 | std::move(ch1); |
124 | 18 | input.channel[m + ((permutation + 2 - permutation / 3) % 3)] = |
125 | 18 | std::move(ch2); |
126 | 18 | return true; |
127 | 18 | } |
128 | 3.23k | constexpr decltype(&InvRCTRow<0>) inv_rct_row[] = { |
129 | 3.23k | InvRCTRow<0>, InvRCTRow<1>, InvRCTRow<2>, InvRCTRow<3>, |
130 | 3.23k | InvRCTRow<4>, InvRCTRow<5>, InvRCTRow<6>}; |
131 | 3.23k | const auto process_row = [&](const uint32_t task, |
132 | 85.9k | size_t /* thread */) -> Status { |
133 | 85.9k | const size_t y = task; |
134 | 85.9k | const pixel_type* in0 = input.channel[m].Row(y); |
135 | 85.9k | const pixel_type* in1 = input.channel[m + 1].Row(y); |
136 | 85.9k | const pixel_type* in2 = input.channel[m + 2].Row(y); |
137 | 85.9k | pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y); |
138 | 85.9k | pixel_type* out1 = |
139 | 85.9k | input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y); |
140 | 85.9k | pixel_type* out2 = |
141 | 85.9k | input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y); |
142 | 85.9k | inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w); |
143 | 85.9k | return true; |
144 | 85.9k | }; Unexecuted instantiation: rct.cc:jxl::N_SSE4::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const rct.cc:jxl::N_AVX2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const Line | Count | Source | 132 | 85.9k | size_t /* thread */) -> Status { | 133 | 85.9k | const size_t y = task; | 134 | 85.9k | const pixel_type* in0 = input.channel[m].Row(y); | 135 | 85.9k | const pixel_type* in1 = input.channel[m + 1].Row(y); | 136 | 85.9k | const pixel_type* in2 = input.channel[m + 2].Row(y); | 137 | 85.9k | pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y); | 138 | 85.9k | pixel_type* out1 = | 139 | 85.9k | input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y); | 140 | 85.9k | pixel_type* out2 = | 141 | 85.9k | input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y); | 142 | 85.9k | inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w); | 143 | 85.9k | return true; | 144 | 85.9k | }; |
Unexecuted instantiation: rct.cc:jxl::N_SSE2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const |
145 | 3.23k | JXL_RETURN_IF_ERROR( |
146 | 3.23k | RunOnPool(pool, 0, h, ThreadPool::NoInit, process_row, "InvRCT")); |
147 | 3.23k | return true; |
148 | 3.23k | } Unexecuted instantiation: jxl::N_SSE4::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*) jxl::N_AVX2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*) Line | Count | Source | 97 | 4.67k | Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) { | 98 | 4.67k | JXL_RETURN_IF_ERROR(CheckEqualChannels(input, begin_c, begin_c + 2)); | 99 | 4.67k | size_t m = begin_c; | 100 | 4.67k | Channel& c0 = input.channel[m + 0]; | 101 | 4.67k | size_t w = c0.w; | 102 | 4.67k | size_t h = c0.h; | 103 | 4.67k | if (rct_type == 0) { // noop | 104 | 1.42k | return true; | 105 | 1.42k | } | 106 | | // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR | 107 | 3.25k | int permutation = rct_type / 7; | 108 | 3.25k | JXL_ENSURE(permutation < 6); | 109 | | // 0-5 values have the low bit corresponding to Third and the high bits | 110 | | // corresponding to Second. 6 corresponds to YCoCg. | 111 | | // | 112 | | // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird | 113 | | // | 114 | | // Third: 0=nop, 1=SubtractFirst | 115 | 3.25k | int custom = rct_type % 7; | 116 | | // Special case: permute-only. Swap channels around. | 117 | 3.25k | if (custom == 0) { | 118 | 18 | Channel ch0 = std::move(input.channel[m]); | 119 | 18 | Channel ch1 = std::move(input.channel[m + 1]); | 120 | 18 | Channel ch2 = std::move(input.channel[m + 2]); | 121 | 18 | input.channel[m + (permutation % 3)] = std::move(ch0); | 122 | 18 | input.channel[m + ((permutation + 1 + permutation / 3) % 3)] = | 123 | 18 | std::move(ch1); | 124 | 18 | input.channel[m + ((permutation + 2 - permutation / 3) % 3)] = | 125 | 18 | std::move(ch2); | 126 | 18 | return true; | 127 | 18 | } | 128 | 3.23k | constexpr decltype(&InvRCTRow<0>) inv_rct_row[] = { | 129 | 3.23k | InvRCTRow<0>, InvRCTRow<1>, InvRCTRow<2>, InvRCTRow<3>, | 130 | 3.23k | InvRCTRow<4>, InvRCTRow<5>, InvRCTRow<6>}; | 131 | 3.23k | const auto process_row = [&](const uint32_t task, | 132 | 3.23k | size_t /* thread */) -> Status { | 133 | 3.23k | const size_t y = task; | 134 | 3.23k | const pixel_type* in0 = input.channel[m].Row(y); | 135 | 3.23k | const pixel_type* in1 = input.channel[m + 1].Row(y); | 136 | 3.23k | const pixel_type* in2 = input.channel[m + 2].Row(y); | 137 | 3.23k | pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y); | 138 | 3.23k | pixel_type* out1 = | 139 | 3.23k | input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y); | 140 | 3.23k | pixel_type* out2 = | 141 | 3.23k | input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y); | 142 | 3.23k | inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w); | 143 | 3.23k | return true; | 144 | 3.23k | }; | 145 | 3.23k | JXL_RETURN_IF_ERROR( | 146 | 3.23k | RunOnPool(pool, 0, h, ThreadPool::NoInit, process_row, "InvRCT")); | 147 | 3.23k | return true; | 148 | 3.23k | } |
Unexecuted instantiation: jxl::N_SSE2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*) |
149 | | |
150 | | } // namespace HWY_NAMESPACE |
151 | | } // namespace jxl |
152 | | HWY_AFTER_NAMESPACE(); |
153 | | |
154 | | #if HWY_ONCE |
155 | | namespace jxl { |
156 | | |
157 | | HWY_EXPORT(InvRCT); |
158 | 4.67k | Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) { |
159 | 4.67k | return HWY_DYNAMIC_DISPATCH(InvRCT)(input, begin_c, rct_type, pool); |
160 | 4.67k | } |
161 | | |
162 | | } // namespace jxl |
163 | | #endif |