Coverage Report

Created: 2026-08-14 08:18

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libjxl/lib/jxl/modular/transform/rct.cc
Line
Count
Source
1
// Copyright (c) the JPEG XL Project Authors. All rights reserved.
2
//
3
// Use of this source code is governed by a BSD-style
4
// license that can be found in the LICENSE file.
5
6
#include "lib/jxl/modular/transform/rct.h"
7
8
#include <cstddef>
9
#include <cstdint>
10
#include <utility>
11
12
#include "lib/jxl/base/data_parallel.h"
13
#include "lib/jxl/base/status.h"
14
#include "lib/jxl/modular/modular_image.h"
15
#include "lib/jxl/modular/transform/transform.h"
16
#undef HWY_TARGET_INCLUDE
17
#define HWY_TARGET_INCLUDE "lib/jxl/modular/transform/rct.cc"
18
#include <hwy/foreach_target.h>
19
#include <hwy/highway.h>
20
HWY_BEFORE_NAMESPACE();
21
namespace jxl {
22
namespace HWY_NAMESPACE {
23
24
// These templates are not found via ADL.
25
using hwy::HWY_NAMESPACE::Add;
26
using hwy::HWY_NAMESPACE::ShiftRight;
27
using hwy::HWY_NAMESPACE::Sub;
28
29
template <int transform_type>
30
void InvRCTRow(const pixel_type* in0, const pixel_type* in1,
31
               const pixel_type* in2, pixel_type* out0, pixel_type* out1,
32
6.88M
               pixel_type* out2, size_t w) {
33
6.88M
  static_assert(transform_type >= 0 && transform_type < 7,
34
6.88M
                "Invalid transform type");
35
6.88M
  int second = transform_type >> 1;
36
6.88M
  int third = transform_type & 1;
37
38
6.88M
  size_t x = 0;
39
6.88M
  const HWY_FULL(pixel_type) d;
40
6.88M
  const size_t N = Lanes(d);
41
41.1M
  for (; x + N - 1 < w; x += N) {
42
34.2M
    if (transform_type == 6) {
43
19.1M
      auto Y = Load(d, in0 + x);
44
19.1M
      auto Co = Load(d, in1 + x);
45
19.1M
      auto Cg = Load(d, in2 + x);
46
19.1M
      Y = Sub(Y, ShiftRight<1>(Cg));
47
19.1M
      auto G = Add(Cg, Y);
48
19.1M
      Y = Sub(Y, ShiftRight<1>(Co));
49
19.1M
      auto R = Add(Y, Co);
50
19.1M
      Store(R, d, out0 + x);
51
19.1M
      Store(G, d, out1 + x);
52
19.1M
      Store(Y, d, out2 + x);
53
19.1M
    } else {
54
15.0M
      auto First = Load(d, in0 + x);
55
15.0M
      auto Second = Load(d, in1 + x);
56
15.0M
      auto Third = Load(d, in2 + x);
57
15.0M
      if (third) Third = Add(Third, First);
58
15.0M
      if (second == 1) {
59
2.66M
        Second = Add(Second, First);
60
12.4M
      } else if (second == 2) {
61
11.6M
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
11.6M
      }
63
15.0M
      Store(First, d, out0 + x);
64
15.0M
      Store(Second, d, out1 + x);
65
15.0M
      Store(Third, d, out2 + x);
66
15.0M
    }
67
34.2M
  }
68
47.3M
  for (; x < w; x++) {
69
40.4M
    if (transform_type == 6) {
70
38.2M
      pixel_type Y = in0[x];
71
38.2M
      pixel_type Co = in1[x];
72
38.2M
      pixel_type Cg = in2[x];
73
38.2M
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
38.2M
      pixel_type G = PixelAdd(Cg, tmp);
75
38.2M
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
38.2M
      pixel_type R = PixelAdd(B, Co);
77
38.2M
      out0[x] = R;
78
38.2M
      out1[x] = G;
79
38.2M
      out2[x] = B;
80
38.2M
    } else {
81
2.16M
      pixel_type First = in0[x];
82
2.16M
      pixel_type Second = in1[x];
83
2.16M
      pixel_type Third = in2[x];
84
2.16M
      if (third) Third = PixelAdd(Third, First);
85
2.16M
      if (second == 1) {
86
915k
        Second = PixelAdd(Second, First);
87
1.24M
      } else if (second == 2) {
88
622k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
622k
      }
90
2.16M
      out0[x] = First;
91
2.16M
      out1[x] = Second;
92
2.16M
      out2[x] = Third;
93
2.16M
    }
94
40.4M
  }
95
6.88M
}
Unexecuted instantiation: void jxl::N_SSE4::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
void jxl::N_SSE4::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
5.07k
               pixel_type* out2, size_t w) {
33
5.07k
  static_assert(transform_type >= 0 && transform_type < 7,
34
5.07k
                "Invalid transform type");
35
5.07k
  int second = transform_type >> 1;
36
5.07k
  int third = transform_type & 1;
37
38
5.07k
  size_t x = 0;
39
5.07k
  const HWY_FULL(pixel_type) d;
40
5.07k
  const size_t N = Lanes(d);
41
99.8k
  for (; x + N - 1 < w; x += N) {
42
94.7k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
94.7k
    } else {
54
94.7k
      auto First = Load(d, in0 + x);
55
94.7k
      auto Second = Load(d, in1 + x);
56
94.7k
      auto Third = Load(d, in2 + x);
57
94.8k
      if (third) Third = Add(Third, First);
58
94.7k
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
94.7k
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
94.7k
      Store(First, d, out0 + x);
64
94.7k
      Store(Second, d, out1 + x);
65
94.7k
      Store(Third, d, out2 + x);
66
94.7k
    }
67
94.7k
  }
68
15.5k
  for (; x < w; x++) {
69
10.4k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
10.4k
    } else {
81
10.4k
      pixel_type First = in0[x];
82
10.4k
      pixel_type Second = in1[x];
83
10.4k
      pixel_type Third = in2[x];
84
10.4k
      if (third) Third = PixelAdd(Third, First);
85
10.4k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
10.4k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
10.4k
      out0[x] = First;
91
10.4k
      out1[x] = Second;
92
10.4k
      out2[x] = Third;
93
10.4k
    }
94
10.4k
  }
95
5.07k
}
void jxl::N_SSE4::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
3.52k
               pixel_type* out2, size_t w) {
33
3.52k
  static_assert(transform_type >= 0 && transform_type < 7,
34
3.52k
                "Invalid transform type");
35
3.52k
  int second = transform_type >> 1;
36
3.52k
  int third = transform_type & 1;
37
38
3.52k
  size_t x = 0;
39
3.52k
  const HWY_FULL(pixel_type) d;
40
3.52k
  const size_t N = Lanes(d);
41
85.6k
  for (; x + N - 1 < w; x += N) {
42
82.1k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
82.1k
    } else {
54
82.1k
      auto First = Load(d, in0 + x);
55
82.1k
      auto Second = Load(d, in1 + x);
56
82.1k
      auto Third = Load(d, in2 + x);
57
82.1k
      if (third) Third = Add(Third, First);
58
82.2k
      if (second == 1) {
59
82.2k
        Second = Add(Second, First);
60
18.4E
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
82.1k
      Store(First, d, out0 + x);
64
82.1k
      Store(Second, d, out1 + x);
65
82.1k
      Store(Third, d, out2 + x);
66
82.1k
    }
67
82.1k
  }
68
6.58k
  for (; x < w; x++) {
69
3.05k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
3.05k
    } else {
81
3.05k
      pixel_type First = in0[x];
82
3.05k
      pixel_type Second = in1[x];
83
3.05k
      pixel_type Third = in2[x];
84
3.05k
      if (third) Third = PixelAdd(Third, First);
85
3.05k
      if (second == 1) {
86
3.05k
        Second = PixelAdd(Second, First);
87
3.05k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
3.05k
      out0[x] = First;
91
3.05k
      out1[x] = Second;
92
3.05k
      out2[x] = Third;
93
3.05k
    }
94
3.05k
  }
95
3.52k
}
void jxl::N_SSE4::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
3.17k
               pixel_type* out2, size_t w) {
33
3.17k
  static_assert(transform_type >= 0 && transform_type < 7,
34
3.17k
                "Invalid transform type");
35
3.17k
  int second = transform_type >> 1;
36
3.17k
  int third = transform_type & 1;
37
38
3.17k
  size_t x = 0;
39
3.17k
  const HWY_FULL(pixel_type) d;
40
3.17k
  const size_t N = Lanes(d);
41
75.5k
  for (; x + N - 1 < w; x += N) {
42
72.3k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
72.3k
    } else {
54
72.3k
      auto First = Load(d, in0 + x);
55
72.3k
      auto Second = Load(d, in1 + x);
56
72.3k
      auto Third = Load(d, in2 + x);
57
72.3k
      if (third) Third = Add(Third, First);
58
72.3k
      if (second == 1) {
59
72.3k
        Second = Add(Second, First);
60
72.3k
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
72.3k
      Store(First, d, out0 + x);
64
72.3k
      Store(Second, d, out1 + x);
65
72.3k
      Store(Third, d, out2 + x);
66
72.3k
    }
67
72.3k
  }
68
5.50k
  for (; x < w; x++) {
69
2.32k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
2.32k
    } else {
81
2.32k
      pixel_type First = in0[x];
82
2.32k
      pixel_type Second = in1[x];
83
2.32k
      pixel_type Third = in2[x];
84
2.32k
      if (third) Third = PixelAdd(Third, First);
85
2.32k
      if (second == 1) {
86
2.32k
        Second = PixelAdd(Second, First);
87
2.32k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
2.32k
      out0[x] = First;
91
2.32k
      out1[x] = Second;
92
2.32k
      out2[x] = Third;
93
2.32k
    }
94
2.32k
  }
95
3.17k
}
void jxl::N_SSE4::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
12.2k
               pixel_type* out2, size_t w) {
33
12.2k
  static_assert(transform_type >= 0 && transform_type < 7,
34
12.2k
                "Invalid transform type");
35
12.2k
  int second = transform_type >> 1;
36
12.2k
  int third = transform_type & 1;
37
38
12.2k
  size_t x = 0;
39
12.2k
  const HWY_FULL(pixel_type) d;
40
12.2k
  const size_t N = Lanes(d);
41
340k
  for (; x + N - 1 < w; x += N) {
42
328k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
328k
    } else {
54
328k
      auto First = Load(d, in0 + x);
55
328k
      auto Second = Load(d, in1 + x);
56
328k
      auto Third = Load(d, in2 + x);
57
328k
      if (third) Third = Add(Third, First);
58
328k
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
328k
      } else if (second == 2) {
61
328k
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
328k
      }
63
328k
      Store(First, d, out0 + x);
64
328k
      Store(Second, d, out1 + x);
65
328k
      Store(Third, d, out2 + x);
66
328k
    }
67
328k
  }
68
19.4k
  for (; x < w; x++) {
69
7.27k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
7.27k
    } else {
81
7.27k
      pixel_type First = in0[x];
82
7.27k
      pixel_type Second = in1[x];
83
7.27k
      pixel_type Third = in2[x];
84
7.27k
      if (third) Third = PixelAdd(Third, First);
85
7.27k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
7.27k
      } else if (second == 2) {
88
7.27k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
7.27k
      }
90
7.27k
      out0[x] = First;
91
7.27k
      out1[x] = Second;
92
7.27k
      out2[x] = Third;
93
7.27k
    }
94
7.27k
  }
95
12.2k
}
void jxl::N_SSE4::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
94.7k
               pixel_type* out2, size_t w) {
33
94.7k
  static_assert(transform_type >= 0 && transform_type < 7,
34
94.7k
                "Invalid transform type");
35
94.7k
  int second = transform_type >> 1;
36
94.7k
  int third = transform_type & 1;
37
38
94.7k
  size_t x = 0;
39
94.7k
  const HWY_FULL(pixel_type) d;
40
94.7k
  const size_t N = Lanes(d);
41
4.10M
  for (; x + N - 1 < w; x += N) {
42
4.01M
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
4.01M
    } else {
54
4.01M
      auto First = Load(d, in0 + x);
55
4.01M
      auto Second = Load(d, in1 + x);
56
4.01M
      auto Third = Load(d, in2 + x);
57
4.02M
      if (third) Third = Add(Third, First);
58
4.01M
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
4.01M
      } else if (second == 2) {
61
4.00M
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
4.00M
      }
63
4.01M
      Store(First, d, out0 + x);
64
4.01M
      Store(Second, d, out1 + x);
65
4.01M
      Store(Third, d, out2 + x);
66
4.01M
    }
67
4.01M
  }
68
96.3k
  for (; x < w; x++) {
69
1.60k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
1.60k
    } else {
81
1.60k
      pixel_type First = in0[x];
82
1.60k
      pixel_type Second = in1[x];
83
1.60k
      pixel_type Third = in2[x];
84
1.60k
      if (third) Third = PixelAdd(Third, First);
85
1.60k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
1.60k
      } else if (second == 2) {
88
1.60k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
1.60k
      }
90
1.60k
      out0[x] = First;
91
1.60k
      out1[x] = Second;
92
1.60k
      out2[x] = Third;
93
1.60k
    }
94
1.60k
  }
95
94.7k
}
void jxl::N_SSE4::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
24.5k
               pixel_type* out2, size_t w) {
33
24.5k
  static_assert(transform_type >= 0 && transform_type < 7,
34
24.5k
                "Invalid transform type");
35
24.5k
  int second = transform_type >> 1;
36
24.5k
  int third = transform_type & 1;
37
38
24.5k
  size_t x = 0;
39
24.5k
  const HWY_FULL(pixel_type) d;
40
24.5k
  const size_t N = Lanes(d);
41
749k
  for (; x + N - 1 < w; x += N) {
42
724k
    if (transform_type == 6) {
43
724k
      auto Y = Load(d, in0 + x);
44
724k
      auto Co = Load(d, in1 + x);
45
724k
      auto Cg = Load(d, in2 + x);
46
724k
      Y = Sub(Y, ShiftRight<1>(Cg));
47
724k
      auto G = Add(Cg, Y);
48
724k
      Y = Sub(Y, ShiftRight<1>(Co));
49
724k
      auto R = Add(Y, Co);
50
724k
      Store(R, d, out0 + x);
51
724k
      Store(G, d, out1 + x);
52
724k
      Store(Y, d, out2 + x);
53
18.4E
    } else {
54
18.4E
      auto First = Load(d, in0 + x);
55
18.4E
      auto Second = Load(d, in1 + x);
56
18.4E
      auto Third = Load(d, in2 + x);
57
18.4E
      if (third) Third = Add(Third, First);
58
18.4E
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
18.4E
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
18.4E
      Store(First, d, out0 + x);
64
18.4E
      Store(Second, d, out1 + x);
65
18.4E
      Store(Third, d, out2 + x);
66
18.4E
    }
67
724k
  }
68
44.7k
  for (; x < w; x++) {
69
20.1k
    if (transform_type == 6) {
70
20.1k
      pixel_type Y = in0[x];
71
20.1k
      pixel_type Co = in1[x];
72
20.1k
      pixel_type Cg = in2[x];
73
20.1k
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
20.1k
      pixel_type G = PixelAdd(Cg, tmp);
75
20.1k
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
20.1k
      pixel_type R = PixelAdd(B, Co);
77
20.1k
      out0[x] = R;
78
20.1k
      out1[x] = G;
79
20.1k
      out2[x] = B;
80
20.1k
    } else {
81
0
      pixel_type First = in0[x];
82
0
      pixel_type Second = in1[x];
83
0
      pixel_type Third = in2[x];
84
0
      if (third) Third = PixelAdd(Third, First);
85
0
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
0
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
0
      out0[x] = First;
91
0
      out1[x] = Second;
92
0
      out2[x] = Third;
93
0
    }
94
20.1k
  }
95
24.5k
}
Unexecuted instantiation: void jxl::N_AVX2::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
void jxl::N_AVX2::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
246k
               pixel_type* out2, size_t w) {
33
246k
  static_assert(transform_type >= 0 && transform_type < 7,
34
246k
                "Invalid transform type");
35
246k
  int second = transform_type >> 1;
36
246k
  int third = transform_type & 1;
37
38
246k
  size_t x = 0;
39
246k
  const HWY_FULL(pixel_type) d;
40
246k
  const size_t N = Lanes(d);
41
822k
  for (; x + N - 1 < w; x += N) {
42
575k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
575k
    } else {
54
575k
      auto First = Load(d, in0 + x);
55
575k
      auto Second = Load(d, in1 + x);
56
575k
      auto Third = Load(d, in2 + x);
57
575k
      if (third) Third = Add(Third, First);
58
575k
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
575k
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
575k
      Store(First, d, out0 + x);
64
575k
      Store(Second, d, out1 + x);
65
575k
      Store(Third, d, out2 + x);
66
575k
    }
67
575k
  }
68
854k
  for (; x < w; x++) {
69
608k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
608k
    } else {
81
608k
      pixel_type First = in0[x];
82
608k
      pixel_type Second = in1[x];
83
608k
      pixel_type Third = in2[x];
84
608k
      if (third) Third = PixelAdd(Third, First);
85
608k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
608k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
608k
      out0[x] = First;
91
608k
      out1[x] = Second;
92
608k
      out2[x] = Third;
93
608k
    }
94
608k
  }
95
246k
}
void jxl::N_AVX2::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
173k
               pixel_type* out2, size_t w) {
33
173k
  static_assert(transform_type >= 0 && transform_type < 7,
34
173k
                "Invalid transform type");
35
173k
  int second = transform_type >> 1;
36
173k
  int third = transform_type & 1;
37
38
173k
  size_t x = 0;
39
173k
  const HWY_FULL(pixel_type) d;
40
173k
  const size_t N = Lanes(d);
41
1.39M
  for (; x + N - 1 < w; x += N) {
42
1.22M
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
1.22M
    } else {
54
1.22M
      auto First = Load(d, in0 + x);
55
1.22M
      auto Second = Load(d, in1 + x);
56
1.22M
      auto Third = Load(d, in2 + x);
57
1.22M
      if (third) Third = Add(Third, First);
58
1.22M
      if (second == 1) {
59
1.22M
        Second = Add(Second, First);
60
18.4E
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
1.22M
      Store(First, d, out0 + x);
64
1.22M
      Store(Second, d, out1 + x);
65
1.22M
      Store(Third, d, out2 + x);
66
1.22M
    }
67
1.22M
  }
68
634k
  for (; x < w; x++) {
69
460k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
460k
    } else {
81
460k
      pixel_type First = in0[x];
82
460k
      pixel_type Second = in1[x];
83
460k
      pixel_type Third = in2[x];
84
460k
      if (third) Third = PixelAdd(Third, First);
85
460k
      if (second == 1) {
86
460k
        Second = PixelAdd(Second, First);
87
460k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
460k
      out0[x] = First;
91
460k
      out1[x] = Second;
92
460k
      out2[x] = Third;
93
460k
    }
94
460k
  }
95
173k
}
void jxl::N_AVX2::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
102k
               pixel_type* out2, size_t w) {
33
102k
  static_assert(transform_type >= 0 && transform_type < 7,
34
102k
                "Invalid transform type");
35
102k
  int second = transform_type >> 1;
36
102k
  int third = transform_type & 1;
37
38
102k
  size_t x = 0;
39
102k
  const HWY_FULL(pixel_type) d;
40
102k
  const size_t N = Lanes(d);
41
1.14M
  for (; x + N - 1 < w; x += N) {
42
1.04M
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
1.04M
    } else {
54
1.04M
      auto First = Load(d, in0 + x);
55
1.04M
      auto Second = Load(d, in1 + x);
56
1.04M
      auto Third = Load(d, in2 + x);
57
1.04M
      if (third) Third = Add(Third, First);
58
1.04M
      if (second == 1) {
59
1.04M
        Second = Add(Second, First);
60
18.4E
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
1.04M
      Store(First, d, out0 + x);
64
1.04M
      Store(Second, d, out1 + x);
65
1.04M
      Store(Third, d, out2 + x);
66
1.04M
    }
67
1.04M
  }
68
539k
  for (; x < w; x++) {
69
437k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
437k
    } else {
81
437k
      pixel_type First = in0[x];
82
437k
      pixel_type Second = in1[x];
83
437k
      pixel_type Third = in2[x];
84
437k
      if (third) Third = PixelAdd(Third, First);
85
437k
      if (second == 1) {
86
437k
        Second = PixelAdd(Second, First);
87
437k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
437k
      out0[x] = First;
91
437k
      out1[x] = Second;
92
437k
      out2[x] = Third;
93
437k
    }
94
437k
  }
95
102k
}
void jxl::N_AVX2::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
86.3k
               pixel_type* out2, size_t w) {
33
86.3k
  static_assert(transform_type >= 0 && transform_type < 7,
34
86.3k
                "Invalid transform type");
35
86.3k
  int second = transform_type >> 1;
36
86.3k
  int third = transform_type & 1;
37
38
86.3k
  size_t x = 0;
39
86.3k
  const HWY_FULL(pixel_type) d;
40
86.3k
  const size_t N = Lanes(d);
41
645k
  for (; x + N - 1 < w; x += N) {
42
559k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
559k
    } else {
54
559k
      auto First = Load(d, in0 + x);
55
559k
      auto Second = Load(d, in1 + x);
56
559k
      auto Third = Load(d, in2 + x);
57
559k
      if (third) Third = Add(Third, First);
58
559k
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
559k
      } else if (second == 2) {
61
559k
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
559k
      }
63
559k
      Store(First, d, out0 + x);
64
559k
      Store(Second, d, out1 + x);
65
559k
      Store(Third, d, out2 + x);
66
559k
    }
67
559k
  }
68
204k
  for (; x < w; x++) {
69
118k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
118k
    } else {
81
118k
      pixel_type First = in0[x];
82
118k
      pixel_type Second = in1[x];
83
118k
      pixel_type Third = in2[x];
84
118k
      if (third) Third = PixelAdd(Third, First);
85
118k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
118k
      } else if (second == 2) {
88
118k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
118k
      }
90
118k
      out0[x] = First;
91
118k
      out1[x] = Second;
92
118k
      out2[x] = Third;
93
118k
    }
94
118k
  }
95
86.3k
}
void jxl::N_AVX2::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
221k
               pixel_type* out2, size_t w) {
33
221k
  static_assert(transform_type >= 0 && transform_type < 7,
34
221k
                "Invalid transform type");
35
221k
  int second = transform_type >> 1;
36
221k
  int third = transform_type & 1;
37
38
221k
  size_t x = 0;
39
221k
  const HWY_FULL(pixel_type) d;
40
221k
  const size_t N = Lanes(d);
41
3.40M
  for (; x + N - 1 < w; x += N) {
42
3.18M
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
3.18M
    } else {
54
3.18M
      auto First = Load(d, in0 + x);
55
3.18M
      auto Second = Load(d, in1 + x);
56
3.18M
      auto Third = Load(d, in2 + x);
57
3.18M
      if (third) Third = Add(Third, First);
58
3.18M
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
3.18M
      } else if (second == 2) {
61
3.18M
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
3.18M
      }
63
3.18M
      Store(First, d, out0 + x);
64
3.18M
      Store(Second, d, out1 + x);
65
3.18M
      Store(Third, d, out2 + x);
66
3.18M
    }
67
3.18M
  }
68
696k
  for (; x < w; x++) {
69
474k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
474k
    } else {
81
474k
      pixel_type First = in0[x];
82
474k
      pixel_type Second = in1[x];
83
474k
      pixel_type Third = in2[x];
84
474k
      if (third) Third = PixelAdd(Third, First);
85
474k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
474k
      } else if (second == 2) {
88
474k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
474k
      }
90
474k
      out0[x] = First;
91
474k
      out1[x] = Second;
92
474k
      out2[x] = Third;
93
474k
    }
94
474k
  }
95
221k
}
void jxl::N_AVX2::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
5.76M
               pixel_type* out2, size_t w) {
33
5.76M
  static_assert(transform_type >= 0 && transform_type < 7,
34
5.76M
                "Invalid transform type");
35
5.76M
  int second = transform_type >> 1;
36
5.76M
  int third = transform_type & 1;
37
38
5.76M
  size_t x = 0;
39
5.76M
  const HWY_FULL(pixel_type) d;
40
5.76M
  const size_t N = Lanes(d);
41
23.3M
  for (; x + N - 1 < w; x += N) {
42
17.5M
    if (transform_type == 6) {
43
17.5M
      auto Y = Load(d, in0 + x);
44
17.5M
      auto Co = Load(d, in1 + x);
45
17.5M
      auto Cg = Load(d, in2 + x);
46
17.5M
      Y = Sub(Y, ShiftRight<1>(Cg));
47
17.5M
      auto G = Add(Cg, Y);
48
17.5M
      Y = Sub(Y, ShiftRight<1>(Co));
49
17.5M
      auto R = Add(Y, Co);
50
17.5M
      Store(R, d, out0 + x);
51
17.5M
      Store(G, d, out1 + x);
52
17.5M
      Store(Y, d, out2 + x);
53
17.5M
    } else {
54
9
      auto First = Load(d, in0 + x);
55
9
      auto Second = Load(d, in1 + x);
56
9
      auto Third = Load(d, in2 + x);
57
9
      if (third) Third = Add(Third, First);
58
9
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
9
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
9
      Store(First, d, out0 + x);
64
9
      Store(Second, d, out1 + x);
65
9
      Store(Third, d, out2 + x);
66
9
    }
67
17.5M
  }
68
43.9M
  for (; x < w; x++) {
69
38.2M
    if (transform_type == 6) {
70
38.2M
      pixel_type Y = in0[x];
71
38.2M
      pixel_type Co = in1[x];
72
38.2M
      pixel_type Cg = in2[x];
73
38.2M
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
38.2M
      pixel_type G = PixelAdd(Cg, tmp);
75
38.2M
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
38.2M
      pixel_type R = PixelAdd(B, Co);
77
38.2M
      out0[x] = R;
78
38.2M
      out1[x] = G;
79
38.2M
      out2[x] = B;
80
38.2M
    } else {
81
0
      pixel_type First = in0[x];
82
0
      pixel_type Second = in1[x];
83
0
      pixel_type Third = in2[x];
84
0
      if (third) Third = PixelAdd(Third, First);
85
0
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
0
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
0
      out0[x] = First;
91
0
      out1[x] = Second;
92
0
      out2[x] = Third;
93
0
    }
94
38.2M
  }
95
5.76M
}
Unexecuted instantiation: void jxl::N_SSE2::InvRCTRow<0>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
void jxl::N_SSE2::InvRCTRow<1>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
4.62k
               pixel_type* out2, size_t w) {
33
4.62k
  static_assert(transform_type >= 0 && transform_type < 7,
34
4.62k
                "Invalid transform type");
35
4.62k
  int second = transform_type >> 1;
36
4.62k
  int third = transform_type & 1;
37
38
4.62k
  size_t x = 0;
39
4.62k
  const HWY_FULL(pixel_type) d;
40
4.62k
  const size_t N = Lanes(d);
41
106k
  for (; x + N - 1 < w; x += N) {
42
101k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
101k
    } else {
54
101k
      auto First = Load(d, in0 + x);
55
101k
      auto Second = Load(d, in1 + x);
56
101k
      auto Third = Load(d, in2 + x);
57
102k
      if (third) Third = Add(Third, First);
58
101k
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
101k
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
101k
      Store(First, d, out0 + x);
64
101k
      Store(Second, d, out1 + x);
65
101k
      Store(Third, d, out2 + x);
66
101k
    }
67
101k
  }
68
13.6k
  for (; x < w; x++) {
69
8.99k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
8.99k
    } else {
81
8.99k
      pixel_type First = in0[x];
82
8.99k
      pixel_type Second = in1[x];
83
8.99k
      pixel_type Third = in2[x];
84
8.99k
      if (third) Third = PixelAdd(Third, First);
85
8.99k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
8.99k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
8.99k
      out0[x] = First;
91
8.99k
      out1[x] = Second;
92
8.99k
      out2[x] = Third;
93
8.99k
    }
94
8.99k
  }
95
4.62k
}
void jxl::N_SSE2::InvRCTRow<2>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
5.98k
               pixel_type* out2, size_t w) {
33
5.98k
  static_assert(transform_type >= 0 && transform_type < 7,
34
5.98k
                "Invalid transform type");
35
5.98k
  int second = transform_type >> 1;
36
5.98k
  int third = transform_type & 1;
37
38
5.98k
  size_t x = 0;
39
5.98k
  const HWY_FULL(pixel_type) d;
40
5.98k
  const size_t N = Lanes(d);
41
200k
  for (; x + N - 1 < w; x += N) {
42
194k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
194k
    } else {
54
194k
      auto First = Load(d, in0 + x);
55
194k
      auto Second = Load(d, in1 + x);
56
194k
      auto Third = Load(d, in2 + x);
57
194k
      if (third) Third = Add(Third, First);
58
194k
      if (second == 1) {
59
194k
        Second = Add(Second, First);
60
18.4E
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
194k
      Store(First, d, out0 + x);
64
194k
      Store(Second, d, out1 + x);
65
194k
      Store(Third, d, out2 + x);
66
194k
    }
67
194k
  }
68
15.7k
  for (; x < w; x++) {
69
9.74k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
9.74k
    } else {
81
9.74k
      pixel_type First = in0[x];
82
9.74k
      pixel_type Second = in1[x];
83
9.74k
      pixel_type Third = in2[x];
84
9.74k
      if (third) Third = PixelAdd(Third, First);
85
9.74k
      if (second == 1) {
86
9.74k
        Second = PixelAdd(Second, First);
87
18.4E
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
9.74k
      out0[x] = First;
91
9.74k
      out1[x] = Second;
92
9.74k
      out2[x] = Third;
93
9.74k
    }
94
9.74k
  }
95
5.98k
}
void jxl::N_SSE2::InvRCTRow<3>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
2.33k
               pixel_type* out2, size_t w) {
33
2.33k
  static_assert(transform_type >= 0 && transform_type < 7,
34
2.33k
                "Invalid transform type");
35
2.33k
  int second = transform_type >> 1;
36
2.33k
  int third = transform_type & 1;
37
38
2.33k
  size_t x = 0;
39
2.33k
  const HWY_FULL(pixel_type) d;
40
2.33k
  const size_t N = Lanes(d);
41
53.6k
  for (; x + N - 1 < w; x += N) {
42
51.3k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
51.3k
    } else {
54
51.3k
      auto First = Load(d, in0 + x);
55
51.3k
      auto Second = Load(d, in1 + x);
56
51.3k
      auto Third = Load(d, in2 + x);
57
51.3k
      if (third) Third = Add(Third, First);
58
51.3k
      if (second == 1) {
59
51.3k
        Second = Add(Second, First);
60
51.3k
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
51.3k
      Store(First, d, out0 + x);
64
51.3k
      Store(Second, d, out1 + x);
65
51.3k
      Store(Third, d, out2 + x);
66
51.3k
    }
67
51.3k
  }
68
4.57k
  for (; x < w; x++) {
69
2.24k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
2.24k
    } else {
81
2.24k
      pixel_type First = in0[x];
82
2.24k
      pixel_type Second = in1[x];
83
2.24k
      pixel_type Third = in2[x];
84
2.24k
      if (third) Third = PixelAdd(Third, First);
85
2.24k
      if (second == 1) {
86
2.24k
        Second = PixelAdd(Second, First);
87
2.24k
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
2.24k
      out0[x] = First;
91
2.24k
      out1[x] = Second;
92
2.24k
      out2[x] = Third;
93
2.24k
    }
94
2.24k
  }
95
2.33k
}
void jxl::N_SSE2::InvRCTRow<4>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
26.3k
               pixel_type* out2, size_t w) {
33
26.3k
  static_assert(transform_type >= 0 && transform_type < 7,
34
26.3k
                "Invalid transform type");
35
26.3k
  int second = transform_type >> 1;
36
26.3k
  int third = transform_type & 1;
37
38
26.3k
  size_t x = 0;
39
26.3k
  const HWY_FULL(pixel_type) d;
40
26.3k
  const size_t N = Lanes(d);
41
577k
  for (; x + N - 1 < w; x += N) {
42
551k
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
551k
    } else {
54
551k
      auto First = Load(d, in0 + x);
55
551k
      auto Second = Load(d, in1 + x);
56
551k
      auto Third = Load(d, in2 + x);
57
551k
      if (third) Third = Add(Third, First);
58
551k
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
551k
      } else if (second == 2) {
61
551k
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
551k
      }
63
551k
      Store(First, d, out0 + x);
64
551k
      Store(Second, d, out1 + x);
65
551k
      Store(Third, d, out2 + x);
66
551k
    }
67
551k
  }
68
43.3k
  for (; x < w; x++) {
69
17.0k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
17.0k
    } else {
81
17.0k
      pixel_type First = in0[x];
82
17.0k
      pixel_type Second = in1[x];
83
17.0k
      pixel_type Third = in2[x];
84
17.0k
      if (third) Third = PixelAdd(Third, First);
85
17.0k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
17.0k
      } else if (second == 2) {
88
17.0k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
17.0k
      }
90
17.0k
      out0[x] = First;
91
17.0k
      out1[x] = Second;
92
17.0k
      out2[x] = Third;
93
17.0k
    }
94
17.0k
  }
95
26.3k
}
void jxl::N_SSE2::InvRCTRow<5>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
72.0k
               pixel_type* out2, size_t w) {
33
72.0k
  static_assert(transform_type >= 0 && transform_type < 7,
34
72.0k
                "Invalid transform type");
35
72.0k
  int second = transform_type >> 1;
36
72.0k
  int third = transform_type & 1;
37
38
72.0k
  size_t x = 0;
39
72.0k
  const HWY_FULL(pixel_type) d;
40
72.0k
  const size_t N = Lanes(d);
41
3.09M
  for (; x + N - 1 < w; x += N) {
42
3.02M
    if (transform_type == 6) {
43
0
      auto Y = Load(d, in0 + x);
44
0
      auto Co = Load(d, in1 + x);
45
0
      auto Cg = Load(d, in2 + x);
46
0
      Y = Sub(Y, ShiftRight<1>(Cg));
47
0
      auto G = Add(Cg, Y);
48
0
      Y = Sub(Y, ShiftRight<1>(Co));
49
0
      auto R = Add(Y, Co);
50
0
      Store(R, d, out0 + x);
51
0
      Store(G, d, out1 + x);
52
0
      Store(Y, d, out2 + x);
53
3.02M
    } else {
54
3.02M
      auto First = Load(d, in0 + x);
55
3.02M
      auto Second = Load(d, in1 + x);
56
3.02M
      auto Third = Load(d, in2 + x);
57
3.07M
      if (third) Third = Add(Third, First);
58
3.02M
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
3.06M
      } else if (second == 2) {
61
3.06M
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
3.06M
      }
63
3.02M
      Store(First, d, out0 + x);
64
3.02M
      Store(Second, d, out1 + x);
65
3.02M
      Store(Third, d, out2 + x);
66
3.02M
    }
67
3.02M
  }
68
75.3k
  for (; x < w; x++) {
69
3.32k
    if (transform_type == 6) {
70
0
      pixel_type Y = in0[x];
71
0
      pixel_type Co = in1[x];
72
0
      pixel_type Cg = in2[x];
73
0
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
0
      pixel_type G = PixelAdd(Cg, tmp);
75
0
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
0
      pixel_type R = PixelAdd(B, Co);
77
0
      out0[x] = R;
78
0
      out1[x] = G;
79
0
      out2[x] = B;
80
3.32k
    } else {
81
3.32k
      pixel_type First = in0[x];
82
3.32k
      pixel_type Second = in1[x];
83
3.32k
      pixel_type Third = in2[x];
84
3.32k
      if (third) Third = PixelAdd(Third, First);
85
3.32k
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
3.32k
      } else if (second == 2) {
88
3.32k
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
3.32k
      }
90
3.32k
      out0[x] = First;
91
3.32k
      out1[x] = Second;
92
3.32k
      out2[x] = Third;
93
3.32k
    }
94
3.32k
  }
95
72.0k
}
void jxl::N_SSE2::InvRCTRow<6>(int const*, int const*, int const*, int*, int*, int*, unsigned long)
Line
Count
Source
32
36.8k
               pixel_type* out2, size_t w) {
33
36.8k
  static_assert(transform_type >= 0 && transform_type < 7,
34
36.8k
                "Invalid transform type");
35
36.8k
  int second = transform_type >> 1;
36
36.8k
  int third = transform_type & 1;
37
38
36.8k
  size_t x = 0;
39
36.8k
  const HWY_FULL(pixel_type) d;
40
36.8k
  const size_t N = Lanes(d);
41
958k
  for (; x + N - 1 < w; x += N) {
42
921k
    if (transform_type == 6) {
43
921k
      auto Y = Load(d, in0 + x);
44
921k
      auto Co = Load(d, in1 + x);
45
921k
      auto Cg = Load(d, in2 + x);
46
921k
      Y = Sub(Y, ShiftRight<1>(Cg));
47
921k
      auto G = Add(Cg, Y);
48
921k
      Y = Sub(Y, ShiftRight<1>(Co));
49
921k
      auto R = Add(Y, Co);
50
921k
      Store(R, d, out0 + x);
51
921k
      Store(G, d, out1 + x);
52
921k
      Store(Y, d, out2 + x);
53
18.4E
    } else {
54
18.4E
      auto First = Load(d, in0 + x);
55
18.4E
      auto Second = Load(d, in1 + x);
56
18.4E
      auto Third = Load(d, in2 + x);
57
18.4E
      if (third) Third = Add(Third, First);
58
18.4E
      if (second == 1) {
59
0
        Second = Add(Second, First);
60
18.4E
      } else if (second == 2) {
61
0
        Second = Add(Second, ShiftRight<1>(Add(First, Third)));
62
0
      }
63
18.4E
      Store(First, d, out0 + x);
64
18.4E
      Store(Second, d, out1 + x);
65
18.4E
      Store(Third, d, out2 + x);
66
18.4E
    }
67
921k
  }
68
60.9k
  for (; x < w; x++) {
69
24.1k
    if (transform_type == 6) {
70
24.1k
      pixel_type Y = in0[x];
71
24.1k
      pixel_type Co = in1[x];
72
24.1k
      pixel_type Cg = in2[x];
73
24.1k
      pixel_type tmp = PixelAdd(Y, -(Cg >> 1));
74
24.1k
      pixel_type G = PixelAdd(Cg, tmp);
75
24.1k
      pixel_type B = PixelAdd(tmp, -(Co >> 1));
76
24.1k
      pixel_type R = PixelAdd(B, Co);
77
24.1k
      out0[x] = R;
78
24.1k
      out1[x] = G;
79
24.1k
      out2[x] = B;
80
24.1k
    } else {
81
0
      pixel_type First = in0[x];
82
0
      pixel_type Second = in1[x];
83
0
      pixel_type Third = in2[x];
84
0
      if (third) Third = PixelAdd(Third, First);
85
0
      if (second == 1) {
86
0
        Second = PixelAdd(Second, First);
87
0
      } else if (second == 2) {
88
0
        Second = PixelAdd(Second, (PixelAdd(First, Third) >> 1));
89
0
      }
90
0
      out0[x] = First;
91
0
      out1[x] = Second;
92
0
      out2[x] = Third;
93
0
    }
94
24.1k
  }
95
36.8k
}
96
97
17.3k
Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) {
98
17.3k
  JXL_RETURN_IF_ERROR(CheckEqualChannels(input, begin_c, begin_c + 2));
99
17.3k
  size_t m = begin_c;
100
17.3k
  Channel& c0 = input.channel[m + 0];
101
17.3k
  size_t w = c0.w;
102
17.3k
  size_t h = c0.h;
103
17.3k
  if (rct_type == 0) {  // noop
104
4.82k
    return true;
105
4.82k
  }
106
  // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR
107
12.4k
  int permutation = rct_type / 7;
108
12.4k
  JXL_ENSURE(permutation < 6);
109
  // 0-5 values have the low bit corresponding to Third and the high bits
110
  // corresponding to Second. 6 corresponds to YCoCg.
111
  //
112
  // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird
113
  //
114
  // Third: 0=nop, 1=SubtractFirst
115
12.4k
  int custom = rct_type % 7;
116
  // Special case: permute-only. Swap channels around.
117
12.4k
  if (custom == 0) {
118
363
    Channel ch0 = std::move(input.channel[m]);
119
363
    Channel ch1 = std::move(input.channel[m + 1]);
120
363
    Channel ch2 = std::move(input.channel[m + 2]);
121
363
    input.channel[m + (permutation % 3)] = std::move(ch0);
122
363
    input.channel[m + ((permutation + 1 + permutation / 3) % 3)] =
123
363
        std::move(ch1);
124
363
    input.channel[m + ((permutation + 2 - permutation / 3) % 3)] =
125
363
        std::move(ch2);
126
363
    return true;
127
363
  }
128
12.1k
  constexpr decltype(&InvRCTRow<0>) inv_rct_row[] = {
129
12.1k
      InvRCTRow<0>, InvRCTRow<1>, InvRCTRow<2>, InvRCTRow<3>,
130
12.1k
      InvRCTRow<4>, InvRCTRow<5>, InvRCTRow<6>};
131
12.1k
  const auto process_row = [&](const uint32_t task,
132
6.88M
                               size_t /* thread */) -> Status {
133
6.88M
    const size_t y = task;
134
6.88M
    const pixel_type* in0 = input.channel[m].Row(y);
135
6.88M
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
6.88M
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
6.88M
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
6.88M
    pixel_type* out1 =
139
6.88M
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
6.88M
    pixel_type* out2 =
141
6.88M
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
6.88M
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
6.88M
    return true;
144
6.88M
  };
rct.cc:jxl::N_SSE4::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const
Line
Count
Source
132
145k
                               size_t /* thread */) -> Status {
133
145k
    const size_t y = task;
134
145k
    const pixel_type* in0 = input.channel[m].Row(y);
135
145k
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
145k
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
145k
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
145k
    pixel_type* out1 =
139
145k
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
145k
    pixel_type* out2 =
141
145k
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
145k
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
145k
    return true;
144
145k
  };
rct.cc:jxl::N_AVX2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const
Line
Count
Source
132
6.59M
                               size_t /* thread */) -> Status {
133
6.59M
    const size_t y = task;
134
6.59M
    const pixel_type* in0 = input.channel[m].Row(y);
135
6.59M
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
6.59M
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
6.59M
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
6.59M
    pixel_type* out1 =
139
6.59M
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
6.59M
    pixel_type* out2 =
141
6.59M
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
6.59M
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
6.59M
    return true;
144
6.59M
  };
rct.cc:jxl::N_SSE2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)::$_0::operator()(unsigned int, unsigned long) const
Line
Count
Source
132
149k
                               size_t /* thread */) -> Status {
133
149k
    const size_t y = task;
134
149k
    const pixel_type* in0 = input.channel[m].Row(y);
135
149k
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
149k
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
149k
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
149k
    pixel_type* out1 =
139
149k
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
149k
    pixel_type* out2 =
141
149k
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
149k
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
149k
    return true;
144
149k
  };
145
12.1k
  JXL_RETURN_IF_ERROR(
146
12.1k
      RunOnPool(pool, 0, h, ThreadPool::NoInit, process_row, "InvRCT"));
147
12.1k
  return true;
148
12.1k
}
jxl::N_SSE4::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)
Line
Count
Source
97
2.49k
Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) {
98
2.49k
  JXL_RETURN_IF_ERROR(CheckEqualChannels(input, begin_c, begin_c + 2));
99
2.49k
  size_t m = begin_c;
100
2.49k
  Channel& c0 = input.channel[m + 0];
101
2.49k
  size_t w = c0.w;
102
2.49k
  size_t h = c0.h;
103
2.49k
  if (rct_type == 0) {  // noop
104
243
    return true;
105
243
  }
106
  // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR
107
2.25k
  int permutation = rct_type / 7;
108
2.25k
  JXL_ENSURE(permutation < 6);
109
  // 0-5 values have the low bit corresponding to Third and the high bits
110
  // corresponding to Second. 6 corresponds to YCoCg.
111
  //
112
  // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird
113
  //
114
  // Third: 0=nop, 1=SubtractFirst
115
2.25k
  int custom = rct_type % 7;
116
  // Special case: permute-only. Swap channels around.
117
2.25k
  if (custom == 0) {
118
75
    Channel ch0 = std::move(input.channel[m]);
119
75
    Channel ch1 = std::move(input.channel[m + 1]);
120
75
    Channel ch2 = std::move(input.channel[m + 2]);
121
75
    input.channel[m + (permutation % 3)] = std::move(ch0);
122
75
    input.channel[m + ((permutation + 1 + permutation / 3) % 3)] =
123
75
        std::move(ch1);
124
75
    input.channel[m + ((permutation + 2 - permutation / 3) % 3)] =
125
75
        std::move(ch2);
126
75
    return true;
127
75
  }
128
2.17k
  constexpr decltype(&InvRCTRow<0>) inv_rct_row[] = {
129
2.17k
      InvRCTRow<0>, InvRCTRow<1>, InvRCTRow<2>, InvRCTRow<3>,
130
2.17k
      InvRCTRow<4>, InvRCTRow<5>, InvRCTRow<6>};
131
2.17k
  const auto process_row = [&](const uint32_t task,
132
2.17k
                               size_t /* thread */) -> Status {
133
2.17k
    const size_t y = task;
134
2.17k
    const pixel_type* in0 = input.channel[m].Row(y);
135
2.17k
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
2.17k
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
2.17k
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
2.17k
    pixel_type* out1 =
139
2.17k
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
2.17k
    pixel_type* out2 =
141
2.17k
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
2.17k
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
2.17k
    return true;
144
2.17k
  };
145
2.17k
  JXL_RETURN_IF_ERROR(
146
2.17k
      RunOnPool(pool, 0, h, ThreadPool::NoInit, process_row, "InvRCT"));
147
2.17k
  return true;
148
2.17k
}
jxl::N_AVX2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)
Line
Count
Source
97
11.9k
Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) {
98
11.9k
  JXL_RETURN_IF_ERROR(CheckEqualChannels(input, begin_c, begin_c + 2));
99
11.9k
  size_t m = begin_c;
100
11.9k
  Channel& c0 = input.channel[m + 0];
101
11.9k
  size_t w = c0.w;
102
11.9k
  size_t h = c0.h;
103
11.9k
  if (rct_type == 0) {  // noop
104
4.37k
    return true;
105
4.37k
  }
106
  // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR
107
7.57k
  int permutation = rct_type / 7;
108
7.57k
  JXL_ENSURE(permutation < 6);
109
  // 0-5 values have the low bit corresponding to Third and the high bits
110
  // corresponding to Second. 6 corresponds to YCoCg.
111
  //
112
  // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird
113
  //
114
  // Third: 0=nop, 1=SubtractFirst
115
7.57k
  int custom = rct_type % 7;
116
  // Special case: permute-only. Swap channels around.
117
7.57k
  if (custom == 0) {
118
246
    Channel ch0 = std::move(input.channel[m]);
119
246
    Channel ch1 = std::move(input.channel[m + 1]);
120
246
    Channel ch2 = std::move(input.channel[m + 2]);
121
246
    input.channel[m + (permutation % 3)] = std::move(ch0);
122
246
    input.channel[m + ((permutation + 1 + permutation / 3) % 3)] =
123
246
        std::move(ch1);
124
246
    input.channel[m + ((permutation + 2 - permutation / 3) % 3)] =
125
246
        std::move(ch2);
126
246
    return true;
127
246
  }
128
7.32k
  constexpr decltype(&InvRCTRow<0>) inv_rct_row[] = {
129
7.32k
      InvRCTRow<0>, InvRCTRow<1>, InvRCTRow<2>, InvRCTRow<3>,
130
7.32k
      InvRCTRow<4>, InvRCTRow<5>, InvRCTRow<6>};
131
7.32k
  const auto process_row = [&](const uint32_t task,
132
7.32k
                               size_t /* thread */) -> Status {
133
7.32k
    const size_t y = task;
134
7.32k
    const pixel_type* in0 = input.channel[m].Row(y);
135
7.32k
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
7.32k
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
7.32k
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
7.32k
    pixel_type* out1 =
139
7.32k
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
7.32k
    pixel_type* out2 =
141
7.32k
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
7.32k
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
7.32k
    return true;
144
7.32k
  };
145
7.32k
  JXL_RETURN_IF_ERROR(
146
7.32k
      RunOnPool(pool, 0, h, ThreadPool::NoInit, process_row, "InvRCT"));
147
7.32k
  return true;
148
7.32k
}
jxl::N_SSE2::InvRCT(jxl::Image&, unsigned long, unsigned long, jxl::ThreadPool*)
Line
Count
Source
97
2.86k
Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) {
98
2.86k
  JXL_RETURN_IF_ERROR(CheckEqualChannels(input, begin_c, begin_c + 2));
99
2.86k
  size_t m = begin_c;
100
2.86k
  Channel& c0 = input.channel[m + 0];
101
2.86k
  size_t w = c0.w;
102
2.86k
  size_t h = c0.h;
103
2.86k
  if (rct_type == 0) {  // noop
104
206
    return true;
105
206
  }
106
  // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR
107
2.65k
  int permutation = rct_type / 7;
108
2.65k
  JXL_ENSURE(permutation < 6);
109
  // 0-5 values have the low bit corresponding to Third and the high bits
110
  // corresponding to Second. 6 corresponds to YCoCg.
111
  //
112
  // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird
113
  //
114
  // Third: 0=nop, 1=SubtractFirst
115
2.65k
  int custom = rct_type % 7;
116
  // Special case: permute-only. Swap channels around.
117
2.65k
  if (custom == 0) {
118
42
    Channel ch0 = std::move(input.channel[m]);
119
42
    Channel ch1 = std::move(input.channel[m + 1]);
120
42
    Channel ch2 = std::move(input.channel[m + 2]);
121
42
    input.channel[m + (permutation % 3)] = std::move(ch0);
122
42
    input.channel[m + ((permutation + 1 + permutation / 3) % 3)] =
123
42
        std::move(ch1);
124
42
    input.channel[m + ((permutation + 2 - permutation / 3) % 3)] =
125
42
        std::move(ch2);
126
42
    return true;
127
42
  }
128
2.61k
  constexpr decltype(&InvRCTRow<0>) inv_rct_row[] = {
129
2.61k
      InvRCTRow<0>, InvRCTRow<1>, InvRCTRow<2>, InvRCTRow<3>,
130
2.61k
      InvRCTRow<4>, InvRCTRow<5>, InvRCTRow<6>};
131
2.61k
  const auto process_row = [&](const uint32_t task,
132
2.61k
                               size_t /* thread */) -> Status {
133
2.61k
    const size_t y = task;
134
2.61k
    const pixel_type* in0 = input.channel[m].Row(y);
135
2.61k
    const pixel_type* in1 = input.channel[m + 1].Row(y);
136
2.61k
    const pixel_type* in2 = input.channel[m + 2].Row(y);
137
2.61k
    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
138
2.61k
    pixel_type* out1 =
139
2.61k
        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
140
2.61k
    pixel_type* out2 =
141
2.61k
        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
142
2.61k
    inv_rct_row[custom](in0, in1, in2, out0, out1, out2, w);
143
2.61k
    return true;
144
2.61k
  };
145
2.61k
  JXL_RETURN_IF_ERROR(
146
2.61k
      RunOnPool(pool, 0, h, ThreadPool::NoInit, process_row, "InvRCT"));
147
2.61k
  return true;
148
2.61k
}
149
150
}  // namespace HWY_NAMESPACE
151
}  // namespace jxl
152
HWY_AFTER_NAMESPACE();
153
154
#if HWY_ONCE
155
namespace jxl {
156
157
HWY_EXPORT(InvRCT);
158
17.3k
Status InvRCT(Image& input, size_t begin_c, size_t rct_type, ThreadPool* pool) {
159
17.3k
  return HWY_DYNAMIC_DISPATCH(InvRCT)(input, begin_c, rct_type, pool);
160
17.3k
}
161
162
}  // namespace jxl
163
#endif