Coverage Report

Created: 2026-05-30 06:06

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/astc-encoder/Source/astcenc_internal.h
Line
Count
Source
1
// SPDX-License-Identifier: Apache-2.0
2
// ----------------------------------------------------------------------------
3
// Copyright 2011-2026 Arm Limited
4
//
5
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
6
// use this file except in compliance with the License. You may obtain a copy
7
// of the License at:
8
//
9
//     http://www.apache.org/licenses/LICENSE-2.0
10
//
11
// Unless required by applicable law or agreed to in writing, software
12
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14
// License for the specific language governing permissions and limitations
15
// under the License.
16
// ----------------------------------------------------------------------------
17
18
/**
19
 * @brief Functions and data declarations.
20
 */
21
22
#ifndef ASTCENC_INTERNAL_INCLUDED
23
#define ASTCENC_INTERNAL_INCLUDED
24
25
#include <algorithm>
26
#include <cstddef>
27
#include <cstdint>
28
#if defined(ASTCENC_DIAGNOSTICS)
29
  #include <cstdio>
30
#endif
31
#include <cstdlib>
32
#include <limits>
33
#include <type_traits>
34
35
#include "astcenc.h"
36
#include "astcenc_mathlib.h"
37
#include "astcenc_vecmathlib.h"
38
39
/**
40
 * @brief Make a promise to the compiler's optimizer.
41
 *
42
 * A promise is an expression that the optimizer is can assume is true for to help it generate
43
 * faster code. Common use cases for this are to promise that a for loop will iterate more than
44
 * once, or that the loop iteration count is a multiple of a vector length, which avoids pre-loop
45
 * checks and can avoid loop tails if loops are unrolled by the auto-vectorizer.
46
 */
47
#if defined(NDEBUG)
48
  #if !defined(__clang__) && defined(_MSC_VER)
49
    #define promise(cond) __assume(cond)
50
  #elif defined(__clang__)
51
    #if __has_builtin(__builtin_assume)
52
      #define promise(cond) __builtin_assume(cond)
53
    #elif __has_builtin(__builtin_unreachable)
54
      #define promise(cond) if (!(cond)) { __builtin_unreachable(); }
55
    #else
56
      #define promise(cond)
57
    #endif
58
  #else // Assume GCC
59
    #define promise(cond) if (!(cond)) { __builtin_unreachable(); }
60
  #endif
61
#else
62
263k
  #define promise(cond) assert(cond)
63
#endif
64
65
/* ============================================================================
66
  Constants
67
============================================================================ */
68
#if !defined(ASTCENC_BLOCK_MAX_TEXELS)
69
3.23G
  #define ASTCENC_BLOCK_MAX_TEXELS 216 // A 3D 6x6x6 block
70
#endif
71
72
/** @brief The maximum number of texels a block can support (6x6x6 block). */
73
static constexpr unsigned int BLOCK_MAX_TEXELS { ASTCENC_BLOCK_MAX_TEXELS };
74
75
/** @brief The maximum number of components a block can support. */
76
static constexpr unsigned int BLOCK_MAX_COMPONENTS { 4 };
77
78
/** @brief The maximum number of partitions a block can support. */
79
static constexpr unsigned int BLOCK_MAX_PARTITIONS { 4 };
80
81
/** @brief The number of partitionings, per partition count, suported by the ASTC format. */
82
static constexpr unsigned int BLOCK_MAX_PARTITIONINGS { 1024 };
83
84
/** @brief The maximum number of texels used during partition selection for texel clustering. */
85
static constexpr uint8_t BLOCK_MAX_KMEANS_TEXELS { 64 };
86
87
/** @brief The maximum number of weights a block can support. */
88
static constexpr unsigned int BLOCK_MAX_WEIGHTS { 64 };
89
90
/** @brief The maximum number of weights a block can support per plane in 2 plane mode. */
91
static constexpr unsigned int BLOCK_MAX_WEIGHTS_2PLANE { BLOCK_MAX_WEIGHTS / 2 };
92
93
/** @brief The minimum number of weight bits a candidate encoding must encode. */
94
static constexpr unsigned int BLOCK_MIN_WEIGHT_BITS { 24 };
95
96
/** @brief The maximum number of weight bits a candidate encoding can encode. */
97
static constexpr unsigned int BLOCK_MAX_WEIGHT_BITS { 96 };
98
99
/** @brief The index indicating a bad (unused) block mode in the remap array. */
100
static constexpr uint16_t BLOCK_BAD_BLOCK_MODE { 0xFFFFu };
101
102
/** @brief The index indicating a bad (unused) partitioning in the remap array. */
103
static constexpr uint16_t BLOCK_BAD_PARTITIONING { 0xFFFFu };
104
105
/** @brief The number of partition index bits supported by the ASTC format . */
106
static constexpr unsigned int PARTITION_INDEX_BITS { 10 };
107
108
/** @brief The offset of the plane 2 weights in shared weight arrays. */
109
static constexpr unsigned int WEIGHTS_PLANE2_OFFSET { BLOCK_MAX_WEIGHTS_2PLANE };
110
111
/** @brief The sum of quantized weights for one texel. */
112
static constexpr float WEIGHTS_TEXEL_SUM { 16.0f };
113
114
/** @brief The number of block modes supported by the ASTC format. */
115
static constexpr unsigned int WEIGHTS_MAX_BLOCK_MODES { 2048 };
116
117
/** @brief The number of weight grid decimation modes supported by the ASTC format. */
118
static constexpr unsigned int WEIGHTS_MAX_DECIMATION_MODES { 87 };
119
120
/** @brief The high default error used to initialize error trackers. */
121
static constexpr float ERROR_CALC_DEFAULT { 1e30f };
122
123
/**
124
 * @brief The minimum tuning setting threshold for the one partition fast path.
125
 */
126
static constexpr float TUNE_MIN_SEARCH_MODE0 { 0.85f };
127
128
/**
129
 * @brief The maximum number of candidate encodings tested for each encoding mode.
130
 *
131
 * This can be dynamically reduced by the compression quality preset.
132
 */
133
static constexpr unsigned int TUNE_MAX_TRIAL_CANDIDATES { 8 };
134
135
/**
136
 * @brief The maximum number of candidate partitionings tested for each encoding mode.
137
 *
138
 * This can be dynamically reduced by the compression quality preset.
139
 */
140
static constexpr unsigned int TUNE_MAX_PARTITIONING_CANDIDATES { 8 };
141
142
/**
143
 * @brief The maximum quant level using full angular endpoint search method.
144
 *
145
 * The angular endpoint search is used to find the min/max weight that should
146
 * be used for a given quantization level. It is effective but expensive, so
147
 * we only use it where it has the most value - low quant levels with wide
148
 * spacing. It is used below TUNE_MAX_ANGULAR_QUANT (inclusive). Above this we
149
 * assume the min weight is 0.0f, and the max weight is 1.0f.
150
 *
151
 * Note the angular algorithm is vectorized, and using QUANT_12 exactly fills
152
 * one 8-wide vector. Decreasing by one doesn't buy much performance, and
153
 * increasing by one is disproportionately expensive.
154
 */
155
static constexpr unsigned int TUNE_MAX_ANGULAR_QUANT { 7 }; /* QUANT_12 */
156
157
static_assert((BLOCK_MAX_TEXELS % ASTCENC_SIMD_WIDTH) == 0,
158
              "BLOCK_MAX_TEXELS must be multiple of ASTCENC_SIMD_WIDTH");
159
160
static_assert(BLOCK_MAX_TEXELS <= 216,
161
              "BLOCK_MAX_TEXELS must not be greater than 216");
162
163
static_assert((BLOCK_MAX_WEIGHTS % ASTCENC_SIMD_WIDTH) == 0,
164
              "BLOCK_MAX_WEIGHTS must be multiple of ASTCENC_SIMD_WIDTH");
165
166
static_assert((WEIGHTS_MAX_BLOCK_MODES % ASTCENC_SIMD_WIDTH) == 0,
167
              "WEIGHTS_MAX_BLOCK_MODES must be multiple of ASTCENC_SIMD_WIDTH");
168
169
170
/* ============================================================================
171
  Commonly used data structures
172
============================================================================ */
173
174
/**
175
 * @brief The ASTC endpoint formats.
176
 *
177
 * Note, the values here are used directly in the encoding in the format so do not rearrange.
178
 */
179
enum endpoint_formats
180
{
181
  FMT_LUMINANCE = 0,
182
  FMT_LUMINANCE_DELTA = 1,
183
  FMT_HDR_LUMINANCE_LARGE_RANGE = 2,
184
  FMT_HDR_LUMINANCE_SMALL_RANGE = 3,
185
  FMT_LUMINANCE_ALPHA = 4,
186
  FMT_LUMINANCE_ALPHA_DELTA = 5,
187
  FMT_RGB_SCALE = 6,
188
  FMT_HDR_RGB_SCALE = 7,
189
  FMT_RGB = 8,
190
  FMT_RGB_DELTA = 9,
191
  FMT_RGB_SCALE_ALPHA = 10,
192
  FMT_HDR_RGB = 11,
193
  FMT_RGBA = 12,
194
  FMT_RGBA_DELTA = 13,
195
  FMT_HDR_RGB_LDR_ALPHA = 14,
196
  FMT_HDR_RGBA = 15
197
};
198
199
/**
200
 * @brief The ASTC quantization methods.
201
 *
202
 * Note, the values here are used directly in the encoding in the format so do not rearrange.
203
 */
204
enum quant_method
205
{
206
  QUANT_2 = 0,
207
  QUANT_3 = 1,
208
  QUANT_4 = 2,
209
  QUANT_5 = 3,
210
  QUANT_6 = 4,
211
  QUANT_8 = 5,
212
  QUANT_10 = 6,
213
  QUANT_12 = 7,
214
  QUANT_16 = 8,
215
  QUANT_20 = 9,
216
  QUANT_24 = 10,
217
  QUANT_32 = 11,
218
  QUANT_40 = 12,
219
  QUANT_48 = 13,
220
  QUANT_64 = 14,
221
  QUANT_80 = 15,
222
  QUANT_96 = 16,
223
  QUANT_128 = 17,
224
  QUANT_160 = 18,
225
  QUANT_192 = 19,
226
  QUANT_256 = 20
227
};
228
229
/**
230
 * @brief The number of levels use by an ASTC quantization method.
231
 *
232
 * @param method   The quantization method
233
 *
234
 * @return   The number of levels used by @c method.
235
 */
236
static inline unsigned int get_quant_level(quant_method method)
237
0
{
238
0
  switch (method)
239
0
  {
240
0
  case QUANT_2:   return   2;
241
0
  case QUANT_3:   return   3;
242
0
  case QUANT_4:   return   4;
243
0
  case QUANT_5:   return   5;
244
0
  case QUANT_6:   return   6;
245
0
  case QUANT_8:   return   8;
246
0
  case QUANT_10:  return  10;
247
0
  case QUANT_12:  return  12;
248
0
  case QUANT_16:  return  16;
249
0
  case QUANT_20:  return  20;
250
0
  case QUANT_24:  return  24;
251
0
  case QUANT_32:  return  32;
252
0
  case QUANT_40:  return  40;
253
0
  case QUANT_48:  return  48;
254
0
  case QUANT_64:  return  64;
255
0
  case QUANT_80:  return  80;
256
0
  case QUANT_96:  return  96;
257
0
  case QUANT_128: return 128;
258
0
  case QUANT_160: return 160;
259
0
  case QUANT_192: return 192;
260
0
  case QUANT_256: return 256;
261
0
  }
262
263
  // Unreachable - the enum is fully described
264
0
  return 0;
265
0
}
Unexecuted instantiation: astcenc_entry.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_image.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_percentile_tables.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_symbolic_physical.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_weight_align.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_weight_quant_xfer_tables.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_block_sizes.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_color_unquantize.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_compress_symbolic.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_compute_variance.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_decompress_symbolic.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_find_best_partitioning.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_ideal_endpoints_and_weights.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_integer_sequence.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_partition_tables.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_pick_best_endpoint_format.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_quantization.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_averages_and_directions.cpp:get_quant_level(quant_method)
Unexecuted instantiation: astcenc_color_quantize.cpp:get_quant_level(quant_method)
266
267
/**
268
 * @brief Computed metrics about a partition in a block.
269
 */
270
struct partition_metrics
271
{
272
  /** @brief The error-weighted average color in the partition. */
273
  vfloat4 avg;
274
275
  /** @brief The dominant error-weighted direction in the partition. */
276
  vfloat4 dir;
277
};
278
279
/**
280
 * @brief Computed lines for a a three component analysis.
281
 */
282
struct partition_lines3
283
{
284
  /** @brief Line for uncorrelated chroma. */
285
  line3 uncor_line;
286
287
  /** @brief Line for correlated chroma, passing though the origin. */
288
  line3 samec_line;
289
290
  /** @brief Post-processed line for uncorrelated chroma. */
291
  processed_line3 uncor_pline;
292
293
  /** @brief Post-processed line for correlated chroma, passing though the origin. */
294
  processed_line3 samec_pline;
295
296
  /**
297
   * @brief The length of the line for uncorrelated chroma.
298
   *
299
   * This is used for both the uncorrelated and same chroma lines - they are normally very similar
300
   * and only used for the relative ranking of partitionings against one another.
301
   */
302
  float line_length;
303
};
304
305
/**
306
 * @brief The partition information for a single partition.
307
 *
308
 * ASTC has a total of 1024 candidate partitions for each of 2/3/4 partition counts, although this
309
 * 1024 includes seeds that generate duplicates of other seeds and seeds that generate completely
310
 * empty partitions. These are both valid encodings, but astcenc will skip both during compression
311
 * as they are not useful.
312
 */
313
struct partition_info
314
{
315
  /** @brief The number of partitions in this partitioning. */
316
  uint16_t partition_count;
317
318
  /** @brief The index (seed) of this partitioning. */
319
  uint16_t partition_index;
320
321
  /**
322
   * @brief The number of texels in each partition.
323
   *
324
   * Note that some seeds result in zero texels assigned to a partition. These are valid, but are
325
   * skipped by this compressor as there is no point spending bits encoding an unused endpoints.
326
   */
327
  uint8_t partition_texel_count[BLOCK_MAX_PARTITIONS];
328
329
  /** @brief The partition of each texel in the block. */
330
  ASTCENC_ALIGNAS uint8_t partition_of_texel[BLOCK_MAX_TEXELS];
331
332
  /** @brief The list of texels in each partition. */
333
  ASTCENC_ALIGNAS uint8_t texels_of_partition[BLOCK_MAX_PARTITIONS][BLOCK_MAX_TEXELS];
334
};
335
336
/**
337
 * @brief The weight grid information for a single decimation pattern.
338
 *
339
 * ASTC can store one weight per texel, but is also capable of storing lower resolution weight grids
340
 * that are interpolated during decompression to assign a with to a texel. Storing fewer weights
341
 * can free up a substantial amount of bits that we can then spend on more useful things, such as
342
 * more accurate endpoints and weights, or additional partitions.
343
 *
344
 * This data structure is used to store information about a single weight grid decimation pattern,
345
 * for a single block size.
346
 */
347
struct decimation_info
348
{
349
  /** @brief The total number of texels in the block. */
350
  uint8_t texel_count;
351
352
  /** @brief The maximum number of stored weights that contribute to each texel, between 1 and 4. */
353
  uint8_t max_texel_weight_count;
354
355
  /** @brief The total number of weights stored. */
356
  uint8_t weight_count;
357
358
  /** @brief The number of stored weights in the X dimension. */
359
  uint8_t weight_x;
360
361
  /** @brief The number of stored weights in the Y dimension. */
362
  uint8_t weight_y;
363
364
  /** @brief The number of stored weights in the Z dimension. */
365
  uint8_t weight_z;
366
367
  /**
368
   * @brief The number of weights that contribute to each texel.
369
   * Value is between 1 and 4.
370
   */
371
  ASTCENC_ALIGNAS uint8_t texel_weight_count[BLOCK_MAX_TEXELS];
372
373
  /**
374
   * @brief The weight index of the N weights that are interpolated for each texel.
375
   * Stored transposed to improve vectorization.
376
   */
377
  ASTCENC_ALIGNAS uint8_t texel_weights_tr[4][BLOCK_MAX_TEXELS];
378
379
  /**
380
   * @brief The bilinear contribution of the N weights that are interpolated for each texel.
381
   * Value is between 0 and 16, stored transposed to improve vectorization.
382
   */
383
  ASTCENC_ALIGNAS uint8_t texel_weight_contribs_int_tr[4][BLOCK_MAX_TEXELS];
384
385
#if !defined(ASTCENC_DECOMPRESS_ONLY)
386
  /**
387
   * @brief The bilinear contribution of the N weights that are interpolated for each texel.
388
   * Value is between 0 and 1, stored transposed to improve vectorization.
389
   */
390
  ASTCENC_ALIGNAS float texel_weight_contribs_float_tr[4][BLOCK_MAX_TEXELS];
391
392
  /** @brief The number of texels that each stored weight contributes to. */
393
  ASTCENC_ALIGNAS uint8_t weight_texel_count[BLOCK_MAX_WEIGHTS];
394
395
  /**
396
   * @brief The list of texels that use a specific weight index.
397
   * Stored transposed to improve vectorization.
398
   */
399
  ASTCENC_ALIGNAS uint8_t weight_texels_tr[BLOCK_MAX_TEXELS][BLOCK_MAX_WEIGHTS];
400
401
  /**
402
   * @brief The bilinear contribution to the N texels that use each weight.
403
   * Value is between 0 and 1, stored transposed to improve vectorization.
404
   */
405
  ASTCENC_ALIGNAS float weights_texel_contribs_tr[BLOCK_MAX_TEXELS][BLOCK_MAX_WEIGHTS];
406
407
  /**
408
   * @brief The bilinear contribution to the Nth texel that uses each weight.
409
   * Value is between 0 and 1, stored transposed to improve vectorization.
410
   */
411
  float texel_contrib_for_weight[BLOCK_MAX_TEXELS][BLOCK_MAX_WEIGHTS];
412
#endif
413
};
414
415
/**
416
 * @brief Metadata for single block mode for a specific block size.
417
 */
418
struct block_mode
419
{
420
  /** @brief The block mode index in the ASTC encoded form. */
421
  uint16_t mode_index;
422
423
  /** @brief The decimation mode index in the compressor reindexed list. */
424
  uint8_t decimation_mode;
425
426
  /** @brief The weight quantization used by this block mode. */
427
  uint8_t quant_mode;
428
429
  /** @brief The weight quantization used by this block mode. */
430
  uint8_t weight_bits;
431
432
  /** @brief Is a dual weight plane used by this block mode? */
433
  uint8_t is_dual_plane : 1;
434
435
  /**
436
   * @brief Get the weight quantization used by this block mode.
437
   *
438
   * @return The quantization level.
439
   */
440
  inline quant_method get_weight_quant_mode() const
441
545k
  {
442
545k
    return static_cast<quant_method>(this->quant_mode);
443
545k
  }
444
};
445
446
/**
447
 * @brief Metadata for single decimation mode for a specific block size.
448
 */
449
struct decimation_mode
450
{
451
  /** @brief The max weight precision for 1 plane, or -1 if not supported. */
452
  int8_t maxprec_1plane;
453
454
  /** @brief The max weight precision for 2 planes, or -1 if not supported. */
455
  int8_t maxprec_2planes;
456
457
  /**
458
   * @brief Bitvector indicating weight quant modes used by active 1 plane block modes.
459
   *
460
   * Bit 0 = QUANT_2, Bit 1 = QUANT_3, etc.
461
   */
462
  uint16_t refprec_1plane;
463
464
  /**
465
   * @brief Bitvector indicating weight quant methods used by active 2 plane block modes.
466
   *
467
   * Bit 0 = QUANT_2, Bit 1 = QUANT_3, etc.
468
   */
469
  uint16_t refprec_2planes;
470
471
  /**
472
   * @brief Set a 1 plane weight quant as active.
473
   *
474
   * @param weight_quant   The quant method to set.
475
   */
476
  void set_ref_1plane(quant_method weight_quant)
477
335k
  {
478
335k
    refprec_1plane |= (1 << weight_quant);
479
335k
  }
480
481
  /**
482
   * @brief Test if this mode is active below a given 1 plane weight quant (inclusive).
483
   *
484
   * @param max_weight_quant   The max quant method to test.
485
   */
486
  bool is_ref_1plane(quant_method max_weight_quant) const
487
0
  {
488
0
    uint16_t mask = static_cast<uint16_t>((1 << (max_weight_quant + 1)) - 1);
489
0
    return (refprec_1plane & mask) != 0;
490
0
  }
491
492
  /**
493
   * @brief Set a 2 plane weight quant as active.
494
   *
495
   * @param weight_quant   The quant method to set.
496
   */
497
  void set_ref_2plane(quant_method weight_quant)
498
210k
  {
499
210k
    refprec_2planes |= static_cast<uint16_t>(1 << weight_quant);
500
210k
  }
501
502
  /**
503
   * @brief Test if this mode is active below a given 2 plane weight quant (inclusive).
504
   *
505
   * @param max_weight_quant   The max quant method to test.
506
   */
507
  bool is_ref_2plane(quant_method max_weight_quant) const
508
0
  {
509
0
    uint16_t mask = static_cast<uint16_t>((1 << (max_weight_quant + 1)) - 1);
510
0
    return (refprec_2planes & mask) != 0;
511
0
  }
512
};
513
514
/**
515
 * @brief Data tables for a single block size.
516
 *
517
 * The decimation tables store the information to apply weight grid dimension reductions. We only
518
 * store the decimation modes that are actually needed by the current context; many of the possible
519
 * modes will be unused (too many weights for the current block size or disabled by heuristics). The
520
 * actual number of weights stored is @c decimation_mode_count, and the @c decimation_modes and
521
 * @c decimation_tables arrays store the active modes contiguously at the start of the array. These
522
 * entries are not stored in any particular order.
523
 *
524
 * The block mode tables store the unpacked block mode settings. Block modes are stored in the
525
 * compressed block as an 11 bit field, but for any given block size and set of compressor
526
 * heuristics, only a subset of the block modes will be used. The actual number of block modes
527
 * stored is indicated in @c block_mode_count, and the @c block_modes array store the active modes
528
 * contiguously at the start of the array. These entries are stored in incrementing "packed" value
529
 * order, which doesn't mean much once unpacked. To allow decompressors to reference the packed data
530
 * efficiently the @c block_mode_packed_index array stores the mapping between physical ID and the
531
 * actual remapped array index.
532
 */
533
struct block_size_descriptor
534
{
535
  /** @brief The block X dimension, in texels. */
536
  uint8_t xdim;
537
538
  /** @brief The block Y dimension, in texels. */
539
  uint8_t ydim;
540
541
  /** @brief The block Z dimension, in texels. */
542
  uint8_t zdim;
543
544
  /** @brief The block total texel count. */
545
  uint8_t texel_count;
546
547
  /**
548
   * @brief The number of stored decimation modes which are "always" modes.
549
   *
550
   * Always modes are stored at the start of the decimation_modes list.
551
   */
552
  unsigned int decimation_mode_count_always;
553
554
  /** @brief The number of stored decimation modes for selected encodings. */
555
  unsigned int decimation_mode_count_selected;
556
557
  /** @brief The number of stored decimation modes for any encoding. */
558
  unsigned int decimation_mode_count_all;
559
560
  /**
561
   * @brief The number of stored block modes which are "always" modes.
562
   *
563
   * Always modes are stored at the start of the block_modes list.
564
   */
565
  unsigned int block_mode_count_1plane_always;
566
567
  /** @brief The number of stored block modes for active 1 plane encodings. */
568
  unsigned int block_mode_count_1plane_selected;
569
570
  /** @brief The number of stored block modes for active 1 and 2 plane encodings. */
571
  unsigned int block_mode_count_1plane_2plane_selected;
572
573
  /** @brief The number of stored block modes for any encoding. */
574
  unsigned int block_mode_count_all;
575
576
  /** @brief The number of selected partitionings for 1/2/3/4 partitionings. */
577
  unsigned int partitioning_count_selected[BLOCK_MAX_PARTITIONS];
578
579
  /** @brief The number of partitionings for 1/2/3/4 partitionings. */
580
  unsigned int partitioning_count_all[BLOCK_MAX_PARTITIONS];
581
582
  /** @brief The active decimation modes, stored in low indices. */
583
  decimation_mode decimation_modes[WEIGHTS_MAX_DECIMATION_MODES];
584
585
  /** @brief The active decimation tables, stored in low indices. */
586
  ASTCENC_ALIGNAS decimation_info decimation_tables[WEIGHTS_MAX_DECIMATION_MODES];
587
588
  /** @brief The packed block mode array index, or @c BLOCK_BAD_BLOCK_MODE if not active. */
589
  uint16_t block_mode_packed_index[WEIGHTS_MAX_BLOCK_MODES];
590
591
  /** @brief The active block modes, stored in low indices. */
592
  block_mode block_modes[WEIGHTS_MAX_BLOCK_MODES];
593
594
  /** @brief The active partition tables, stored in low indices per-count. */
595
  partition_info partitionings[(3 * BLOCK_MAX_PARTITIONINGS) + 1];
596
597
  /**
598
   * @brief The packed partition table array index, or @c BLOCK_BAD_PARTITIONING if not active.
599
   *
600
   * Indexed by partition_count - 2, containing 2, 3 and 4 partitions.
601
   */
602
  uint16_t partitioning_packed_index[3][BLOCK_MAX_PARTITIONINGS];
603
604
#if !defined(ASTCENC_DECOMPRESS_ONLY)
605
  /** @brief The active texels for k-means partition selection. */
606
  uint8_t kmeans_texels[BLOCK_MAX_KMEANS_TEXELS];
607
608
  /**
609
   * @brief The canonical 2-partition coverage pattern used during block partition search.
610
   *
611
   * Indexed by remapped index, not physical index.
612
   */
613
  uint64_t coverage_bitmaps_2[BLOCK_MAX_PARTITIONINGS][2];
614
615
  /**
616
   * @brief The canonical 3-partition coverage pattern used during block partition search.
617
   *
618
   * Indexed by remapped index, not physical index.
619
   */
620
  uint64_t coverage_bitmaps_3[BLOCK_MAX_PARTITIONINGS][3];
621
622
  /**
623
   * @brief The canonical 4-partition coverage pattern used during block partition search.
624
   *
625
   * Indexed by remapped index, not physical index.
626
   */
627
  uint64_t coverage_bitmaps_4[BLOCK_MAX_PARTITIONINGS][4];
628
#endif
629
630
  /**
631
   * @brief Get the block mode structure for index @c block_mode.
632
   *
633
   * This function can only return block modes that are enabled by the current compressor config.
634
   * Decompression from an arbitrary source should not use this without first checking that the
635
   * packed block mode index is not @c BLOCK_BAD_BLOCK_MODE.
636
   *
637
   * @param block_mode   The packed block mode index.
638
   *
639
   * @return The block mode structure.
640
   */
641
  const block_mode& get_block_mode(unsigned int block_mode) const
642
5.60k
  {
643
5.60k
    unsigned int packed_index = this->block_mode_packed_index[block_mode];
644
5.60k
    assert(packed_index != BLOCK_BAD_BLOCK_MODE && packed_index < this->block_mode_count_all);
645
5.60k
    return this->block_modes[packed_index];
646
5.60k
  }
647
648
  /**
649
   * @brief Get the decimation mode structure for index @c decimation_mode.
650
   *
651
   * This function can only return decimation modes that are enabled by the current compressor
652
   * config. The mode array is stored packed, but this is only ever indexed by the packed index
653
   * stored in the @c block_mode and never exists in an unpacked form.
654
   *
655
   * @param decimation_mode   The packed decimation mode index.
656
   *
657
   * @return The decimation mode structure.
658
   */
659
  const decimation_mode& get_decimation_mode(unsigned int decimation_mode) const
660
0
  {
661
0
    return this->decimation_modes[decimation_mode];
662
0
  }
663
664
  /**
665
   * @brief Get the decimation info structure for index @c decimation_mode.
666
   *
667
   * This function can only return decimation modes that are enabled by the current compressor
668
   * config. The mode array is stored packed, but this is only ever indexed by the packed index
669
   * stored in the @c block_mode and never exists in an unpacked form.
670
   *
671
   * @param decimation_mode   The packed decimation mode index.
672
   *
673
   * @return The decimation info structure.
674
   */
675
  const decimation_info& get_decimation_info(unsigned int decimation_mode) const
676
5.60k
  {
677
5.60k
    return this->decimation_tables[decimation_mode];
678
5.60k
  }
679
680
  /**
681
   * @brief Get the partition info table for a given partition count.
682
   *
683
   * @param partition_count   The number of partitions we want the table for.
684
   *
685
   * @return The pointer to the table of 1024 entries (for 2/3/4 parts) or 1 entry (for 1 part).
686
   */
687
  const partition_info* get_partition_table(unsigned int partition_count) const
688
2.66k
  {
689
2.66k
    if (partition_count == 1)
690
401
    {
691
401
      partition_count = 5;
692
401
    }
693
2.66k
    unsigned int index = (partition_count - 2) * BLOCK_MAX_PARTITIONINGS;
694
2.66k
    return this->partitionings + index;
695
2.66k
  }
696
697
  /**
698
   * @brief Get the partition info structure for a given partition count and seed.
699
   *
700
   * @param partition_count   The number of partitions we want the info for.
701
   * @param index             The partition seed (between 0 and 1023).
702
   *
703
   * @return The partition info structure.
704
   */
705
  const partition_info& get_partition_info(unsigned int partition_count, unsigned int index) const
706
2.66k
  {
707
2.66k
    unsigned int packed_index = 0;
708
2.66k
    if (partition_count >= 2)
709
2.26k
    {
710
2.26k
      packed_index = this->partitioning_packed_index[partition_count - 2][index];
711
2.26k
    }
712
713
2.66k
    assert(packed_index != BLOCK_BAD_PARTITIONING && packed_index < this->partitioning_count_all[partition_count - 1]);
714
2.66k
    auto& result = get_partition_table(partition_count)[packed_index];
715
2.66k
    assert(index == result.partition_index);
716
2.66k
    return result;
717
2.66k
  }
718
719
  /**
720
   * @brief Get the partition info structure for a given partition count and seed.
721
   *
722
   * @param partition_count   The number of partitions we want the info for.
723
   * @param packed_index      The raw array offset.
724
   *
725
   * @return The partition info structure.
726
   */
727
  const partition_info& get_raw_partition_info(unsigned int partition_count, unsigned int packed_index) const
728
0
  {
729
0
    assert(packed_index != BLOCK_BAD_PARTITIONING && packed_index < this->partitioning_count_all[partition_count - 1]);
730
0
    auto& result = get_partition_table(partition_count)[packed_index];
731
0
    return result;
732
0
  }
733
};
734
735
/**
736
 * @brief The image data for a single block.
737
 *
738
 * The @c data_[rgba] fields store the image data in an encoded SoA float form designed for easy
739
 * vectorization. Input data is converted to float and stored as values between 0 and 65535. LDR
740
 * data is stored as direct UNORM data, HDR data is stored as LNS data. They are allocated SIMD
741
 * elements over-size to allow vectorized stores of unaligned and partial SIMD lanes (e.g. in a
742
 * 6x6x6 block the final row write will read elements 210-217 (vec8) or 214-217 (vec4), which is
743
 * two elements above the last real data element). The overspill values are never written to memory,
744
 * and would be benign, but the padding avoids hitting undefined behavior.
745
 *
746
 * The @c rgb_lns and @c alpha_lns fields that assigned a per-texel use of HDR are only used during
747
 * decompression. The current compressor will always use HDR endpoint formats when in HDR mode.
748
 */
749
struct image_block
750
{
751
  /** @brief The input (compress) or output (decompress) data for the red color component. */
752
  ASTCENC_ALIGNAS float data_r[BLOCK_MAX_TEXELS + ASTCENC_SIMD_WIDTH - 1];
753
754
  /** @brief The input (compress) or output (decompress) data for the green color component. */
755
  ASTCENC_ALIGNAS float data_g[BLOCK_MAX_TEXELS + ASTCENC_SIMD_WIDTH - 1];
756
757
  /** @brief The input (compress) or output (decompress) data for the blue color component. */
758
  ASTCENC_ALIGNAS float data_b[BLOCK_MAX_TEXELS + ASTCENC_SIMD_WIDTH - 1];
759
760
  /** @brief The input (compress) or output (decompress) data for the alpha color component. */
761
  ASTCENC_ALIGNAS float data_a[BLOCK_MAX_TEXELS + ASTCENC_SIMD_WIDTH - 1];
762
763
  /** @brief The number of texels in the block. */
764
  uint8_t texel_count;
765
766
  /** @brief The original data for texel 0 for constant color block encoding. */
767
  vfloat4 origin_texel;
768
769
  /** @brief The min component value of all texels in the block. */
770
  vfloat4 data_min;
771
772
  /** @brief The mean component value of all texels in the block. */
773
  vfloat4 data_mean;
774
775
  /** @brief The max component value of all texels in the block. */
776
  vfloat4 data_max;
777
778
  /** @brief The relative error significance of the color channels. */
779
  vfloat4 channel_weight;
780
781
  /** @brief Is this grayscale block where R == G == B for all texels? */
782
  bool grayscale;
783
784
  /** @brief Is the eventual decode using decode_unorm8 rounding? */
785
  bool decode_unorm8;
786
787
  /** @brief Set to 1 if a texel is using HDR RGB endpoints (decompression only). */
788
  uint8_t rgb_lns[BLOCK_MAX_TEXELS];
789
790
  /** @brief Set to 1 if a texel is using HDR alpha endpoints (decompression only). */
791
  uint8_t alpha_lns[BLOCK_MAX_TEXELS];
792
793
  /** @brief The X position of this block in the input or output image. */
794
  unsigned int xpos;
795
796
  /** @brief The Y position of this block in the input or output image. */
797
  unsigned int ypos;
798
799
  /** @brief The Z position of this block in the input or output image. */
800
  unsigned int zpos;
801
802
  /**
803
   * @brief Get an RGBA texel value from the data.
804
   *
805
   * @param index   The texel index.
806
   *
807
   * @return The texel in RGBA component ordering.
808
   */
809
  inline vfloat4 texel(unsigned int index) const
810
0
  {
811
0
    return vfloat4(data_r[index],
812
0
                   data_g[index],
813
0
                   data_b[index],
814
0
                   data_a[index]);
815
0
  }
816
817
  /**
818
   * @brief Get an RGB texel value from the data.
819
   *
820
   * @param index   The texel index.
821
   *
822
   * @return The texel in RGB0 component ordering.
823
   */
824
  inline vfloat4 texel3(unsigned int index) const
825
0
  {
826
0
    return vfloat3(data_r[index],
827
0
                   data_g[index],
828
0
                   data_b[index]);
829
0
  }
830
831
  /**
832
   * @brief Get the default alpha value for endpoints that don't store it.
833
   *
834
   * The default depends on whether the alpha endpoint is LDR or HDR.
835
   *
836
   * @return The alpha value in the scaled range used by the compressor.
837
   */
838
  inline float get_default_alpha() const
839
0
  {
840
0
    return this->alpha_lns[0] ? static_cast<float>(0x7800) : static_cast<float>(0xFFFF);
841
0
  }
842
843
  /**
844
   * @brief Test if a single color channel is constant across the block.
845
   *
846
   * Constant color channels are easier to compress as interpolating between two identical colors
847
   * always returns the same value, irrespective of the weight used. They therefore can be ignored
848
   * for the purposes of weight selection and use of a second weight plane.
849
   *
850
   * @return @c true if the channel is constant across the block, @c false otherwise.
851
   */
852
  inline bool is_constant_channel(int channel) const
853
0
  {
854
0
    vmask4 lane_mask = vint4::lane_id() == vint4(channel);
855
0
    vmask4 color_mask = this->data_min == this->data_max;
856
0
    return any(lane_mask & color_mask);
857
0
  }
858
859
  /**
860
   * @brief Test if this block is a luminance block with constant 1.0 alpha.
861
   *
862
   * @return @c true if the block is a luminance block , @c false otherwise.
863
   */
864
  inline bool is_luminance() const
865
0
  {
866
0
    float default_alpha = this->get_default_alpha();
867
0
    bool alpha1 = (this->data_min.lane<3>() == default_alpha) &&
868
0
                  (this->data_max.lane<3>() == default_alpha);
869
0
    return this->grayscale && alpha1;
870
0
  }
871
872
  /**
873
   * @brief Test if this block is a luminance block with variable alpha.
874
   *
875
   * @return @c true if the block is a luminance + alpha block , @c false otherwise.
876
   */
877
  inline bool is_luminancealpha() const
878
0
  {
879
0
    float default_alpha = this->get_default_alpha();
880
0
    bool alpha1 = (this->data_min.lane<3>() == default_alpha) &&
881
0
                  (this->data_max.lane<3>() == default_alpha);
882
0
    return this->grayscale && !alpha1;
883
0
  }
884
};
885
886
/**
887
 * @brief Data structure storing the color endpoints for a block.
888
 */
889
struct endpoints
890
{
891
  /** @brief The number of partition endpoints stored. */
892
  unsigned int partition_count;
893
894
  /** @brief The colors for endpoint 0. */
895
  vfloat4 endpt0[BLOCK_MAX_PARTITIONS];
896
897
  /** @brief The colors for endpoint 1. */
898
  vfloat4 endpt1[BLOCK_MAX_PARTITIONS];
899
};
900
901
/**
902
 * @brief Data structure storing the color endpoints and weights.
903
 */
904
struct endpoints_and_weights
905
{
906
  /** @brief True if all active values in weight_error_scale are the same. */
907
  bool is_constant_weight_error_scale;
908
909
  /** @brief The color endpoints. */
910
  endpoints ep;
911
912
  /** @brief The ideal weight for each texel; may be undecimated or decimated. */
913
  ASTCENC_ALIGNAS float weights[BLOCK_MAX_TEXELS];
914
915
  /** @brief The ideal weight error scaling for each texel; may be undecimated or decimated. */
916
  ASTCENC_ALIGNAS float weight_error_scale[BLOCK_MAX_TEXELS];
917
};
918
919
/**
920
 * @brief Utility storing estimated errors from choosing particular endpoint encodings.
921
 */
922
struct encoding_choice_errors
923
{
924
  /** @brief Error of using LDR RGB-scale instead of complete endpoints. */
925
  float rgb_scale_error;
926
927
  /** @brief Error of using HDR RGB-scale instead of complete endpoints. */
928
  float rgb_luma_error;
929
930
  /** @brief Error of using luminance instead of RGB. */
931
  float luminance_error;
932
933
  /** @brief Error of discarding alpha and using a constant 1.0 alpha. */
934
  float alpha_drop_error;
935
936
  /** @brief Can we use delta offset encoding? */
937
  bool can_offset_encode;
938
939
  /** @brief Can we use blue contraction encoding? */
940
  bool can_blue_contract;
941
};
942
943
/**
944
 * @brief Preallocated working buffers, allocated per thread during context creation.
945
 */
946
struct ASTCENC_ALIGNAS compression_working_buffers
947
{
948
  /** @brief Ideal endpoints and weights for plane 1. */
949
  endpoints_and_weights ei1;
950
951
  /** @brief Ideal endpoints and weights for plane 2. */
952
  endpoints_and_weights ei2;
953
954
  /**
955
   * @brief Decimated ideal weight values in the ~0-1 range.
956
   *
957
   * Note that values can be slightly below zero or higher than one due to
958
   * endpoint extents being inside the ideal color representation.
959
   *
960
   * For two planes, second plane starts at @c WEIGHTS_PLANE2_OFFSET offsets.
961
   */
962
  ASTCENC_ALIGNAS float dec_weights_ideal[WEIGHTS_MAX_DECIMATION_MODES * BLOCK_MAX_WEIGHTS];
963
964
  /**
965
   * @brief Decimated quantized weight values in the unquantized 0-64 range.
966
   *
967
   * For two planes, second plane starts at @c WEIGHTS_PLANE2_OFFSET offsets.
968
   */
969
  ASTCENC_ALIGNAS uint8_t dec_weights_uquant[WEIGHTS_MAX_BLOCK_MODES * BLOCK_MAX_WEIGHTS];
970
971
  /** @brief Error of the best encoding combination for each block mode. */
972
  ASTCENC_ALIGNAS float errors_of_best_combination[WEIGHTS_MAX_BLOCK_MODES];
973
974
  /** @brief The best color quant for each block mode. */
975
  uint8_t best_quant_levels[WEIGHTS_MAX_BLOCK_MODES];
976
977
  /** @brief The best color quant for each block mode if modes are the same and we have spare bits. */
978
  uint8_t best_quant_levels_mod[WEIGHTS_MAX_BLOCK_MODES];
979
980
  /** @brief The best endpoint format for each partition. */
981
  uint8_t best_ep_formats[WEIGHTS_MAX_BLOCK_MODES][BLOCK_MAX_PARTITIONS];
982
983
  /** @brief The total bit storage needed for quantized weights for each block mode. */
984
  int8_t qwt_bitcounts[WEIGHTS_MAX_BLOCK_MODES];
985
986
  /** @brief The cumulative error for quantized weights for each block mode. */
987
  float qwt_errors[WEIGHTS_MAX_BLOCK_MODES];
988
989
  /** @brief The low weight value in plane 1 for each block mode. */
990
  float weight_low_value1[WEIGHTS_MAX_BLOCK_MODES];
991
992
  /** @brief The high weight value in plane 1 for each block mode. */
993
  float weight_high_value1[WEIGHTS_MAX_BLOCK_MODES];
994
995
  /** @brief The low weight value in plane 1 for each quant level and decimation mode. */
996
  float weight_low_values1[WEIGHTS_MAX_DECIMATION_MODES][TUNE_MAX_ANGULAR_QUANT + 1];
997
998
  /** @brief The high weight value in plane 1 for each quant level and decimation mode. */
999
  float weight_high_values1[WEIGHTS_MAX_DECIMATION_MODES][TUNE_MAX_ANGULAR_QUANT + 1];
1000
1001
  /** @brief The low weight value in plane 2 for each block mode. */
1002
  float weight_low_value2[WEIGHTS_MAX_BLOCK_MODES];
1003
1004
  /** @brief The high weight value in plane 2 for each block mode. */
1005
  float weight_high_value2[WEIGHTS_MAX_BLOCK_MODES];
1006
1007
  /** @brief The low weight value in plane 2 for each quant level and decimation mode. */
1008
  float weight_low_values2[WEIGHTS_MAX_DECIMATION_MODES][TUNE_MAX_ANGULAR_QUANT + 1];
1009
1010
  /** @brief The high weight value in plane 2 for each quant level and decimation mode. */
1011
  float weight_high_values2[WEIGHTS_MAX_DECIMATION_MODES][TUNE_MAX_ANGULAR_QUANT + 1];
1012
};
1013
1014
struct dt_init_working_buffers
1015
{
1016
  uint8_t weight_count_of_texel[BLOCK_MAX_TEXELS];
1017
  uint8_t grid_weights_of_texel[BLOCK_MAX_TEXELS][4];
1018
  uint8_t weights_of_texel[BLOCK_MAX_TEXELS][4];
1019
1020
  uint8_t texel_count_of_weight[BLOCK_MAX_WEIGHTS];
1021
  uint8_t texels_of_weight[BLOCK_MAX_WEIGHTS][BLOCK_MAX_TEXELS];
1022
  uint8_t texel_weights_of_weight[BLOCK_MAX_WEIGHTS][BLOCK_MAX_TEXELS];
1023
};
1024
1025
/**
1026
 * @brief Weight quantization transfer table.
1027
 *
1028
 * ASTC can store texel weights at many quantization levels, so for performance we store essential
1029
 * information about each level as a precomputed data structure. Unquantized weights are integers
1030
 * or floats in the range [0, 64].
1031
 *
1032
 * This structure provides a table, used to estimate the closest quantized weight for a given
1033
 * floating-point weight. For each quantized weight, the corresponding unquantized values. For each
1034
 * quantized weight, a previous-value and a next-value.
1035
*/
1036
struct quant_and_transfer_table
1037
{
1038
  /** @brief The unscrambled unquantized value. */
1039
  uint8_t quant_to_unquant[32];
1040
1041
  /** @brief The scrambling order: scrambled_quant = map[unscrambled_quant]. */
1042
  uint8_t scramble_map[32];
1043
1044
  /** @brief The unscrambling order: unscrambled_unquant = map[scrambled_quant]. */
1045
  uint8_t unscramble_and_unquant_map[32];
1046
1047
  /**
1048
   * @brief A table of previous-and-next weights, indexed by the current unquantized value.
1049
   *  * bits 7:0 = previous-index, unquantized
1050
   *  * bits 15:8 = next-index, unquantized
1051
   */
1052
  uint16_t prev_next_values[65];
1053
};
1054
1055
/** @brief The precomputed quant and transfer table. */
1056
extern const quant_and_transfer_table quant_and_xfer_tables[12];
1057
1058
/** @brief The block is an error block, and will return error color or NaN. */
1059
static constexpr uint8_t SYM_BTYPE_ERROR { 0 };
1060
1061
/** @brief The block is a constant color block using FP16 colors. */
1062
static constexpr uint8_t SYM_BTYPE_CONST_F16 { 1 };
1063
1064
/** @brief The block is a constant color block using UNORM16 colors. */
1065
static constexpr uint8_t SYM_BTYPE_CONST_U16 { 2 };
1066
1067
/** @brief The block is a normal non-constant color block. */
1068
static constexpr uint8_t SYM_BTYPE_NONCONST { 3 };
1069
1070
/**
1071
 * @brief A symbolic representation of a compressed block.
1072
 *
1073
 * The symbolic representation stores the unpacked content of a single
1074
 * physical compressed block, in a form which is much easier to access for
1075
 * the rest of the compressor code.
1076
 */
1077
struct symbolic_compressed_block
1078
{
1079
  /** @brief The block type, one of the @c SYM_BTYPE_* constants. */
1080
  uint8_t block_type;
1081
1082
  /** @brief The number of partitions; valid for @c NONCONST blocks. */
1083
  uint8_t partition_count;
1084
1085
  /** @brief Non-zero if the color formats matched; valid for @c NONCONST blocks. */
1086
  uint8_t color_formats_matched;
1087
1088
  /** @brief The plane 2 color component, or -1 if single plane; valid for @c NONCONST blocks. */
1089
  int8_t plane2_component;
1090
1091
  /** @brief The block mode; valid for @c NONCONST blocks. */
1092
  uint16_t block_mode;
1093
1094
  /** @brief The partition index; valid for @c NONCONST blocks if 2 or more partitions. */
1095
  uint16_t partition_index;
1096
1097
  /** @brief The endpoint color formats for each partition; valid for @c NONCONST blocks. */
1098
  uint8_t color_formats[BLOCK_MAX_PARTITIONS];
1099
1100
  /** @brief The endpoint color quant mode; valid for @c NONCONST blocks. */
1101
  quant_method quant_mode;
1102
1103
  /** @brief The error of the current encoding; valid for @c NONCONST blocks. */
1104
  float errorval;
1105
1106
  // We can't have both of these at the same time
1107
  union {
1108
    /** @brief The constant color; valid for @c CONST blocks. */
1109
    int constant_color[BLOCK_MAX_COMPONENTS];
1110
1111
    /** @brief The quantized endpoint color pairs; valid for @c NONCONST blocks. */
1112
    uint8_t color_values[BLOCK_MAX_PARTITIONS][8];
1113
  };
1114
1115
  /** @brief The quantized and decimated weights.
1116
   *
1117
   * Weights are stored in the 0-64 unpacked range allowing them to be used
1118
   * directly in encoding passes without per-use unpacking. Packing happens
1119
   * when converting to/from the physical bitstream encoding.
1120
   *
1121
   * If dual plane, the second plane starts at @c weights[WEIGHTS_PLANE2_OFFSET].
1122
   */
1123
  ASTCENC_ALIGNAS uint8_t weights[BLOCK_MAX_WEIGHTS];
1124
1125
  /**
1126
   * @brief Get the weight quantization used by this block mode.
1127
   *
1128
   * @return The quantization level.
1129
   */
1130
  inline quant_method get_color_quant_mode() const
1131
0
  {
1132
0
    return this->quant_mode;
1133
0
  }
1134
};
1135
1136
/**
1137
 * @brief Parameter structure for @c compute_pixel_region_variance().
1138
 *
1139
 * This function takes a structure to avoid spilling arguments to the stack on every function
1140
 * invocation, as there are a lot of parameters.
1141
 */
1142
struct pixel_region_args
1143
{
1144
  /** @brief The image to analyze. */
1145
  const astcenc_image* img;
1146
1147
  /** @brief The component swizzle pattern. */
1148
  astcenc_swizzle swz;
1149
1150
  /** @brief Should the algorithm bother with Z axis processing? */
1151
  bool have_z;
1152
1153
  /** @brief The kernel radius for alpha processing. */
1154
  unsigned int alpha_kernel_radius;
1155
1156
  /** @brief The X dimension of the working data to process. */
1157
  unsigned int size_x;
1158
1159
  /** @brief The Y dimension of the working data to process. */
1160
  unsigned int size_y;
1161
1162
  /** @brief The Z dimension of the working data to process. */
1163
  unsigned int size_z;
1164
1165
  /** @brief The X position of first src and dst data in the data set. */
1166
  unsigned int offset_x;
1167
1168
  /** @brief The Y position of first src and dst data in the data set. */
1169
  unsigned int offset_y;
1170
1171
  /** @brief The Z position of first src and dst data in the data set. */
1172
  unsigned int offset_z;
1173
1174
  /** @brief The working memory buffer. */
1175
  vfloat4 *work_memory;
1176
};
1177
1178
/**
1179
 * @brief Parameter structure for @c compute_averages_proc().
1180
 */
1181
struct avg_args
1182
{
1183
  /** @brief The arguments for the nested variance computation. */
1184
  pixel_region_args arg;
1185
1186
  /** @brief The image X dimensions. */
1187
  unsigned int img_size_x;
1188
1189
  /** @brief The image Y dimensions. */
1190
  unsigned int img_size_y;
1191
1192
  /** @brief The image Z dimensions. */
1193
  unsigned int img_size_z;
1194
1195
  /** @brief The maximum working block dimensions in X and Y dimensions. */
1196
  unsigned int blk_size_xy;
1197
1198
  /** @brief The maximum working block dimensions in Z dimensions. */
1199
  unsigned int blk_size_z;
1200
1201
  /** @brief The working block memory size. */
1202
  unsigned int work_memory_size;
1203
};
1204
1205
#if defined(ASTCENC_DIAGNOSTICS)
1206
/* See astcenc_diagnostic_trace header for details. */
1207
class TraceLog;
1208
#endif
1209
1210
/**
1211
 * @brief The astcenc compression context.
1212
 */
1213
struct astcenc_contexti
1214
{
1215
  /** @brief The configuration this context was created with. */
1216
  astcenc_config config;
1217
1218
  /** @brief The thread count supported by this context. */
1219
  unsigned int thread_count;
1220
1221
  /** @brief Is this context the owner of @c bsd, or is it a child inheriting it. */
1222
  bool owns_bsd;
1223
1224
  /** @brief The block size descriptor this context was created with. */
1225
  const block_size_descriptor* bsd;
1226
1227
  /*
1228
   * Fields below here are not needed in a decompress-only build, but some remain as they are
1229
   * small and it avoids littering the code with #ifdefs. The most significant contributors to
1230
   * large structure size are omitted.
1231
   */
1232
1233
  /** @brief The input image alpha channel averages table, may be @c nullptr if not needed. */
1234
  float* input_alpha_averages;
1235
1236
  /** @brief The scratch working buffers, one per thread (see @c thread_count). */
1237
  compression_working_buffers* working_buffers;
1238
1239
#if !defined(ASTCENC_DECOMPRESS_ONLY)
1240
  /** @brief The pixel region and variance worker arguments. */
1241
  avg_args avg_preprocess_args;
1242
#endif
1243
1244
#if defined(ASTCENC_DIAGNOSTICS)
1245
  /**
1246
   * @brief The diagnostic trace logger.
1247
   *
1248
   * Note that this is a singleton, so can only be used in single threaded mode. It only exists
1249
   * here so we have a reference to close the file at the end of the capture.
1250
   */
1251
  TraceLog* trace_log;
1252
#endif
1253
};
1254
1255
/* ============================================================================
1256
  Functionality for managing block sizes and partition tables.
1257
============================================================================ */
1258
1259
/**
1260
 * @brief Populate the block size descriptor for the target block size.
1261
 *
1262
 * This will also initialize the partition table metadata, which is stored as part of the BSD
1263
 * structure.
1264
 *
1265
 * @param      x_texels                 The number of texels in the block X dimension.
1266
 * @param      y_texels                 The number of texels in the block Y dimension.
1267
 * @param      z_texels                 The number of texels in the block Z dimension.
1268
 * @param      can_omit_modes           Can we discard modes and partitionings that astcenc won't use?
1269
 * @param      partition_count_cutoff   The partition count cutoff to use, if we can omit partitionings.
1270
 * @param      mode_cutoff              The block mode percentile cutoff [0-1].
1271
 * @param[out] bsd                      The descriptor to initialize.
1272
 */
1273
void init_block_size_descriptor(
1274
  unsigned int x_texels,
1275
  unsigned int y_texels,
1276
  unsigned int z_texels,
1277
  bool can_omit_modes,
1278
  unsigned int partition_count_cutoff,
1279
  float mode_cutoff,
1280
  block_size_descriptor& bsd);
1281
1282
/**
1283
 * @brief Populate the partition tables for the target block size.
1284
 *
1285
 * Note the @c bsd descriptor must be initialized by calling @c init_block_size_descriptor() before
1286
 * calling this function.
1287
 *
1288
 * @param[out] bsd                      The block size information structure to populate.
1289
 * @param      can_omit_partitionings   True if we can we drop partitionings that astcenc won't use.
1290
 * @param      partition_count_cutoff   The partition count cutoff to use, if we can omit partitionings.
1291
 */
1292
void init_partition_tables(
1293
  block_size_descriptor& bsd,
1294
  bool can_omit_partitionings,
1295
  unsigned int partition_count_cutoff);
1296
1297
/**
1298
 * @brief Get the percentile table for 2D block modes.
1299
 *
1300
 * This is an empirically determined prioritization of which block modes to use in the search in
1301
 * terms of their centile (lower centiles = more useful).
1302
 *
1303
 * Returns a dynamically allocated array; caller must free with delete[].
1304
 *
1305
 * @param xdim The block x size.
1306
 * @param ydim The block y size.
1307
 *
1308
 * @return The unpacked table.
1309
 */
1310
const float* get_2d_percentile_table(
1311
  unsigned int xdim,
1312
  unsigned int ydim);
1313
1314
/**
1315
 * @brief Query if a 2D block size is legal.
1316
 *
1317
 * @return True if legal, false otherwise.
1318
 */
1319
bool is_legal_2d_block_size(
1320
  unsigned int xdim,
1321
  unsigned int ydim);
1322
1323
/**
1324
 * @brief Query if a 3D block size is legal.
1325
 *
1326
 * @return True if legal, false otherwise.
1327
 */
1328
bool is_legal_3d_block_size(
1329
  unsigned int xdim,
1330
  unsigned int ydim,
1331
  unsigned int zdim);
1332
1333
/* ============================================================================
1334
  Functionality for managing BISE quantization and unquantization.
1335
============================================================================ */
1336
1337
/**
1338
 * @brief The precomputed table for quantizing color values.
1339
 *
1340
 * Converts unquant value in 0-255 range into quant value in 0-255 range.
1341
 * No BISE scrambling is applied at this stage.
1342
 *
1343
 * The BISE encoding results in ties where available quant<256> values are
1344
 * equidistant the available quant<BISE> values. This table stores two values
1345
 * for each input - one for use with a negative residual, and one for use with
1346
 * a positive residual.
1347
 *
1348
 * Indexed by [quant_mode - 4][data_value * 2 + residual].
1349
 */
1350
extern const uint8_t color_unquant_to_uquant_tables[17][512];
1351
1352
/**
1353
 * @brief The precomputed table for packing quantized color values.
1354
 *
1355
 * Converts quant value in 0-255 range into packed quant value in 0-N range,
1356
 * with BISE scrambling applied.
1357
 *
1358
 * Indexed by [quant_mode - 4][data_value].
1359
 */
1360
extern const uint8_t color_uquant_to_scrambled_pquant_tables[17][256];
1361
1362
/**
1363
 * @brief The precomputed table for unpacking color values.
1364
 *
1365
 * Converts quant value in 0-N range into unpacked value in 0-255 range,
1366
 * with BISE unscrambling applied.
1367
 *
1368
 * Indexed by [quant_mode - 4][data_value].
1369
 */
1370
extern const uint8_t* color_scrambled_pquant_to_uquant_tables[17];
1371
1372
/**
1373
 * @brief The precomputed quant mode storage table.
1374
 *
1375
 * Indexing by [integer_count/2][bits] gives us the quantization level for a given integer count and
1376
 * number of compressed storage bits. Returns -1 for cases where the requested integer count cannot
1377
 * ever fit in the supplied storage size.
1378
 */
1379
extern const int8_t quant_mode_table[10][128];
1380
1381
/**
1382
 * @brief Encode a packed string using BISE.
1383
 *
1384
 * Note that BISE can return strings that are not a whole number of bytes in length, and ASTC can
1385
 * start storing strings in a block at arbitrary bit offsets in the encoded data.
1386
 *
1387
 * @param         quant_level       The BISE alphabet size.
1388
 * @param         character_count   The number of characters in the string.
1389
 * @param         input_data        The unpacked string, one byte per character.
1390
 * @param[in,out] output_data       The output packed string.
1391
 * @param         bit_offset        The starting offset in the output storage.
1392
 */
1393
void encode_ise(
1394
  quant_method quant_level,
1395
  unsigned int character_count,
1396
  const uint8_t* input_data,
1397
  uint8_t* output_data,
1398
  unsigned int bit_offset);
1399
1400
/**
1401
 * @brief Decode a packed string using BISE.
1402
 *
1403
 * Note that BISE input strings are not a whole number of bytes in length, and ASTC can start
1404
 * strings at arbitrary bit offsets in the encoded data.
1405
 *
1406
 * @param         quant_level       The BISE alphabet size.
1407
 * @param         character_count   The number of characters in the string.
1408
 * @param         input_data        The packed string.
1409
 * @param[in,out] output_data       The output storage, one byte per character.
1410
 * @param         bit_offset        The starting offset in the output storage.
1411
 */
1412
void decode_ise(
1413
  quant_method quant_level,
1414
  unsigned int character_count,
1415
  const uint8_t* input_data,
1416
  uint8_t* output_data,
1417
  unsigned int bit_offset);
1418
1419
/**
1420
 * @brief Return the number of bits needed to encode an ISE sequence.
1421
 *
1422
 * This implementation assumes that the @c quant level is untrusted, given it may come from random
1423
 * data being decompressed, so we return an arbitrary unencodable size if that is the case.
1424
 *
1425
 * @param character_count   The number of items in the sequence.
1426
 * @param quant_level       The desired quantization level.
1427
 *
1428
 * @return The number of bits needed to encode the BISE string.
1429
 */
1430
unsigned int get_ise_sequence_bitcount(
1431
  unsigned int character_count,
1432
  quant_method quant_level);
1433
1434
/* ============================================================================
1435
  Functionality for managing color partitioning.
1436
============================================================================ */
1437
1438
/**
1439
 * @brief Compute averages and dominant directions for each partition in a 2 component texture.
1440
 *
1441
 * @param      pi           The partition info for the current trial.
1442
 * @param      blk          The image block color data to be compressed.
1443
 * @param      component1   The first component included in the analysis.
1444
 * @param      component2   The second component included in the analysis.
1445
 * @param[out] pm           The output partition metrics.
1446
 *                          - Only pi.partition_count array entries actually get initialized.
1447
 *                          - Direction vectors @c pm.dir are not normalized.
1448
 */
1449
void compute_avgs_and_dirs_2_comp(
1450
  const partition_info& pi,
1451
  const image_block& blk,
1452
  unsigned int component1,
1453
  unsigned int component2,
1454
  partition_metrics pm[BLOCK_MAX_PARTITIONS]);
1455
1456
/**
1457
 * @brief Compute averages and dominant directions for each partition in a 3 component texture.
1458
 *
1459
 * @param      pi                  The partition info for the current trial.
1460
 * @param      blk                 The image block color data to be compressed.
1461
 * @param      omitted_component   The component excluded from the analysis.
1462
 * @param[out] pm                  The output partition metrics.
1463
 *                                 - Only pi.partition_count array entries actually get initialized.
1464
 *                                 - Direction vectors @c pm.dir are not normalized.
1465
 */
1466
void compute_avgs_and_dirs_3_comp(
1467
  const partition_info& pi,
1468
  const image_block& blk,
1469
  unsigned int omitted_component,
1470
  partition_metrics pm[BLOCK_MAX_PARTITIONS]);
1471
1472
/**
1473
 * @brief Compute averages and dominant directions for each partition in a 3 component texture.
1474
 *
1475
 * This is a specialization of @c compute_avgs_and_dirs_3_comp where the omitted component is
1476
 * always alpha, a common case during partition search.
1477
 *
1478
 * @param      pi    The partition info for the current trial.
1479
 * @param      blk   The image block color data to be compressed.
1480
 * @param[out] pm    The output partition metrics.
1481
 *                   - Only pi.partition_count array entries actually get initialized.
1482
 *                   - Direction vectors @c pm.dir are not normalized.
1483
 */
1484
void compute_avgs_and_dirs_3_comp_rgb(
1485
  const partition_info& pi,
1486
  const image_block& blk,
1487
  partition_metrics pm[BLOCK_MAX_PARTITIONS]);
1488
1489
/**
1490
 * @brief Compute averages and dominant directions for each partition in a 4 component texture.
1491
 *
1492
 * @param      pi    The partition info for the current trial.
1493
 * @param      blk   The image block color data to be compressed.
1494
 * @param[out] pm    The output partition metrics.
1495
 *                   - Only pi.partition_count array entries actually get initialized.
1496
 *                   - Direction vectors @c pm.dir are not normalized.
1497
 */
1498
void compute_avgs_and_dirs_4_comp(
1499
  const partition_info& pi,
1500
  const image_block& blk,
1501
  partition_metrics pm[BLOCK_MAX_PARTITIONS]);
1502
1503
/**
1504
 * @brief Compute the RGB error for uncorrelated and same chroma projections.
1505
 *
1506
 * The output of compute averages and dirs is post processed to define two lines, both of which go
1507
 * through the mean-color-value.  One line has a direction defined by the dominant direction; this
1508
 * is used to assess the error from using an uncorrelated color representation. The other line goes
1509
 * through (0,0,0) and is used to assess the error from using an RGBS color representation.
1510
 *
1511
 * This function computes the squared error when using these two representations.
1512
 *
1513
 * @param         pi            The partition info for the current trial.
1514
 * @param         blk           The image block color data to be compressed.
1515
 * @param[in,out] plines        Processed line inputs, and line length outputs.
1516
 * @param[out]    uncor_error   The cumulative error for using the uncorrelated line.
1517
 * @param[out]    samec_error   The cumulative error for using the same chroma line.
1518
 */
1519
void compute_error_squared_rgb(
1520
  const partition_info& pi,
1521
  const image_block& blk,
1522
  partition_lines3 plines[BLOCK_MAX_PARTITIONS],
1523
  float& uncor_error,
1524
  float& samec_error);
1525
1526
/**
1527
 * @brief Compute the RGBA error for uncorrelated and same chroma projections.
1528
 *
1529
 * The output of compute averages and dirs is post processed to define two lines, both of which go
1530
 * through the mean-color-value.  One line has a direction defined by the dominant direction; this
1531
 * is used to assess the error from using an uncorrelated color representation. The other line goes
1532
 * through (0,0,0,1) and is used to assess the error from using an RGBS color representation.
1533
 *
1534
 * This function computes the squared error when using these two representations.
1535
 *
1536
 * @param      pi              The partition info for the current trial.
1537
 * @param      blk             The image block color data to be compressed.
1538
 * @param      uncor_plines    Processed uncorrelated partition lines for each partition.
1539
 * @param      samec_plines    Processed same chroma partition lines for each partition.
1540
 * @param[out] line_lengths    The length of each components deviation from the line.
1541
 * @param[out] uncor_error     The cumulative error for using the uncorrelated line.
1542
 * @param[out] samec_error     The cumulative error for using the same chroma line.
1543
 */
1544
void compute_error_squared_rgba(
1545
  const partition_info& pi,
1546
  const image_block& blk,
1547
  const processed_line4 uncor_plines[BLOCK_MAX_PARTITIONS],
1548
  const processed_line4 samec_plines[BLOCK_MAX_PARTITIONS],
1549
  float line_lengths[BLOCK_MAX_PARTITIONS],
1550
  float& uncor_error,
1551
  float& samec_error);
1552
1553
/**
1554
 * @brief Find the best set of partitions to trial for a given block.
1555
 *
1556
 * On return the @c best_partitions list will contain the two best partition
1557
 * candidates; one assuming data has uncorrelated chroma and one assuming the
1558
 * data has correlated chroma. The best candidate is returned first in the list.
1559
 *
1560
 * @param      bsd                      The block size information.
1561
 * @param      blk                      The image block color data to compress.
1562
 * @param      partition_count          The number of partitions in the block.
1563
 * @param      partition_search_limit   The number of candidate partition encodings to trial.
1564
 * @param[out] best_partitions          The best partition candidates.
1565
 * @param      requested_candidates     The number of requested partitionings. May return fewer if
1566
 *                                      candidates are not available.
1567
 *
1568
 * @return The actual number of candidates returned.
1569
 */
1570
unsigned int find_best_partition_candidates(
1571
  const block_size_descriptor& bsd,
1572
  const image_block& blk,
1573
  unsigned int partition_count,
1574
  unsigned int partition_search_limit,
1575
  unsigned int best_partitions[TUNE_MAX_PARTITIONING_CANDIDATES],
1576
  unsigned int requested_candidates);
1577
1578
/* ============================================================================
1579
  Functionality for managing images and image related data.
1580
============================================================================ */
1581
1582
/**
1583
 * @brief Get a vector mask indicating lanes decompressing into a UNORM8 value.
1584
 *
1585
 * @param decode_mode   The color profile for LDR_SRGB settings.
1586
 * @param blk           The image block for output image bitness settings.
1587
 *
1588
 * @return The component mask vector.
1589
 */
1590
static inline vmask4 get_u8_component_mask(
1591
  astcenc_profile decode_mode,
1592
  const image_block& blk
1593
2.88k
) {
1594
  // Decode mode or sRGB forces writing to unorm8 output value
1595
2.88k
  if (blk.decode_unorm8 || decode_mode == ASTCENC_PRF_LDR_SRGB)
1596
2.88k
  {
1597
2.88k
    return vmask4(true);
1598
2.88k
  }
1599
1600
0
  return vmask4(false);
1601
2.88k
}
Unexecuted instantiation: astcenc_entry.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_image.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_percentile_tables.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_symbolic_physical.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_weight_align.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_weight_quant_xfer_tables.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_block_sizes.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_color_unquantize.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_compress_symbolic.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_compute_variance.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
astcenc_decompress_symbolic.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Line
Count
Source
1593
2.88k
) {
1594
  // Decode mode or sRGB forces writing to unorm8 output value
1595
2.88k
  if (blk.decode_unorm8 || decode_mode == ASTCENC_PRF_LDR_SRGB)
1596
2.88k
  {
1597
2.88k
    return vmask4(true);
1598
2.88k
  }
1599
1600
0
  return vmask4(false);
1601
2.88k
}
Unexecuted instantiation: astcenc_find_best_partitioning.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_ideal_endpoints_and_weights.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_integer_sequence.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_partition_tables.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_pick_best_endpoint_format.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_quantization.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_averages_and_directions.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
Unexecuted instantiation: astcenc_color_quantize.cpp:get_u8_component_mask(astcenc_profile, image_block const&)
1602
1603
/**
1604
 * @brief Setup computation of regional averages in an image.
1605
 *
1606
 * This must be done by only a single thread per image, before any thread calls
1607
 * @c compute_averages().
1608
 *
1609
 * Results are written back into @c img->input_alpha_averages.
1610
 *
1611
 * @param      img                   The input image data, also holds output data.
1612
 * @param      alpha_kernel_radius   The kernel radius (in pixels) for alpha mods.
1613
 * @param      swz                   Input data component swizzle.
1614
 * @param[out] ag                    The average variance arguments to init.
1615
 *
1616
 * @return The number of tasks in the processing stage.
1617
 */
1618
unsigned int init_compute_averages(
1619
  const astcenc_image& img,
1620
  unsigned int alpha_kernel_radius,
1621
  const astcenc_swizzle& swz,
1622
  avg_args& ag);
1623
1624
/**
1625
 * @brief Compute averages for a pixel region.
1626
 *
1627
 * The routine computes both in a single pass, using a summed-area table to decouple the running
1628
 * time from the averaging/variance kernel size.
1629
 *
1630
 * @param[out] ctx   The compressor context storing the output data.
1631
 * @param      arg   The input parameter structure.
1632
 */
1633
void compute_pixel_region_variance(
1634
  astcenc_contexti& ctx,
1635
  const pixel_region_args& arg);
1636
/**
1637
 * @brief Load a single image block from the input image.
1638
 *
1639
 * @param      decode_mode   The compression color profile.
1640
 * @param      img           The input image data.
1641
 * @param[out] blk           The image block to populate.
1642
 * @param      bsd           The block size information.
1643
 * @param      xpos          The block X coordinate in the input image.
1644
 * @param      ypos          The block Y coordinate in the input image.
1645
 * @param      zpos          The block Z coordinate in the input image.
1646
 * @param      swz           The swizzle to apply on load.
1647
 */
1648
void load_image_block(
1649
  astcenc_profile decode_mode,
1650
  const astcenc_image& img,
1651
  image_block& blk,
1652
  const block_size_descriptor& bsd,
1653
  unsigned int xpos,
1654
  unsigned int ypos,
1655
  unsigned int zpos,
1656
  const astcenc_swizzle& swz);
1657
1658
/**
1659
 * @brief Load a single image block from the input image.
1660
 *
1661
 * This specialized variant can be used only if the block is 2D LDR U8 data,
1662
 * with no swizzle.
1663
 *
1664
 * @param      decode_mode   The compression color profile.
1665
 * @param      img           The input image data.
1666
 * @param[out] blk           The image block to populate.
1667
 * @param      bsd           The block size information.
1668
 * @param      xpos          The block X coordinate in the input image.
1669
 * @param      ypos          The block Y coordinate in the input image.
1670
 * @param      zpos          The block Z coordinate in the input image.
1671
 * @param      swz           The swizzle to apply on load.
1672
 */
1673
void load_image_block_fast_ldr(
1674
  astcenc_profile decode_mode,
1675
  const astcenc_image& img,
1676
  image_block& blk,
1677
  const block_size_descriptor& bsd,
1678
  unsigned int xpos,
1679
  unsigned int ypos,
1680
  unsigned int zpos,
1681
  const astcenc_swizzle& swz);
1682
1683
/**
1684
 * @brief Store a single image block to the output image.
1685
 *
1686
 * @param[out] img    The output image data.
1687
 * @param      blk    The image block to export.
1688
 * @param      bsd    The block size information.
1689
 * @param      xpos   The block X coordinate in the input image.
1690
 * @param      ypos   The block Y coordinate in the input image.
1691
 * @param      zpos   The block Z coordinate in the input image.
1692
 * @param      swz    The swizzle to apply on store.
1693
 */
1694
void store_image_block(
1695
  astcenc_image& img,
1696
  const image_block& blk,
1697
  const block_size_descriptor& bsd,
1698
  unsigned int xpos,
1699
  unsigned int ypos,
1700
  unsigned int zpos,
1701
  const astcenc_swizzle& swz);
1702
1703
/* ============================================================================
1704
  Functionality for computing endpoint colors and weights for a block.
1705
============================================================================ */
1706
1707
/**
1708
 * @brief Compute ideal endpoint colors and weights for 1 plane of weights.
1709
 *
1710
 * The ideal endpoints define a color line for the partition. For each texel the ideal weight
1711
 * defines an exact position on the partition color line. We can then use these to assess the error
1712
 * introduced by removing and quantizing the weight grid.
1713
 *
1714
 * @param      blk   The image block color data to compress.
1715
 * @param      pi    The partition info for the current trial.
1716
 * @param[out] ei    The endpoint and weight values.
1717
 */
1718
void compute_ideal_colors_and_weights_1plane(
1719
  const image_block& blk,
1720
  const partition_info& pi,
1721
  endpoints_and_weights& ei);
1722
1723
/**
1724
 * @brief Compute ideal endpoint colors and weights for 2 planes of weights.
1725
 *
1726
 * The ideal endpoints define a color line for the partition. For each texel the ideal weight
1727
 * defines an exact position on the partition color line. We can then use these to assess the error
1728
 * introduced by removing and quantizing the weight grid.
1729
 *
1730
 * @param      bsd                The block size information.
1731
 * @param      blk                The image block color data to compress.
1732
 * @param      plane2_component   The component assigned to plane 2.
1733
 * @param[out] ei1                The endpoint and weight values for plane 1.
1734
 * @param[out] ei2                The endpoint and weight values for plane 2.
1735
 */
1736
void compute_ideal_colors_and_weights_2planes(
1737
  const block_size_descriptor& bsd,
1738
  const image_block& blk,
1739
  unsigned int plane2_component,
1740
  endpoints_and_weights& ei1,
1741
  endpoints_and_weights& ei2);
1742
1743
/**
1744
 * @brief Compute the optimal unquantized weights for a decimation table.
1745
 *
1746
 * After computing ideal weights for the case for a complete weight grid, we we want to compute the
1747
 * ideal weights for the case where weights exist only for some texels. We do this with a
1748
 * steepest-descent grid solver which works as follows:
1749
 *
1750
 * First, for each actual weight, perform a weighted averaging of the texels affected by the weight.
1751
 * Then, set step size to <some initial value> and attempt one step towards the original ideal
1752
 * weight if it helps to reduce error.
1753
 *
1754
 * @param      ei                       The non-decimated endpoints and weights.
1755
 * @param      di                       The selected weight decimation.
1756
 * @param[out] dec_weight_ideal_value   The ideal values for the decimated weight set.
1757
 */
1758
void compute_ideal_weights_for_decimation(
1759
  const endpoints_and_weights& ei,
1760
  const decimation_info& di,
1761
  float* dec_weight_ideal_value);
1762
1763
/**
1764
 * @brief Compute the optimal quantized weights for a decimation table.
1765
 *
1766
 * We test the two closest weight indices in the allowed quantization range and keep the weight that
1767
 * is the closest match.
1768
 *
1769
 * @param      di                        The selected weight decimation.
1770
 * @param      low_bound                 The lowest weight allowed.
1771
 * @param      high_bound                The highest weight allowed.
1772
 * @param      dec_weight_ideal_value    The ideal weight set.
1773
 * @param[out] dec_weight_quant_uvalue   The output quantized weight as a float.
1774
 * @param[out] dec_weight_uquant         The output quantized weight as encoded int.
1775
 * @param      quant_level               The desired weight quant level.
1776
 */
1777
void compute_quantized_weights_for_decimation(
1778
  const decimation_info& di,
1779
  float low_bound,
1780
  float high_bound,
1781
  const float* dec_weight_ideal_value,
1782
  float* dec_weight_quant_uvalue,
1783
  uint8_t* dec_weight_uquant,
1784
  quant_method quant_level);
1785
1786
/**
1787
 * @brief Compute the error of a decimated weight set for 1 plane.
1788
 *
1789
 * After computing ideal weights for the case with one weight per texel, we want to compute the
1790
 * error for decimated weight grids where weights are stored at a lower resolution. This function
1791
 * computes the error of the reduced grid, compared to the full grid.
1792
 *
1793
 * @param eai                       The ideal weights for the full grid.
1794
 * @param di                        The selected weight decimation.
1795
 * @param dec_weight_quant_uvalue   The quantized weights for the decimated grid.
1796
 *
1797
 * @return The accumulated error.
1798
 */
1799
float compute_error_of_weight_set_1plane(
1800
  const endpoints_and_weights& eai,
1801
  const decimation_info& di,
1802
  const float* dec_weight_quant_uvalue);
1803
1804
/**
1805
 * @brief Compute the error of a decimated weight set for 2 planes.
1806
 *
1807
 * After computing ideal weights for the case with one weight per texel, we want to compute the
1808
 * error for decimated weight grids where weights are stored at a lower resolution. This function
1809
 * computes the error of the reduced grid, compared to the full grid.
1810
 *
1811
 * @param eai1                             The ideal weights for the full grid and plane 1.
1812
 * @param eai2                             The ideal weights for the full grid and plane 2.
1813
 * @param di                               The selected weight decimation.
1814
 * @param dec_weight_quant_uvalue_plane1   The quantized weights for the decimated grid plane 1.
1815
 * @param dec_weight_quant_uvalue_plane2   The quantized weights for the decimated grid plane 2.
1816
 *
1817
 * @return The accumulated error.
1818
 */
1819
float compute_error_of_weight_set_2planes(
1820
  const endpoints_and_weights& eai1,
1821
  const endpoints_and_weights& eai2,
1822
  const decimation_info& di,
1823
  const float* dec_weight_quant_uvalue_plane1,
1824
  const float* dec_weight_quant_uvalue_plane2);
1825
1826
/**
1827
 * @brief Pack a single pair of color endpoints as effectively as possible.
1828
 *
1829
 * The user requests a base color endpoint mode in @c format, but the quantizer may choose a
1830
 * delta-based representation. It will report back the format variant it actually used.
1831
 *
1832
 * @param      color0        The input unquantized color0 endpoint for absolute endpoint pairs.
1833
 * @param      color1        The input unquantized color1 endpoint for absolute endpoint pairs.
1834
 * @param      rgbs_color    The input unquantized RGBS variant endpoint for same chroma endpoints.
1835
 * @param      rgbo_color    The input unquantized RGBS variant endpoint for HDR endpoints.
1836
 * @param      format        The desired base format.
1837
 * @param[out] output        The output storage for the quantized colors/
1838
 * @param      quant_level   The quantization level requested.
1839
 *
1840
 * @return The actual endpoint mode used.
1841
 */
1842
uint8_t pack_color_endpoints(
1843
  vfloat4 color0,
1844
  vfloat4 color1,
1845
  vfloat4 rgbs_color,
1846
  vfloat4 rgbo_color,
1847
  int format,
1848
  uint8_t* output,
1849
  quant_method quant_level);
1850
1851
/**
1852
 * @brief Unpack a single pair of encoded endpoints.
1853
 *
1854
 * Endpoints must be unscrambled and converted into the 0-255 range before calling this functions.
1855
 *
1856
 * @param      decode_mode   The decode mode (LDR, HDR, etc).
1857
 * @param      format        The color endpoint mode used.
1858
 * @param      input         The raw array of encoded input integers. The length of this array
1859
 *                           depends on @c format; it can be safely assumed to be large enough.
1860
 * @param[out] rgb_hdr       Is the endpoint using HDR for the RGB channels?
1861
 * @param[out] alpha_hdr     Is the endpoint using HDR for the A channel?
1862
 * @param[out] output0       The output color for endpoint 0.
1863
 * @param[out] output1       The output color for endpoint 1.
1864
 */
1865
void unpack_color_endpoints(
1866
  astcenc_profile decode_mode,
1867
  int format,
1868
  const uint8_t* input,
1869
  bool& rgb_hdr,
1870
  bool& alpha_hdr,
1871
  vint4& output0,
1872
  vint4& output1);
1873
1874
/**
1875
 * @brief Unpack an LDR RGBA color that uses delta encoding.
1876
 *
1877
 * @param      input0    The packed endpoint 0 color.
1878
 * @param      input1    The packed endpoint 1 color deltas.
1879
 * @param[out] output0   The unpacked endpoint 0 color.
1880
 * @param[out] output1   The unpacked endpoint 1 color.
1881
 */
1882
void rgba_delta_unpack(
1883
  vint4 input0,
1884
  vint4 input1,
1885
  vint4& output0,
1886
  vint4& output1);
1887
1888
/**
1889
 * @brief Unpack an LDR RGBA color that uses direct encoding.
1890
 *
1891
 * @param      input0    The packed endpoint 0 color.
1892
 * @param      input1    The packed endpoint 1 color.
1893
 * @param[out] output0   The unpacked endpoint 0 color.
1894
 * @param[out] output1   The unpacked endpoint 1 color.
1895
 */
1896
void rgba_unpack(
1897
  vint4 input0,
1898
  vint4 input1,
1899
  vint4& output0,
1900
  vint4& output1);
1901
1902
/**
1903
 * @brief Unpack a set of quantized and decimated weights.
1904
 *
1905
 * TODO: Can we skip this for non-decimated weights now that the @c scb is
1906
 * already storing unquantized weights?
1907
 *
1908
 * @param      bsd              The block size information.
1909
 * @param      scb              The symbolic compressed encoding.
1910
 * @param      di               The weight grid decimation table.
1911
 * @param      is_dual_plane    @c true if this is a dual plane block, @c false otherwise.
1912
 * @param[out] weights_plane1   The output array for storing the plane 1 weights.
1913
 * @param[out] weights_plane2   The output array for storing the plane 2 weights.
1914
 */
1915
void unpack_weights(
1916
  const block_size_descriptor& bsd,
1917
  const symbolic_compressed_block& scb,
1918
  const decimation_info& di,
1919
  bool is_dual_plane,
1920
  int weights_plane1[BLOCK_MAX_TEXELS],
1921
  int weights_plane2[BLOCK_MAX_TEXELS]);
1922
1923
/**
1924
 * @brief Identify, for each mode, which set of color endpoint produces the best result.
1925
 *
1926
 * Returns the best @c tune_candidate_limit best looking modes, along with the ideal color encoding
1927
 * combination for each. The modified quantization level can be used when all formats are the same,
1928
 * as this frees up two additional bits of storage.
1929
 *
1930
 * @param      pi                            The partition info for the current trial.
1931
 * @param      blk                           The image block color data to compress.
1932
 * @param      ep                            The ideal endpoints.
1933
 * @param      qwt_bitcounts                 Bit counts for different quantization methods.
1934
 * @param      qwt_errors                    Errors for different quantization methods.
1935
 * @param      tune_candidate_limit          The max number of candidates to return, may be less.
1936
 * @param      start_block_mode              The first block mode to inspect.
1937
 * @param      end_block_mode                The last block mode to inspect.
1938
 * @param[out] partition_format_specifiers   The best formats per partition.
1939
 * @param[out] block_mode                    The best packed block mode indexes.
1940
 * @param[out] quant_level                   The best color quant level.
1941
 * @param[out] quant_level_mod               The best color quant level if endpoints are the same.
1942
 * @param[out] tmpbuf                        Preallocated scratch buffers for the compressor.
1943
 *
1944
 * @return The actual number of candidate matches returned.
1945
 */
1946
unsigned int compute_ideal_endpoint_formats(
1947
  const partition_info& pi,
1948
  const image_block& blk,
1949
  const endpoints& ep,
1950
  const int8_t* qwt_bitcounts,
1951
  const float* qwt_errors,
1952
  unsigned int tune_candidate_limit,
1953
  unsigned int start_block_mode,
1954
  unsigned int end_block_mode,
1955
  uint8_t partition_format_specifiers[TUNE_MAX_TRIAL_CANDIDATES][BLOCK_MAX_PARTITIONS],
1956
  int block_mode[TUNE_MAX_TRIAL_CANDIDATES],
1957
  quant_method quant_level[TUNE_MAX_TRIAL_CANDIDATES],
1958
  quant_method quant_level_mod[TUNE_MAX_TRIAL_CANDIDATES],
1959
  compression_working_buffers& tmpbuf);
1960
1961
/**
1962
 * @brief For a given 1 plane weight set recompute the endpoint colors.
1963
 *
1964
 * As we quantize and decimate weights the optimal endpoint colors may change slightly, so we must
1965
 * recompute the ideal colors for a specific weight set.
1966
 *
1967
 * @param         blk                  The image block color data to compress.
1968
 * @param         pi                   The partition info for the current trial.
1969
 * @param         di                   The weight grid decimation table.
1970
 * @param         dec_weights_uquant   The quantized weight set.
1971
 * @param[in,out] ep                   The color endpoints (modifed in place).
1972
 * @param[out]    rgbs_vectors         The RGB+scale vectors for LDR blocks.
1973
 * @param[out]    rgbo_vectors         The RGB+offset vectors for HDR blocks.
1974
 */
1975
void recompute_ideal_colors_1plane(
1976
  const image_block& blk,
1977
  const partition_info& pi,
1978
  const decimation_info& di,
1979
  const uint8_t* dec_weights_uquant,
1980
  endpoints& ep,
1981
  vfloat4 rgbs_vectors[BLOCK_MAX_PARTITIONS],
1982
  vfloat4 rgbo_vectors[BLOCK_MAX_PARTITIONS]);
1983
1984
/**
1985
 * @brief For a given 2 plane weight set recompute the endpoint colors.
1986
 *
1987
 * As we quantize and decimate weights the optimal endpoint colors may change slightly, so we must
1988
 * recompute the ideal colors for a specific weight set.
1989
 *
1990
 * @param         blk                         The image block color data to compress.
1991
 * @param         bsd                         The block_size descriptor.
1992
 * @param         di                          The weight grid decimation table.
1993
 * @param         dec_weights_uquant_plane1   The quantized weight set for plane 1.
1994
 * @param         dec_weights_uquant_plane2   The quantized weight set for plane 2.
1995
 * @param[in,out] ep                          The color endpoints (modifed in place).
1996
 * @param[out]    rgbs_vector                 The RGB+scale color for LDR blocks.
1997
 * @param[out]    rgbo_vector                 The RGB+offset color for HDR blocks.
1998
 * @param         plane2_component            The component assigned to plane 2.
1999
 */
2000
void recompute_ideal_colors_2planes(
2001
  const image_block& blk,
2002
  const block_size_descriptor& bsd,
2003
  const decimation_info& di,
2004
  const uint8_t* dec_weights_uquant_plane1,
2005
  const uint8_t* dec_weights_uquant_plane2,
2006
  endpoints& ep,
2007
  vfloat4& rgbs_vector,
2008
  vfloat4& rgbo_vector,
2009
  int plane2_component);
2010
2011
/**
2012
 * @brief Expand the angular tables needed for the alternative to PCA that we use.
2013
 */
2014
void prepare_angular_tables();
2015
2016
/**
2017
 * @brief Compute the angular endpoints for one plane for each block mode.
2018
 *
2019
 * @param      only_always              Only consider block modes that are always enabled.
2020
 * @param      bsd                      The block size descriptor for the current trial.
2021
 * @param      dec_weight_ideal_value   The ideal decimated unquantized weight values.
2022
 * @param      max_weight_quant         The maximum block mode weight quantization allowed.
2023
 * @param[out] tmpbuf                   Preallocated scratch buffers for the compressor.
2024
 */
2025
void compute_angular_endpoints_1plane(
2026
  bool only_always,
2027
  const block_size_descriptor& bsd,
2028
  const float* dec_weight_ideal_value,
2029
  unsigned int max_weight_quant,
2030
  compression_working_buffers& tmpbuf);
2031
2032
/**
2033
 * @brief Compute the angular endpoints for two planes for each block mode.
2034
 *
2035
 * @param      bsd                      The block size descriptor for the current trial.
2036
 * @param      dec_weight_ideal_value   The ideal decimated unquantized weight values.
2037
 * @param      max_weight_quant         The maximum block mode weight quantization allowed.
2038
 * @param[out] tmpbuf                   Preallocated scratch buffers for the compressor.
2039
 */
2040
void compute_angular_endpoints_2planes(
2041
  const block_size_descriptor& bsd,
2042
  const float* dec_weight_ideal_value,
2043
  unsigned int max_weight_quant,
2044
  compression_working_buffers& tmpbuf);
2045
2046
/* ============================================================================
2047
  Functionality for high level compression and decompression access.
2048
============================================================================ */
2049
2050
/**
2051
 * @brief Compress an image block into a physical block.
2052
 *
2053
 * @param      ctx      The compressor context and configuration.
2054
 * @param      blk      The image block color data to compress.
2055
 * @param[out] pcb      The physical compressed block output.
2056
 * @param[out] tmpbuf   Preallocated scratch buffers for the compressor.
2057
 */
2058
void compress_block(
2059
  const astcenc_contexti& ctx,
2060
  const image_block& blk,
2061
  uint8_t pcb[16],
2062
  compression_working_buffers& tmpbuf);
2063
2064
/**
2065
 * @brief Decompress a symbolic block in to an image block.
2066
 *
2067
 * @param      decode_mode   The decode mode (LDR, HDR, etc).
2068
 * @param      bsd           The block size information.
2069
 * @param      xpos          The X coordinate of the block in the overall image.
2070
 * @param      ypos          The Y coordinate of the block in the overall image.
2071
 * @param      zpos          The Z coordinate of the block in the overall image.
2072
 * @param[out] blk           The decompressed image block color data.
2073
 */
2074
void decompress_symbolic_block(
2075
  astcenc_profile decode_mode,
2076
  const block_size_descriptor& bsd,
2077
  int xpos,
2078
  int ypos,
2079
  int zpos,
2080
  const symbolic_compressed_block& scb,
2081
  image_block& blk);
2082
2083
/**
2084
 * @brief Compute the error between a symbolic block and the original input data.
2085
 *
2086
 * This function is specialized for 2 plane and 1 partition search.
2087
 *
2088
 * In RGBM mode this will reject blocks that attempt to encode a zero M value.
2089
 *
2090
 * @param config   The compressor config.
2091
 * @param bsd      The block size information.
2092
 * @param scb      The symbolic compressed encoding.
2093
 * @param blk      The original image block color data.
2094
 *
2095
 * @return Returns the computed error, or a negative value if the encoding
2096
 *         should be rejected for any reason.
2097
 */
2098
float compute_symbolic_block_difference_2plane(
2099
  const astcenc_config& config,
2100
  const block_size_descriptor& bsd,
2101
  const symbolic_compressed_block& scb,
2102
  const image_block& blk);
2103
2104
/**
2105
 * @brief Compute the error between a symbolic block and the original input data.
2106
 *
2107
 * This function is specialized for 1 plane and N partition search.
2108
 *
2109
 * In RGBM mode this will reject blocks that attempt to encode a zero M value.
2110
 *
2111
 * @param config   The compressor config.
2112
 * @param bsd      The block size information.
2113
 * @param scb      The symbolic compressed encoding.
2114
 * @param blk      The original image block color data.
2115
 *
2116
 * @return Returns the computed error, or a negative value if the encoding
2117
 *         should be rejected for any reason.
2118
 */
2119
float compute_symbolic_block_difference_1plane(
2120
  const astcenc_config& config,
2121
  const block_size_descriptor& bsd,
2122
  const symbolic_compressed_block& scb,
2123
  const image_block& blk);
2124
2125
/**
2126
 * @brief Compute the error between a symbolic block and the original input data.
2127
 *
2128
 * This function is specialized for 1 plane and 1 partition search.
2129
 *
2130
 * In RGBM mode this will reject blocks that attempt to encode a zero M value.
2131
 *
2132
 * @param config   The compressor config.
2133
 * @param bsd      The block size information.
2134
 * @param scb      The symbolic compressed encoding.
2135
 * @param blk      The original image block color data.
2136
 *
2137
 * @return Returns the computed error, or a negative value if the encoding
2138
 *         should be rejected for any reason.
2139
 */
2140
float compute_symbolic_block_difference_1plane_1partition(
2141
  const astcenc_config& config,
2142
  const block_size_descriptor& bsd,
2143
  const symbolic_compressed_block& scb,
2144
  const image_block& blk);
2145
2146
/**
2147
 * @brief Convert a symbolic representation into a binary physical encoding.
2148
 *
2149
 * It is assumed that the symbolic encoding is valid and encodable, or
2150
 * previously flagged as an error block if an error color it to be encoded.
2151
 *
2152
 * @param      bsd   The block size information.
2153
 * @param      scb   The symbolic representation.
2154
 * @param[out] pcb   The physical compressed block output.
2155
 */
2156
void symbolic_to_physical(
2157
  const block_size_descriptor& bsd,
2158
  const symbolic_compressed_block& scb,
2159
  uint8_t pcb[16]);
2160
2161
/**
2162
 * @brief Convert a binary physical encoding into a symbolic representation.
2163
 *
2164
 * This function can cope with arbitrary input data; output blocks will be
2165
 * flagged as an error block if the encoding is invalid.
2166
 *
2167
 * @param      bsd   The block size information.
2168
 * @param      pcb   The physical compresesd block input.
2169
 * @param[out] scb   The output symbolic representation.
2170
 */
2171
void physical_to_symbolic(
2172
  const block_size_descriptor& bsd,
2173
  const uint8_t pcb[16],
2174
  symbolic_compressed_block& scb);
2175
2176
/* ============================================================================
2177
Platform-specific functions.
2178
============================================================================ */
2179
/**
2180
 * @brief Allocate an aligned memory buffer.
2181
 *
2182
 * Allocated memory must be freed by aligned_free.
2183
 *
2184
 * @param size    The desired buffer size.
2185
 * @param align   The desired buffer alignment; must be 2^N, may be increased
2186
 *                by the implementation to a minimum allowable alignment.
2187
 *
2188
 * @return The memory buffer pointer or nullptr on allocation failure.
2189
 */
2190
template<typename T>
2191
T* aligned_malloc(size_t size, size_t align)
2192
2.13k
{
2193
2.13k
  void* ptr;
2194
2.13k
  int error = 0;
2195
2196
  // Don't allow this to under-align a type
2197
2.13k
  size_t min_align = astc::max(alignof(T), sizeof(void*));
2198
2.13k
  size_t real_align = astc::max(min_align, align);
2199
2200
#if defined(_WIN32)
2201
  ptr = _aligned_malloc(size, real_align);
2202
#else
2203
2.13k
  error = posix_memalign(&ptr, real_align, size);
2204
2.13k
#endif
2205
2206
2.13k
  if (error || (!ptr))
2207
0
  {
2208
0
    return nullptr;
2209
0
  }
2210
2211
2.13k
  return static_cast<T*>(ptr);
2212
2.13k
}
block_size_descriptor* aligned_malloc<block_size_descriptor>(unsigned long, unsigned long)
Line
Count
Source
2192
1.21k
{
2193
1.21k
  void* ptr;
2194
1.21k
  int error = 0;
2195
2196
  // Don't allow this to under-align a type
2197
1.21k
  size_t min_align = astc::max(alignof(T), sizeof(void*));
2198
1.21k
  size_t real_align = astc::max(min_align, align);
2199
2200
#if defined(_WIN32)
2201
  ptr = _aligned_malloc(size, real_align);
2202
#else
2203
1.21k
  error = posix_memalign(&ptr, real_align, size);
2204
1.21k
#endif
2205
2206
1.21k
  if (error || (!ptr))
2207
0
  {
2208
0
    return nullptr;
2209
0
  }
2210
2211
1.21k
  return static_cast<T*>(ptr);
2212
1.21k
}
compression_working_buffers* aligned_malloc<compression_working_buffers>(unsigned long, unsigned long)
Line
Count
Source
2192
911
{
2193
911
  void* ptr;
2194
911
  int error = 0;
2195
2196
  // Don't allow this to under-align a type
2197
911
  size_t min_align = astc::max(alignof(T), sizeof(void*));
2198
911
  size_t real_align = astc::max(min_align, align);
2199
2200
#if defined(_WIN32)
2201
  ptr = _aligned_malloc(size, real_align);
2202
#else
2203
911
  error = posix_memalign(&ptr, real_align, size);
2204
911
#endif
2205
2206
911
  if (error || (!ptr))
2207
0
  {
2208
0
    return nullptr;
2209
0
  }
2210
2211
911
  return static_cast<T*>(ptr);
2212
911
}
2213
2214
/**
2215
 * @brief Free an aligned memory buffer.
2216
 *
2217
 * @param ptr   The buffer to free.
2218
 */
2219
template<typename T>
2220
void aligned_free(T* ptr)
2221
2.43k
{
2222
#if defined(_WIN32)
2223
  _aligned_free(const_cast<typename std::remove_const<T>::type *>(ptr));
2224
#else
2225
2.43k
  free(const_cast<typename std::remove_const<T>::type *>(ptr));
2226
2.43k
#endif
2227
2.43k
}
void aligned_free<block_size_descriptor const>(block_size_descriptor const*)
Line
Count
Source
2221
1.21k
{
2222
#if defined(_WIN32)
2223
  _aligned_free(const_cast<typename std::remove_const<T>::type *>(ptr));
2224
#else
2225
1.21k
  free(const_cast<typename std::remove_const<T>::type *>(ptr));
2226
1.21k
#endif
2227
1.21k
}
void aligned_free<compression_working_buffers>(compression_working_buffers*)
Line
Count
Source
2221
1.21k
{
2222
#if defined(_WIN32)
2223
  _aligned_free(const_cast<typename std::remove_const<T>::type *>(ptr));
2224
#else
2225
1.21k
  free(const_cast<typename std::remove_const<T>::type *>(ptr));
2226
1.21k
#endif
2227
1.21k
}
2228
2229
#endif