Coverage Report

Created: 2026-07-25 07:06

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/lerc/src/LercLib/Lerc2.h
Line
Count
Source
1
/*
2
Copyright 2015 - 2026 Esri
3
4
Licensed under the Apache License, Version 2.0 (the "License");
5
you may not use this file except in compliance with the License.
6
You may obtain a copy of the License at
7
8
http://www.apache.org/licenses/LICENSE-2.0
9
10
Unless required by applicable law or agreed to in writing, software
11
distributed under the License is distributed on an "AS IS" BASIS,
12
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
See the License for the specific language governing permissions and
14
limitations under the License.
15
16
A local copy of the license and additional notices are located with the
17
source distribution at:
18
19
http://github.com/Esri/lerc/
20
21
Contributors:  Thomas Maurer
22
*/
23
24
#ifndef LERC2_H
25
#define LERC2_H
26
27
#include <cfloat>
28
#include <cmath>
29
#include <climits>
30
#include <algorithm>
31
#include <string>
32
#include "BitMask.h"
33
#include "BitStuffer2.h"
34
#include "fpl_Lerc2Ext.h"
35
36
NAMESPACE_LERC_START
37
38
/**   Lerc2 v1
39
 *
40
 *    -- allow for lossless compression of all common data types
41
 *    -- avoid data type conversions and copies
42
 *    -- optimized compression for segmented rasters (10-15x lossless)
43
 *    -- micro block is 8x8 fixed, only gets doubled to 16x16 if bit rate < 1 bpp
44
 *    -- cnt is replaced by bit mask
45
 *    -- Lerc blob header has data range [min, max]
46
 *    -- harden consistency checks to detect if the byte blob has been tampered with
47
 *    -- drop support for big endian, this is legacy now
48
 *
49
 *    Lerc2 v2
50
 *
51
 *    -- add Huffman coding for better lossless compression of 8 bit data types Char, Byte
52
 *
53
 *    Lerc2 v3
54
 *
55
 *    -- add checksum for the entire byte blob, for more rigorous detection of compressed data corruption
56
 *    -- for the main bit stuffing routine, use an extra uint buffer for guaranteed memory alignment
57
 *    -- this also allows to drop the NumExtraBytesToAllocate functions
58
 *
59
 *    Lerc2 v4
60
 *
61
 *    -- allow array per pixel, nDepth values per pixel. Such as RGB, complex number, or larger arrays per pixel
62
 *    -- extend Huffman coding for 8 bit data types from delta only to trying both delta and orig
63
 *    -- for integer data types, allow to drop bit planes containing only random noise
64
 *
65
 *    Lerc2 v5
66
 *    -- for float data (as it might be lower precision like %.2f), try raise maxZError if possible w/o extra loss
67
 *    -- add delta encoding of a block iDepth relative to previous block (iDepth - 1)
68
 *
69
 */
70
71
class Lerc2
72
{
73
public:
74
  Lerc2();
75
  Lerc2(int nDepth, int nCols, int nRows, const Byte* pMaskBits = nullptr);    // valid / invalid bits as byte array
76
0
  ~Lerc2()  {}
77
78
0
  static int CurrentVersion() { return 6; }
79
80
  bool SetEncoderToOldVersion(int version);    // call this to encode compatible to an old decoder
81
82
  bool Set(int nDepth, int nCols, int nRows, const Byte* pMaskBits = nullptr);     // set mask and dimensions
83
84
  // v6
85
  bool SetNoDataValues(bool bNeedsNoDataVal, double noDataVal, double noDataValOrig);
86
  bool SetNumBlobsMoreToCome(int nBlobsMore);    // for safer decode of multi-band blobs
87
  bool SetIsAllInt(bool bIsAllInt);
88
89
  bool SetMinMax(int nDepth, double minVal, double maxVal);    // set min / max but only for nDepth = 1
90
  void ClearMinMax();
91
92
  template<class T>
93
  unsigned int ComputeNumBytesNeededToWrite(const T* arr, double maxZError, bool encodeMask);
94
95
  /// dst buffer already allocated;  byte ptr is moved like a file pointer
96
  template<class T>
97
  bool Encode(const T* arr, Byte** ppByte);
98
99
  // data types supported by Lerc2
100
  enum DataType {DT_Char = 0, DT_Byte, DT_Short, DT_UShort, DT_Int, DT_UInt, DT_Float, DT_Double, DT_Undefined};
101
102
  struct HeaderInfo
103
  {
104
    int version;
105
    unsigned int checksum;
106
    int nRows,
107
      nCols,
108
      nDepth,
109
      numValidPixel,
110
      microBlockSize,
111
      blobSize,
112
      nBlobsMore;    // for multi-band Lerc blob, how many more blobs or bands appended to this one
113
114
    Byte bPassNoDataValues,  // 1 - pass noData values to decoder, 0 - don't pass, ignore
115
      bIsInt,    // 1 - float or double data is all integer numbers, 0 - not
116
      bReserved3,
117
      bReserved4;
118
119
    DataType dt;
120
121
    double maxZError,
122
      zMin,    // if nDepth > 1, this is the overall range
123
      zMax,
124
      noDataVal,      // temp noData value used for nDepth > 1 if bit mask cannot cover it
125
      noDataValOrig;  // orig noData value to map to in decode
126
127
0
    void RawInit()  { memset(this, 0, sizeof(struct HeaderInfo)); }
128
129
0
    bool TryHuffmanInt() const { return (version >= 2) && (dt == DT_Byte || dt == DT_Char) && (maxZError == 0.5); }
130
0
    bool TryHuffmanFlt() const { return (version >= 6) && (dt == DT_Float || dt == DT_Double) && (maxZError == 0); }
131
  };
132
133
  static bool GetHeaderInfo(const Byte* pByte, size_t nBytesRemaining, struct HeaderInfo& headerInfo, bool& bHasMask);
134
135
  bool GetRanges(const Byte* pByte, size_t nBytesRemaining, double* pMins, double* pMaxs);
136
137
  /// dst buffer already allocated;  byte ptr is moved like a file pointer
138
  template<class T>
139
  bool Decode(const Byte** ppByte, size_t& nBytesRemaining, T* arr, Byte* pMaskBits = nullptr);    // if mask ptr is not 0, mask bits are returned (even if all valid or same as previous)
140
141
private:
142
143
  enum ImageEncodeMode { IEM_Tiling = 0, IEM_DeltaHuffman, IEM_Huffman, IEM_DeltaDeltaHuffman };
144
  enum BlockEncodeMode { BEM_RawBinary = 0, BEM_BitStuffSimple, BEM_BitStuffLUT };
145
146
  int         m_microBlockSize,
147
              m_maxValToQuantize;
148
  BitMask     m_bitMask;
149
  HeaderInfo  m_headerInfo;
150
  BitStuffer2 m_bitStuffer2;
151
  bool        m_encodeMask,
152
              m_writeDataOneSweep,
153
              m_minMaxSet;
154
  ImageEncodeMode  m_imageEncodeMode;
155
156
  std::vector<double> m_zMinVec, m_zMaxVec;
157
  std::vector<std::pair<unsigned short, unsigned int> > m_huffmanCodes;    // <= 256 codes, 1.5 kB
158
159
  LosslessFPCompression m_lfpc;
160
161
private:
162
0
  static std::string FileKey()  { return "Lerc2 "; }
163
0
  static bool IsLittleEndianSystem()  { int n = 1;  return (1 == *((Byte*)&n)) && (4 == sizeof(int)); }
164
  void Init();
165
166
  static unsigned int ComputeNumBytesHeaderToWrite(const struct HeaderInfo& hd);
167
  static bool WriteHeader(Byte** ppByte, const struct HeaderInfo& hd);
168
  static bool ReadHeader(const Byte** ppByte, size_t& nBytesRemaining, struct HeaderInfo& hd);
169
170
  bool WriteMask(Byte** ppByte) const;
171
  bool ReadMask(const Byte** ppByte, size_t& nBytesRemaining);
172
173
  bool DoChecksOnEncode(Byte* pBlobBegin, Byte* pBlobEnd) const;
174
  static unsigned int ComputeChecksumFletcher32(const Byte* pByte, int len);
175
176
  static void AddUIntToCounts(int* pCounts, unsigned int val, int nBits);
177
  static void AddIntToCounts(int* pCounts, int val, int nBits);
178
179
  template<class T>
180
  bool TryBitPlaneCompression(const T* data, double eps, double& newMaxZError) const;
181
182
  template<class T>
183
  bool TryRaiseMaxZError(const T* data, double& maxZError) const;
184
185
  static bool PruneCandidates(std::vector<double>& roundErr, std::vector<double>& zErr,
186
    std::vector<int>& zFac, double maxZError);
187
188
  template<class T>
189
  bool WriteDataOneSweep(const T* data, Byte** ppByte) const;
190
191
  template<class T>
192
  bool ReadDataOneSweep(const Byte** ppByte, size_t& nBytesRemaining, T* data) const;
193
194
  template<class T>
195
  bool ComputeMinMaxRanges(const T* data, std::vector<double>& zMinVec, std::vector<double>& zMaxVec) const;
196
197
  template<class T>
198
  bool WriteTiles(const T* data, Byte** ppByte, int& numBytes) const;
199
200
  template<class T>
201
  bool ReadTiles(const Byte** ppByte, size_t& nBytesRemaining, T* data) const;
202
203
  template<class T>
204
  bool GetValidDataAndStats(const T* data, int i0, int i1, int j0, int j1, int iDepth,
205
    T* dataBuf, T& zMin, T& zMax, int& numValidPixel, bool& tryLut) const;
206
207
  template<class T>
208
  static bool ComputeDiffSliceInt(const T* data, const T* prevData, int numValidPixel, bool bCheckForIntOverflow,
209
    double maxZError, std::vector<int>& diffDataVec, int& zMin, int& zMax, bool& tryLut);
210
211
  template<class T>
212
  static bool ComputeDiffSliceFlt(const T* data, const T* prevData, int numValidPixel, bool bCheckForFltRndErr,
213
    double maxZError, std::vector<T>& diffDataVec, T& zMin, T& zMax, bool& tryLut);
214
215
  static bool NeedToCheckForIntOverflow(const HeaderInfo& hd);
216
  static bool NeedToCheckForFltRndErr(const HeaderInfo& hd);
217
218
  static double ComputeMaxVal(double zMin, double zMax, double maxZError);
219
220
  template<class T>
221
  bool NeedToQuantize(int numValidPixel, T zMin, T zMax) const;
222
223
  template<class T>
224
  void Quantize(const T* dataBuf, int num, T zMin, std::vector<unsigned int>& quantVec) const;
225
226
  template<class T>
227
  static void ScaleBack(T* dataBuf, const std::vector<unsigned int>& quantVec,
228
    double zMin, bool bDiff, bool bClamp, double zMaxClamp, double maxZError);
229
230
  template<class T>
231
  static void ScaleBackConstBlock(T* dataBuf, int num, double zMin, bool bClamp, double zMaxClamp);
232
233
  template<class T>
234
  int NumBytesTile(int numValidPixel, T zMin, T zMax, DataType dtZ, bool tryLut, BlockEncodeMode& blockEncodeMode,
235
                   const std::vector<std::pair<unsigned int, unsigned int> >& sortedQuantVec) const;
236
237
  template<class T>
238
  bool WriteTile(const T* dataBuf, int num, Byte** ppByte, int& numBytesWritten, int j0, T zMin, T zMax,
239
    DataType dtZ, bool bDiffEnc, const std::vector<unsigned int>& quantVec, BlockEncodeMode blockEncodeMode,
240
    const std::vector<std::pair<unsigned int, unsigned int> >& sortedQuantVec) const;
241
242
  template<class T>
243
  bool ReadTile(const Byte** ppByte, size_t& nBytesRemaining, T* data, int i0, int i1, int j0, int j1, int iDepth,
244
                std::vector<unsigned int>& bufferVec) const;
245
246
  template<class T>
247
  static int ReduceDataType(T z, DataType dt, DataType& dtReduced);
248
249
  static DataType GetDataTypeUsed(DataType dt, int reducedTypeCode);
250
251
  static DataType ValidateDataType(int dt);
252
253
  static bool WriteVariableDataType(Byte** ppByte, double z, DataType dtUsed);
254
255
  static double ReadVariableDataType(const Byte** ppByte, DataType dtUsed);
256
257
  template<class T>
258
  static DataType GetDataType(T z);
259
260
  static unsigned int GetMaxValToQuantize(DataType dt);
261
262
  static unsigned int GetDataTypeSize(DataType dt);
263
264
  static void SortQuantArray(const std::vector<unsigned int>& quantVec,
265
    std::vector<std::pair<unsigned int, unsigned int> >& sortedQuantVec);
266
267
  template<class T>
268
  void ComputeHuffmanCodes(const T* data, int& numBytes, ImageEncodeMode& imageEncodeMode,
269
    std::vector<std::pair<unsigned short, unsigned int> >& codes) const;
270
271
  template<class T>
272
  void ComputeHistoForHuffman(const T* data, std::vector<int>& histo, std::vector<int>& deltaHisto) const;
273
274
  template<class T>
275
  bool EncodeHuffman(const T* data, Byte** ppByte) const;
276
277
  template<class T>
278
  bool DecodeHuffman(const Byte** ppByte, size_t& nBytesRemaining, T* data) const;
279
280
  template<class T>
281
  bool WriteMinMaxRanges(const T* data, Byte** ppByte) const;
282
283
  template<class T>
284
  bool ReadMinMaxRanges(const Byte** ppByte, size_t& nBytesRemaining, const T* data);
285
286
  bool CheckMinMaxRanges(bool& minMaxEqual) const;
287
288
  template<class T>
289
  bool FillConstImage(T* data) const;
290
};
291
292
// -------------------------------------------------------------------------- ;
293
// -------------------------------------------------------------------------- ;
294
295
inline void Lerc2::AddUIntToCounts(int* pCounts, unsigned int val, int nBits)
296
0
{
297
0
  pCounts[0] += val & 1;
298
0
  for (int i = 1; i < nBits; i++)
299
0
    pCounts[i] += (val >>= 1) & 1;
300
0
}
301
302
// -------------------------------------------------------------------------- ;
303
304
inline void Lerc2::AddIntToCounts(int* pCounts, int val, int nBits)
305
0
{
306
0
  pCounts[0] += val & 1;
307
0
  for (int i = 1; i < nBits; i++)
308
0
    pCounts[i] += (val >>= 1) & 1;
309
0
}
310
311
// -------------------------------------------------------------------------- ;
312
313
inline bool Lerc2::NeedToCheckForIntOverflow(const HeaderInfo& hd)
314
0
{
315
0
  return (hd.dt == DT_Int || hd.dt == DT_UInt) && (hd.zMax - hd.zMin >= 0x7FFFFFFF);
316
0
}
317
318
// -------------------------------------------------------------------------- ;
319
320
inline bool Lerc2::NeedToCheckForFltRndErr(const HeaderInfo& hd)
321
0
{
322
0
  if (hd.dt != DT_Float)
323
0
    return false;
324
325
0
  if (hd.zMax - hd.zMin > 100000)  // prevent the below test falls through with extreme numbers like 10^38
326
0
    return true;
327
328
0
  float diff = (float)(hd.zMax - hd.zMin);
329
0
  double testMax = (double)diff + hd.zMin;
330
0
  double fltRndErr = fabs(testMax - hd.zMax);
331
332
0
  return (fltRndErr > hd.maxZError / 8);
333
0
}
334
335
// -------------------------------------------------------------------------- ;
336
337
inline double Lerc2::ComputeMaxVal(double zMin, double zMax, double maxZError)
338
0
{
339
0
  double fac = 1 / (2 * maxZError);    // must match the code in Decode(), don't touch it
340
0
  return (zMax - zMin) * fac;
341
0
}
342
343
// -------------------------------------------------------------------------- ;
344
345
template<class T>
346
inline bool Lerc2::NeedToQuantize(int numValidPixel, T zMin, T zMax) const
347
0
{
348
0
  if (numValidPixel == 0 || m_headerInfo.maxZError == 0)
349
0
    return false;
350
351
0
  double maxVal = ComputeMaxVal(zMin, zMax, m_headerInfo.maxZError);
352
0
  return !(maxVal > m_maxValToQuantize || (unsigned int)(maxVal + 0.5) == 0);
353
0
}
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<signed char>(int, signed char, signed char) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<double>(int, double, double) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<unsigned char>(int, unsigned char, unsigned char) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<short>(int, short, short) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<unsigned short>(int, unsigned short, unsigned short) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<int>(int, int, int) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<unsigned int>(int, unsigned int, unsigned int) const
Unexecuted instantiation: bool LercNS::Lerc2::NeedToQuantize<float>(int, float, float) const
354
355
// -------------------------------------------------------------------------- ;
356
357
template<class T>
358
inline void Lerc2::Quantize(const T* dataBuf, int num, T zMin, std::vector<unsigned int>& quantVec) const
359
0
{
360
0
  quantVec.resize(num);
361
362
0
  if (m_headerInfo.dt < DT_Float && m_headerInfo.maxZError == 0.5)    // int lossless
363
0
  {
364
0
    for (int i = 0; i < num; i++)
365
0
      quantVec[i] = (unsigned int)(dataBuf[i] - zMin);    // ok: char, short get promoted to int by C++ integral promotion rule
366
0
  }
367
0
  else    // float and/or lossy
368
0
  {
369
0
    double scale = 1 / (2 * m_headerInfo.maxZError);
370
0
    double zMinDbl = (double)zMin;
371
372
0
    for (int i = 0; i < num; i++)
373
0
      quantVec[i] = (unsigned int)(((double)dataBuf[i] - zMinDbl) * scale + 0.5);    // ok, consistent with ComputeMaxVal(...)
374
      //quantVec[i] = (unsigned int)((dataBuf[i] - zMin) * scale + 0.5);    // bad, not consistent with ComputeMaxVal(...)
375
0
  }
376
0
}
Unexecuted instantiation: void LercNS::Lerc2::Quantize<signed char>(signed char const*, int, signed char, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<int>(int const*, int, int, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<unsigned char>(unsigned char const*, int, unsigned char, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<short>(short const*, int, short, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<unsigned short>(unsigned short const*, int, unsigned short, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<unsigned int>(unsigned int const*, int, unsigned int, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<float>(float const*, int, float, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
Unexecuted instantiation: void LercNS::Lerc2::Quantize<double>(double const*, int, double, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> >&) const
377
378
// -------------------------------------------------------------------------- ;
379
380
template<class T>
381
inline void Lerc2::ScaleBack(T* dataBuf, const std::vector<unsigned int>& quantVec,
382
  double zMin, bool bDiff, bool bClamp, double zMaxClamp, double maxZError)
383
0
{
384
0
  double invScale = 2 * maxZError;    // for int types this is int
385
0
  int num = (int)quantVec.size();
386
387
0
  if (!bClamp)
388
0
    for (int i = 0; i < num; i++)
389
0
    {
390
0
      double z = zMin + quantVec[i] * invScale + (bDiff ? dataBuf[i] : 0);
391
0
      dataBuf[i] = (T)z;
392
0
    }
393
0
  else
394
0
    for (int i = 0; i < num; i++)
395
0
    {
396
0
      double z = zMin + quantVec[i] * invScale + (bDiff ? dataBuf[i] : 0);
397
0
      dataBuf[i] = (T)std::min(z, zMaxClamp);
398
0
    }
399
0
}
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<signed char>(signed char*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<unsigned char>(unsigned char*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<short>(short*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<unsigned short>(unsigned short*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<int>(int*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<unsigned int>(unsigned int*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<float>(float*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBack<double>(double*, std::__1::vector<unsigned int, std::__1::allocator<unsigned int> > const&, double, bool, bool, double, double)
400
401
// -------------------------------------------------------------------------- ;
402
403
template<class T>
404
inline void Lerc2::ScaleBackConstBlock(T* dataBuf, int num, double zMin, bool bClamp, double zMaxClamp)
405
0
{
406
0
  if (!bClamp)
407
0
    for (int i = 0; i < num; i++)
408
0
      dataBuf[i] = (T)(zMin + dataBuf[i]);
409
0
  else
410
0
    for (int i = 0; i < num; i++)
411
0
      dataBuf[i] = (T)std::min(zMin + dataBuf[i], zMaxClamp);
412
0
}
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<signed char>(signed char*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<unsigned char>(unsigned char*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<short>(short*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<unsigned short>(unsigned short*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<int>(int*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<unsigned int>(unsigned int*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<float>(float*, int, double, bool, double)
Unexecuted instantiation: void LercNS::Lerc2::ScaleBackConstBlock<double>(double*, int, double, bool, double)
413
414
// -------------------------------------------------------------------------- ;
415
416
template<class T>
417
inline int Lerc2::NumBytesTile(int numValidPixel, T zMin, T zMax, DataType dtZ, bool tryLut,
418
  BlockEncodeMode& blockEncodeMode, const std::vector<std::pair<unsigned int, unsigned int> >& sortedQuantVec) const
419
0
{
420
0
  blockEncodeMode = BEM_RawBinary;
421
422
0
  if (numValidPixel == 0 || (zMin == 0 && zMax == 0))
423
0
    return 1;
424
425
0
  double maxVal = 0, maxZError = m_headerInfo.maxZError;
426
0
  int nBytesRaw = (int)(1 + numValidPixel * sizeof(T));
427
428
0
  if ((maxZError == 0 && zMax > zMin)
429
0
    || (maxZError > 0 && (maxVal = ComputeMaxVal(zMin, zMax, maxZError)) > m_maxValToQuantize))
430
0
  {
431
0
    return nBytesRaw;
432
0
  }
433
0
  else
434
0
  {
435
0
    DataType dtReduced;
436
0
    ReduceDataType(zMin, dtZ, dtReduced);
437
0
    int nBytes = 1 + GetDataTypeSize(dtReduced);
438
439
0
    unsigned int maxElem = (unsigned int)(maxVal + 0.5);
440
0
    if (maxElem > 0)
441
0
    {
442
0
      nBytes += (!tryLut) ? BitStuffer2::ComputeNumBytesNeededSimple(numValidPixel, maxElem)
443
0
                          : BitStuffer2::ComputeNumBytesNeededLut(sortedQuantVec, tryLut);
444
0
    }
445
446
0
    if (nBytes < nBytesRaw)
447
0
      blockEncodeMode = (!tryLut || maxElem == 0) ? BEM_BitStuffSimple : BEM_BitStuffLUT;
448
0
    else
449
0
      nBytes = nBytesRaw;
450
451
0
    return nBytes;
452
0
  }
453
0
}
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<signed char>(int, signed char, signed char, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<int>(int, int, int, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<unsigned char>(int, unsigned char, unsigned char, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<short>(int, short, short, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<unsigned short>(int, unsigned short, unsigned short, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<unsigned int>(int, unsigned int, unsigned int, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<float>(int, float, float, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
Unexecuted instantiation: int LercNS::Lerc2::NumBytesTile<double>(int, double, double, LercNS::Lerc2::DataType, bool, LercNS::Lerc2::BlockEncodeMode&, std::__1::vector<std::__1::pair<unsigned int, unsigned int>, std::__1::allocator<std::__1::pair<unsigned int, unsigned int> > > const&) const
454
455
// -------------------------------------------------------------------------- ;
456
457
template<class T>
458
inline int Lerc2::ReduceDataType(T z, DataType dt, DataType& dtReduced)
459
0
{
460
  // clamp to dst range first to guarantee portable code, see  https://www.cplusplus.com/doc/tutorial/typecasting/
461
462
0
  Byte b = (z >= 0 && z <= 255) ? (Byte)z : 0;
463
0
  switch (dt)
464
0
  {
465
0
    case DT_Short:
466
0
    {
467
0
      signed char c = (z >= (double)-128 && z <= 127) ? (signed char)z : 0;    // avoid signed / unsigned warnings on some compilers
468
0
      int tc = (T)c == z ? 2 : (T)b == z ? 1 : 0;
469
0
      dtReduced = (DataType)(dt - tc);
470
0
      return tc;
471
0
    }
472
0
    case DT_UShort:
473
0
    {
474
0
      int tc = (T)b == z ? 1 : 0;
475
0
      dtReduced = (DataType)(dt - 2 * tc);
476
0
      return tc;
477
0
    }
478
0
    case DT_Int:
479
0
    {
480
0
      short s = (z >= (double)SHRT_MIN && z <= SHRT_MAX) ? (short)z : 0;
481
0
      unsigned short us = (z >= 0 && z <= USHRT_MAX) ? (unsigned short)z : 0;
482
0
      int tc = (T)b == z ? 3 : (T)s == z ? 2 : (T)us == z ? 1 : 0;
483
0
      dtReduced = (DataType)(dt - tc);
484
0
      return tc;
485
0
    }
486
0
    case DT_UInt:
487
0
    {
488
0
      unsigned short us = (z >= 0 && z <= USHRT_MAX) ? (unsigned short)z : 0;
489
0
      int tc = (T)b == z ? 2 : (T)us == z ? 1 : 0;
490
0
      dtReduced = (DataType)(dt - 2 * tc);
491
0
      return tc;
492
0
    }
493
0
    case DT_Float:
494
0
    {
495
0
      short s = (z >= (float)SHRT_MIN && z <= SHRT_MAX) ? (short)z : 0;
496
0
      int tc = (T)b == z ? 2 : (T)s == z ? 1 : 0;
497
0
      dtReduced = tc == 0 ? dt : (tc == 1 ? DT_Short : DT_Byte);
498
0
      return tc;
499
0
    }
500
0
    case DT_Double:
501
0
    {
502
0
      short s = (z >= (double)SHRT_MIN && z <= SHRT_MAX) ? (short)z : 0;
503
0
      int l = (z >= (double)INT_MIN && z <= (double)INT_MAX) ? (int)z : 0;
504
0
      float f = (z >= -FLT_MAX && z <= FLT_MAX) ? (float)z : 0;
505
0
      int tc = (T)s == z ? 3 : (T)l == z ? 2 : (T)f == z ? 1 : 0;
506
0
      dtReduced = tc == 0 ? dt : (DataType)(dt - 2 * tc + 1);
507
0
      return tc;
508
0
    }
509
0
    default:
510
0
    {
511
0
      dtReduced = dt;
512
0
      return 0;
513
0
    }
514
0
  }
515
0
}
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<signed char>(signed char, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<int>(int, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<unsigned char>(unsigned char, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<short>(short, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<unsigned short>(unsigned short, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<unsigned int>(unsigned int, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<float>(float, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
Unexecuted instantiation: int LercNS::Lerc2::ReduceDataType<double>(double, LercNS::Lerc2::DataType, LercNS::Lerc2::DataType&)
516
517
// -------------------------------------------------------------------------- ;
518
519
inline Lerc2::DataType Lerc2::ValidateDataType(int dt)
520
0
{
521
0
  if (dt >= DT_Char && dt <= DT_Double)
522
0
    return static_cast<DataType>(dt);
523
0
  return DT_Undefined;
524
0
}
525
526
// -------------------------------------------------------------------------- ;
527
528
inline
529
Lerc2::DataType Lerc2::GetDataTypeUsed(DataType dt, int tc)
530
0
{
531
0
  switch (dt)
532
0
  {
533
0
    case DT_Short:
534
0
    case DT_Int:     return ValidateDataType(dt - tc);
535
0
    case DT_UShort:
536
0
    case DT_UInt:    return ValidateDataType(dt - 2 * tc);
537
0
    case DT_Float:   return tc == 0 ? dt : (tc == 1 ? DT_Short : DT_Byte);
538
0
    case DT_Double:  return tc == 0 ? dt : ValidateDataType(dt - 2 * tc + 1);
539
0
    default:
540
0
      return dt;
541
0
  }
542
0
}
543
544
// -------------------------------------------------------------------------- ;
545
546
inline
547
bool Lerc2::WriteVariableDataType(Byte** ppByte, double z, DataType dtUsed)
548
0
{
549
0
  Byte* ptr = *ppByte;
550
551
0
  switch (dtUsed)
552
0
  {
553
0
    case DT_Char:
554
0
    {
555
0
      *((signed char*)ptr) = (signed char)z;
556
0
      ptr++;
557
0
      break;
558
0
    }
559
0
    case DT_Byte:
560
0
    {
561
0
      *((Byte*)ptr) = (Byte)z;
562
0
      ptr++;
563
0
      break;
564
0
    }
565
0
    case DT_Short:
566
0
    {
567
0
      short s = (short)z;
568
0
      memcpy(ptr, &s, sizeof(short));
569
0
      ptr += 2;
570
0
      break;
571
0
    }
572
0
    case DT_UShort:
573
0
    {
574
0
      unsigned short us = (unsigned short)z;
575
0
      memcpy(ptr, &us, sizeof(unsigned short));
576
0
      ptr += 2;
577
0
      break;
578
0
    }
579
0
    case DT_Int:
580
0
    {
581
0
      int i = (int)z;
582
0
      memcpy(ptr, &i, sizeof(int));
583
0
      ptr += 4;
584
0
      break;
585
0
    }
586
0
    case DT_UInt:
587
0
    {
588
0
      unsigned int n = (unsigned int)z;
589
0
      memcpy(ptr, &n, sizeof(unsigned int));
590
0
      ptr += 4;
591
0
      break;
592
0
    }
593
0
    case DT_Float:
594
0
    {
595
0
      float f = (float)z;
596
0
      memcpy(ptr, &f, sizeof(float));
597
0
      ptr += 4;
598
0
      break;
599
0
    }
600
0
    case DT_Double:
601
0
    {
602
0
      memcpy(ptr, &z, sizeof(double));
603
0
      ptr += 8;
604
0
      break;
605
0
    }
606
607
0
    default:
608
0
      return false;
609
0
  }
610
611
0
  *ppByte = ptr;
612
0
  return true;
613
0
}
614
615
// -------------------------------------------------------------------------- ;
616
617
inline
618
double Lerc2::ReadVariableDataType(const Byte** ppByte, DataType dtUsed)
619
0
{
620
0
  const Byte* ptr = *ppByte;
621
622
0
  switch (dtUsed)
623
0
  {
624
0
    case DT_Char:
625
0
    {
626
0
      signed char c = *((signed char*)ptr);
627
0
      *ppByte = ptr + 1;
628
0
      return c;
629
0
    }
630
0
    case DT_Byte:
631
0
    {
632
0
      Byte b = *((Byte*)ptr);
633
0
      *ppByte = ptr + 1;
634
0
      return b;
635
0
    }
636
0
    case DT_Short:
637
0
    {
638
0
      short s;
639
0
      memcpy(&s, ptr, sizeof(short));
640
0
      *ppByte = ptr + 2;
641
0
      return s;
642
0
    }
643
0
    case DT_UShort:
644
0
    {
645
0
      unsigned short us;
646
0
      memcpy(&us, ptr, sizeof(unsigned short));
647
0
      *ppByte = ptr + 2;
648
0
      return us;
649
0
    }
650
0
    case DT_Int:
651
0
    {
652
0
      int i;
653
0
      memcpy(&i, ptr, sizeof(int));
654
0
      *ppByte = ptr + 4;
655
0
      return i;
656
0
    }
657
0
    case DT_UInt:
658
0
    {
659
0
      unsigned int n;
660
0
      memcpy(&n, ptr, sizeof(unsigned int));
661
0
      *ppByte = ptr + 4;
662
0
      return n;
663
0
    }
664
0
    case DT_Float:
665
0
    {
666
0
      float f;
667
0
      memcpy(&f, ptr, sizeof(float));
668
0
      *ppByte = ptr + 4;
669
0
      return f;
670
0
    }
671
0
    case DT_Double:
672
0
    {
673
0
      double d;
674
0
      memcpy(&d, ptr, sizeof(double));
675
0
      *ppByte = ptr + 8;
676
0
      return d;
677
0
    }
678
0
    default:
679
0
      return 0;
680
0
  }
681
0
}
682
683
// -------------------------------------------------------------------------- ;
684
685
inline
686
unsigned int Lerc2::GetMaxValToQuantize(DataType dt)
687
0
{
688
0
  switch (dt)
689
0
  {
690
0
    case DT_Char:
691
0
    case DT_Byte:    //return (1 <<  7) - 1;    // disabled: allow LUT mode for 8 bit segmented
692
0
    case DT_Short:
693
0
    case DT_UShort:  return (1 << 15) - 1;
694
695
0
    case DT_Int:
696
0
    case DT_UInt:
697
0
    case DT_Float:
698
0
    case DT_Double:  return (1 << 30) - 1;
699
700
0
    default:
701
0
      return 0;
702
0
  }
703
0
}
704
705
// -------------------------------------------------------------------------- ;
706
707
inline
708
unsigned int Lerc2::GetDataTypeSize(DataType dt)
709
0
{
710
0
  switch (dt)
711
0
  {
712
0
    case DT_Char:
713
0
    case DT_Byte:   return 1;
714
0
    case DT_Short:
715
0
    case DT_UShort: return 2;
716
0
    case DT_Int:
717
0
    case DT_UInt:
718
0
    case DT_Float:  return 4;
719
0
    case DT_Double: return 8;
720
721
0
    default:
722
0
      return 0;
723
0
  }
724
0
}
725
726
// -------------------------------------------------------------------------- ;
727
728
inline
729
bool Lerc2::CheckMinMaxRanges(bool& minMaxEqual) const
730
0
{
731
0
  int nDepth = m_headerInfo.nDepth;
732
0
  if ((int)m_zMinVec.size() != nDepth || (int)m_zMaxVec.size() != nDepth)
733
0
    return false;
734
735
0
  minMaxEqual = (0 == memcmp(&m_zMinVec[0], &m_zMaxVec[0], nDepth * sizeof(m_zMinVec[0])));
736
0
  return true;
737
0
}
738
739
// -------------------------------------------------------------------------- ;
740
741
NAMESPACE_LERC_END
742
#endif