/src/gdal/gcore/rasterio.cpp
Line | Count | Source |
1 | | /****************************************************************************** |
2 | | * |
3 | | * Project: GDAL Core |
4 | | * Purpose: Contains default implementation of GDALRasterBand::IRasterIO() |
5 | | * and supporting functions of broader utility. |
6 | | * Author: Frank Warmerdam, warmerdam@pobox.com |
7 | | * |
8 | | ****************************************************************************** |
9 | | * Copyright (c) 1998, Frank Warmerdam |
10 | | * Copyright (c) 2007-2014, Even Rouault <even dot rouault at spatialys.com> |
11 | | * |
12 | | * SPDX-License-Identifier: MIT |
13 | | ****************************************************************************/ |
14 | | |
15 | | #include "cpl_port.h" |
16 | | #include "gdal.h" |
17 | | #include "gdal_priv.h" |
18 | | |
19 | | #include <cassert> |
20 | | #include <climits> |
21 | | #include <cmath> |
22 | | #include <cstddef> |
23 | | #include <cstdio> |
24 | | #include <cstdlib> |
25 | | #include <cstring> |
26 | | |
27 | | #include <algorithm> |
28 | | #include <limits> |
29 | | #include <stdexcept> |
30 | | #include <type_traits> |
31 | | |
32 | | #include "cpl_conv.h" |
33 | | #include "cpl_cpu_features.h" |
34 | | #include "cpl_error.h" |
35 | | #include "cpl_float.h" |
36 | | #include "cpl_progress.h" |
37 | | #include "cpl_string.h" |
38 | | #include "cpl_vsi.h" |
39 | | #include "gdal_priv_templates.hpp" |
40 | | #include "gdal_vrt.h" |
41 | | #include "gdalwarper.h" |
42 | | #include "memdataset.h" |
43 | | #include "vrtdataset.h" |
44 | | |
45 | | #if defined(__x86_64) || defined(_M_X64) |
46 | | #include <emmintrin.h> |
47 | | #include <immintrin.h> |
48 | | #define HAVE_SSE2 |
49 | | // AVX2 dispatch: compile AVX2 code with target attribute, detect at runtime |
50 | | #if (defined(__GNUC__) || defined(__clang__)) && \ |
51 | | defined(HAVE_AVX2_AT_COMPILE_TIME) |
52 | | #define HAVE_AVX2_DISPATCH |
53 | | #elif defined(_MSC_VER) |
54 | | #include <intrin.h> |
55 | | #define HAVE_AVX2_DISPATCH |
56 | | #endif |
57 | | #elif defined(USE_NEON_OPTIMIZATIONS) |
58 | | #include "include_sse2neon.h" |
59 | | #define HAVE_SSE2 |
60 | | #endif |
61 | | |
62 | | #ifdef HAVE_SSSE3_AT_COMPILE_TIME |
63 | | #include "rasterio_ssse3.h" |
64 | | #ifdef __SSSE3__ |
65 | | #include <tmmintrin.h> |
66 | | #endif |
67 | | #endif |
68 | | |
69 | | #ifdef __SSE4_1__ |
70 | | #include <smmintrin.h> |
71 | | #endif |
72 | | |
73 | | #ifdef __GNUC__ |
74 | | #define CPL_NOINLINE __attribute__((noinline)) |
75 | | #else |
76 | | #define CPL_NOINLINE |
77 | | #endif |
78 | | |
79 | | static void GDALFastCopyByte(const GByte *CPL_RESTRICT pSrcData, |
80 | | int nSrcPixelStride, GByte *CPL_RESTRICT pDstData, |
81 | | int nDstPixelStride, GPtrDiff_t nWordCount); |
82 | | |
83 | | /************************************************************************/ |
84 | | /* DownsamplingIntegerXFactor() */ |
85 | | /************************************************************************/ |
86 | | |
87 | | template <bool bSameDataType, int DATA_TYPE_SIZE> |
88 | | static bool DownsamplingIntegerXFactor( |
89 | | GDALRasterBand *poBand, int iSrcX, int nSrcXInc, GPtrDiff_t iSrcOffsetCst, |
90 | | GByte *CPL_RESTRICT pabyDstData, int nPixelSpace, int nBufXSize, |
91 | | GDALDataType eDataType, GDALDataType eBufType, int &nStartBlockX, |
92 | | int nBlockXSize, GDALRasterBlock *&poBlock, int nLBlockY) |
93 | 0 | { |
94 | 0 | const int nBandDataSize = |
95 | 0 | bSameDataType ? DATA_TYPE_SIZE : GDALGetDataTypeSizeBytes(eDataType); |
96 | 0 | int nOuterLoopIters = nBufXSize - 1; |
97 | 0 | const int nIncSrcOffset = nSrcXInc * nBandDataSize; |
98 | 0 | const GByte *CPL_RESTRICT pabySrcData; |
99 | 0 | int nEndBlockX = nBlockXSize + nStartBlockX; |
100 | |
|
101 | 0 | if (iSrcX < nEndBlockX) |
102 | 0 | { |
103 | 0 | CPLAssert(poBlock); |
104 | 0 | goto no_reload_block; |
105 | 0 | } |
106 | 0 | goto reload_block; |
107 | | |
108 | | // Don't do the last iteration in the loop, as iSrcX might go beyond |
109 | | // nRasterXSize - 1 |
110 | 0 | while (--nOuterLoopIters >= 1) |
111 | 0 | { |
112 | 0 | iSrcX += nSrcXInc; |
113 | 0 | pabySrcData += nIncSrcOffset; |
114 | 0 | pabyDstData += nPixelSpace; |
115 | | |
116 | | /* -------------------------------------------------------------------- |
117 | | */ |
118 | | /* Ensure we have the appropriate block loaded. */ |
119 | | /* -------------------------------------------------------------------- |
120 | | */ |
121 | 0 | if (iSrcX >= nEndBlockX) |
122 | 0 | { |
123 | 0 | reload_block: |
124 | 0 | { |
125 | 0 | const int nLBlockX = iSrcX / nBlockXSize; |
126 | 0 | nStartBlockX = nLBlockX * nBlockXSize; |
127 | 0 | nEndBlockX = nStartBlockX + nBlockXSize; |
128 | |
|
129 | 0 | if (poBlock != nullptr) |
130 | 0 | poBlock->DropLock(); |
131 | |
|
132 | 0 | poBlock = poBand->GetLockedBlockRef(nLBlockX, nLBlockY, FALSE); |
133 | 0 | if (poBlock == nullptr) |
134 | 0 | { |
135 | 0 | return false; |
136 | 0 | } |
137 | 0 | } |
138 | | |
139 | 0 | no_reload_block: |
140 | 0 | const GByte *pabySrcBlock = |
141 | 0 | static_cast<const GByte *>(poBlock->GetDataRef()); |
142 | 0 | GPtrDiff_t iSrcOffset = |
143 | 0 | (iSrcX - nStartBlockX + iSrcOffsetCst) * nBandDataSize; |
144 | 0 | pabySrcData = pabySrcBlock + iSrcOffset; |
145 | 0 | } |
146 | | |
147 | | /* -------------------------------------------------------------------- |
148 | | */ |
149 | | /* Copy the maximum run of pixels. */ |
150 | | /* -------------------------------------------------------------------- |
151 | | */ |
152 | | |
153 | 0 | const int nIters = std::min( |
154 | 0 | (nEndBlockX - iSrcX + (nSrcXInc - 1)) / nSrcXInc, nOuterLoopIters); |
155 | 0 | if (bSameDataType) |
156 | 0 | { |
157 | 0 | memcpy(pabyDstData, pabySrcData, nBandDataSize); |
158 | 0 | if (nIters > 1) |
159 | 0 | { |
160 | 0 | if (DATA_TYPE_SIZE == 1) |
161 | 0 | { |
162 | 0 | pabySrcData += nIncSrcOffset; |
163 | 0 | pabyDstData += nPixelSpace; |
164 | 0 | GDALFastCopyByte(pabySrcData, nIncSrcOffset, pabyDstData, |
165 | 0 | nPixelSpace, nIters - 1); |
166 | 0 | pabySrcData += |
167 | 0 | static_cast<GPtrDiff_t>(nIncSrcOffset) * (nIters - 2); |
168 | 0 | pabyDstData += |
169 | 0 | static_cast<GPtrDiff_t>(nPixelSpace) * (nIters - 2); |
170 | 0 | } |
171 | 0 | else |
172 | 0 | { |
173 | 0 | for (int i = 0; i < nIters - 1; i++) |
174 | 0 | { |
175 | 0 | pabySrcData += nIncSrcOffset; |
176 | 0 | pabyDstData += nPixelSpace; |
177 | 0 | memcpy(pabyDstData, pabySrcData, nBandDataSize); |
178 | 0 | } |
179 | 0 | } |
180 | 0 | iSrcX += nSrcXInc * (nIters - 1); |
181 | 0 | nOuterLoopIters -= nIters - 1; |
182 | 0 | } |
183 | 0 | } |
184 | 0 | else |
185 | 0 | { |
186 | | // Type to type conversion ... |
187 | 0 | GDALCopyWords64(pabySrcData, eDataType, nIncSrcOffset, pabyDstData, |
188 | 0 | eBufType, nPixelSpace, std::max(1, nIters)); |
189 | 0 | if (nIters > 1) |
190 | 0 | { |
191 | 0 | pabySrcData += |
192 | 0 | static_cast<GPtrDiff_t>(nIncSrcOffset) * (nIters - 1); |
193 | 0 | pabyDstData += |
194 | 0 | static_cast<GPtrDiff_t>(nPixelSpace) * (nIters - 1); |
195 | 0 | iSrcX += nSrcXInc * (nIters - 1); |
196 | 0 | nOuterLoopIters -= nIters - 1; |
197 | 0 | } |
198 | 0 | } |
199 | 0 | } |
200 | | |
201 | | // Deal with last iteration to avoid iSrcX to go beyond nRasterXSize - 1 |
202 | 0 | if (nOuterLoopIters == 0) |
203 | 0 | { |
204 | 0 | const int nRasterXSize = poBand->GetXSize(); |
205 | 0 | iSrcX = |
206 | 0 | static_cast<int>(std::min(static_cast<GInt64>(iSrcX) + nSrcXInc, |
207 | 0 | static_cast<GInt64>(nRasterXSize - 1))); |
208 | 0 | pabyDstData += nPixelSpace; |
209 | 0 | if (iSrcX < nEndBlockX) |
210 | 0 | { |
211 | 0 | goto no_reload_block; |
212 | 0 | } |
213 | 0 | goto reload_block; |
214 | 0 | } |
215 | 0 | return true; |
216 | 0 | } Unexecuted instantiation: rasterio.cpp:bool DownsamplingIntegerXFactor<true, 1>(GDALRasterBand*, int, int, long long, unsigned char*, int, int, GDALDataType, GDALDataType, int&, int, GDALRasterBlock*&, int) Unexecuted instantiation: rasterio.cpp:bool DownsamplingIntegerXFactor<true, 2>(GDALRasterBand*, int, int, long long, unsigned char*, int, int, GDALDataType, GDALDataType, int&, int, GDALRasterBlock*&, int) Unexecuted instantiation: rasterio.cpp:bool DownsamplingIntegerXFactor<true, 4>(GDALRasterBand*, int, int, long long, unsigned char*, int, int, GDALDataType, GDALDataType, int&, int, GDALRasterBlock*&, int) Unexecuted instantiation: rasterio.cpp:bool DownsamplingIntegerXFactor<true, 8>(GDALRasterBand*, int, int, long long, unsigned char*, int, int, GDALDataType, GDALDataType, int&, int, GDALRasterBlock*&, int) Unexecuted instantiation: rasterio.cpp:bool DownsamplingIntegerXFactor<true, 16>(GDALRasterBand*, int, int, long long, unsigned char*, int, int, GDALDataType, GDALDataType, int&, int, GDALRasterBlock*&, int) Unexecuted instantiation: rasterio.cpp:bool DownsamplingIntegerXFactor<false, 0>(GDALRasterBand*, int, int, long long, unsigned char*, int, int, GDALDataType, GDALDataType, int&, int, GDALRasterBlock*&, int) |
217 | | |
218 | | template <class A, class B> |
219 | | CPL_NOSANITIZE_UNSIGNED_INT_OVERFLOW inline auto CPLUnsanitizedMul(A a, B b) |
220 | 0 | { |
221 | 0 | return a * b; |
222 | 0 | } |
223 | | |
224 | | /************************************************************************/ |
225 | | /* IRasterIO() */ |
226 | | /* */ |
227 | | /* Default internal implementation of RasterIO() ... utilizes */ |
228 | | /* the Block access methods to satisfy the request. This would */ |
229 | | /* normally only be overridden by formats with overviews. */ |
230 | | /************************************************************************/ |
231 | | |
232 | | CPLErr GDALRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff, |
233 | | int nXSize, int nYSize, void *pData, |
234 | | int nBufXSize, int nBufYSize, |
235 | | GDALDataType eBufType, GSpacing nPixelSpace, |
236 | | GSpacing nLineSpace, |
237 | | GDALRasterIOExtraArg *psExtraArg) |
238 | | |
239 | 0 | { |
240 | 0 | if (eRWFlag == GF_Write && eFlushBlockErr != CE_None) |
241 | 0 | { |
242 | 0 | CPLError(eFlushBlockErr, CPLE_AppDefined, |
243 | 0 | "An error occurred while writing a dirty block " |
244 | 0 | "from GDALRasterBand::IRasterIO"); |
245 | 0 | CPLErr eErr = eFlushBlockErr; |
246 | 0 | eFlushBlockErr = CE_None; |
247 | 0 | return eErr; |
248 | 0 | } |
249 | 0 | if (nBlockXSize <= 0 || nBlockYSize <= 0) |
250 | 0 | { |
251 | 0 | CPLError(CE_Failure, CPLE_AppDefined, "Invalid block size"); |
252 | 0 | return CE_Failure; |
253 | 0 | } |
254 | | |
255 | 0 | const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType); |
256 | 0 | const int nBufDataSize = GDALGetDataTypeSizeBytes(eBufType); |
257 | 0 | GByte dummyBlock[2] = {0, 0}; |
258 | 0 | GByte *pabySrcBlock = |
259 | 0 | dummyBlock; /* to avoid Coverity warning about nullptr dereference */ |
260 | 0 | GDALRasterBlock *poBlock = nullptr; |
261 | 0 | const bool bUseIntegerRequestCoords = |
262 | 0 | (!psExtraArg->bFloatingPointWindowValidity || |
263 | 0 | (nXOff == psExtraArg->dfXOff && nYOff == psExtraArg->dfYOff && |
264 | 0 | nXSize == psExtraArg->dfXSize && nYSize == psExtraArg->dfYSize)); |
265 | | |
266 | | /* ==================================================================== */ |
267 | | /* A common case is the data requested with the destination */ |
268 | | /* is packed, and the block width is the raster width. */ |
269 | | /* ==================================================================== */ |
270 | 0 | if (nPixelSpace == nBufDataSize && nLineSpace == nPixelSpace * nXSize && |
271 | 0 | nBlockXSize == GetXSize() && nBufXSize == nXSize && |
272 | 0 | nBufYSize == nYSize && bUseIntegerRequestCoords) |
273 | 0 | { |
274 | 0 | CPLErr eErr = CE_None; |
275 | 0 | int nLBlockY = -1; |
276 | |
|
277 | 0 | for (int iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++) |
278 | 0 | { |
279 | 0 | const int iSrcY = iBufYOff + nYOff; |
280 | |
|
281 | 0 | if (iSrcY < nLBlockY * nBlockYSize || |
282 | 0 | iSrcY - nBlockYSize >= nLBlockY * nBlockYSize) |
283 | 0 | { |
284 | 0 | nLBlockY = iSrcY / nBlockYSize; |
285 | 0 | bool bJustInitialize = |
286 | 0 | eRWFlag == GF_Write && nXOff == 0 && |
287 | 0 | nXSize == nBlockXSize && nYOff <= nLBlockY * nBlockYSize && |
288 | 0 | nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize; |
289 | | |
290 | | // Is this a partial tile at right and/or bottom edges of |
291 | | // the raster, and that is going to be completely written? |
292 | | // If so, do not load it from storage, but zero it so that |
293 | | // the content outsize of the validity area is initialized. |
294 | 0 | bool bMemZeroBuffer = false; |
295 | 0 | if (eRWFlag == GF_Write && !bJustInitialize && nXOff == 0 && |
296 | 0 | nXSize == nBlockXSize && nYOff <= nLBlockY * nBlockYSize && |
297 | 0 | nYOff + nYSize == GetYSize() && |
298 | 0 | nLBlockY * nBlockYSize > GetYSize() - nBlockYSize) |
299 | 0 | { |
300 | 0 | bJustInitialize = true; |
301 | 0 | bMemZeroBuffer = true; |
302 | 0 | } |
303 | |
|
304 | 0 | if (poBlock) |
305 | 0 | poBlock->DropLock(); |
306 | |
|
307 | 0 | const GUInt32 nErrorCounter = CPLGetErrorCounter(); |
308 | 0 | poBlock = GetLockedBlockRef(0, nLBlockY, bJustInitialize); |
309 | 0 | if (poBlock == nullptr) |
310 | 0 | { |
311 | 0 | if (strstr(CPLGetLastErrorMsg(), "IReadBlock failed") == |
312 | 0 | nullptr) |
313 | 0 | { |
314 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
315 | 0 | "GetBlockRef failed at X block offset %d, " |
316 | 0 | "Y block offset %d%s", |
317 | 0 | 0, nLBlockY, |
318 | 0 | (nErrorCounter != CPLGetErrorCounter()) |
319 | 0 | ? CPLSPrintf(": %s", CPLGetLastErrorMsg()) |
320 | 0 | : ""); |
321 | 0 | } |
322 | 0 | eErr = CE_Failure; |
323 | 0 | break; |
324 | 0 | } |
325 | | |
326 | 0 | if (eRWFlag == GF_Write) |
327 | 0 | poBlock->MarkDirty(); |
328 | |
|
329 | 0 | pabySrcBlock = static_cast<GByte *>(poBlock->GetDataRef()); |
330 | 0 | if (bMemZeroBuffer) |
331 | 0 | { |
332 | 0 | memset(pabySrcBlock, 0, |
333 | 0 | static_cast<GPtrDiff_t>(nBandDataSize) * |
334 | 0 | nBlockXSize * nBlockYSize); |
335 | 0 | } |
336 | 0 | } |
337 | | |
338 | 0 | const auto nSrcByteOffset = |
339 | 0 | (static_cast<GPtrDiff_t>(iSrcY - nLBlockY * nBlockYSize) * |
340 | 0 | nBlockXSize + |
341 | 0 | nXOff) * |
342 | 0 | nBandDataSize; |
343 | |
|
344 | 0 | if (eDataType == eBufType) |
345 | 0 | { |
346 | 0 | if (eRWFlag == GF_Read) |
347 | 0 | memcpy(static_cast<GByte *>(pData) + |
348 | 0 | static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace, |
349 | 0 | pabySrcBlock + nSrcByteOffset, |
350 | 0 | static_cast<size_t>(nLineSpace)); |
351 | 0 | else |
352 | 0 | memcpy(pabySrcBlock + nSrcByteOffset, |
353 | 0 | static_cast<GByte *>(pData) + |
354 | 0 | static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace, |
355 | 0 | static_cast<size_t>(nLineSpace)); |
356 | 0 | } |
357 | 0 | else |
358 | 0 | { |
359 | | // Type to type conversion. |
360 | 0 | if (eRWFlag == GF_Read) |
361 | 0 | GDALCopyWords64( |
362 | 0 | pabySrcBlock + nSrcByteOffset, eDataType, nBandDataSize, |
363 | 0 | static_cast<GByte *>(pData) + |
364 | 0 | static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace, |
365 | 0 | eBufType, static_cast<int>(nPixelSpace), nBufXSize); |
366 | 0 | else |
367 | 0 | GDALCopyWords64(static_cast<GByte *>(pData) + |
368 | 0 | static_cast<GPtrDiff_t>(iBufYOff) * |
369 | 0 | nLineSpace, |
370 | 0 | eBufType, static_cast<int>(nPixelSpace), |
371 | 0 | pabySrcBlock + nSrcByteOffset, eDataType, |
372 | 0 | nBandDataSize, nBufXSize); |
373 | 0 | } |
374 | |
|
375 | 0 | if (psExtraArg->pfnProgress != nullptr && |
376 | 0 | !psExtraArg->pfnProgress(1.0 * (iBufYOff + 1) / nBufYSize, "", |
377 | 0 | psExtraArg->pProgressData)) |
378 | 0 | { |
379 | 0 | eErr = CE_Failure; |
380 | 0 | break; |
381 | 0 | } |
382 | 0 | } |
383 | |
|
384 | 0 | if (poBlock) |
385 | 0 | poBlock->DropLock(); |
386 | |
|
387 | 0 | return eErr; |
388 | 0 | } |
389 | | |
390 | | /* ==================================================================== */ |
391 | | /* Do we have overviews that would be appropriate to satisfy */ |
392 | | /* this request? */ |
393 | | /* ==================================================================== */ |
394 | 0 | if ((nBufXSize < nXSize || nBufYSize < nYSize) && GetOverviewCount() > 0 && |
395 | 0 | eRWFlag == GF_Read) |
396 | 0 | { |
397 | 0 | GDALRasterIOExtraArg sExtraArg; |
398 | 0 | GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg); |
399 | |
|
400 | 0 | const int nOverview = |
401 | 0 | GDALBandGetBestOverviewLevel2(this, nXOff, nYOff, nXSize, nYSize, |
402 | 0 | nBufXSize, nBufYSize, &sExtraArg); |
403 | 0 | if (nOverview >= 0) |
404 | 0 | { |
405 | 0 | GDALRasterBand *poOverviewBand = GetOverview(nOverview); |
406 | 0 | if (poOverviewBand == nullptr) |
407 | 0 | return CE_Failure; |
408 | | |
409 | 0 | return poOverviewBand->RasterIO( |
410 | 0 | eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, nBufXSize, |
411 | 0 | nBufYSize, eBufType, nPixelSpace, nLineSpace, &sExtraArg); |
412 | 0 | } |
413 | 0 | } |
414 | | |
415 | 0 | if (eRWFlag == GF_Read && nBufXSize < nXSize / 100 && |
416 | 0 | nBufYSize < nYSize / 100 && nPixelSpace == nBufDataSize && |
417 | 0 | nLineSpace == nPixelSpace * nBufXSize && |
418 | 0 | CPLTestBool(CPLGetConfigOption("GDAL_NO_COSTLY_OVERVIEW", "NO"))) |
419 | 0 | { |
420 | 0 | memset(pData, 0, static_cast<size_t>(nLineSpace * nBufYSize)); |
421 | 0 | return CE_None; |
422 | 0 | } |
423 | | |
424 | | /* ==================================================================== */ |
425 | | /* The second case when we don't need subsample data but likely */ |
426 | | /* need data type conversion. */ |
427 | | /* ==================================================================== */ |
428 | 0 | if ( // nPixelSpace == nBufDataSize && |
429 | 0 | nXSize == nBufXSize && nYSize == nBufYSize && bUseIntegerRequestCoords) |
430 | 0 | { |
431 | | #if DEBUG_VERBOSE |
432 | | printf("IRasterIO(%d,%d,%d,%d) rw=%d case 2\n", /*ok*/ |
433 | | nXOff, nYOff, nXSize, nYSize, static_cast<int>(eRWFlag)); |
434 | | #endif |
435 | | |
436 | | /* -------------------------------------------------------------------- |
437 | | */ |
438 | | /* Loop over buffer computing source locations. */ |
439 | | /* -------------------------------------------------------------------- |
440 | | */ |
441 | | // Calculate starting values out of loop |
442 | 0 | const int nLBlockXStart = nXOff / nBlockXSize; |
443 | 0 | const int nXSpanEnd = nBufXSize + nXOff; |
444 | |
|
445 | 0 | int iBufYOff = 0; |
446 | 0 | int iSrcY = nYOff; |
447 | 0 | while (true) |
448 | 0 | { |
449 | 0 | GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) * |
450 | 0 | static_cast<GPtrDiff_t>(nLineSpace); |
451 | 0 | int nLBlockY = iSrcY / nBlockYSize; |
452 | 0 | int nLBlockX = nLBlockXStart; |
453 | 0 | int iSrcX = nXOff; |
454 | 0 | while (iSrcX < nXSpanEnd) |
455 | 0 | { |
456 | 0 | int nXSpan = nLBlockX * nBlockXSize; |
457 | 0 | if (nXSpan < INT_MAX - nBlockXSize) |
458 | 0 | nXSpan += nBlockXSize; |
459 | 0 | else |
460 | 0 | nXSpan = INT_MAX; |
461 | 0 | const int nXRight = nXSpan; |
462 | 0 | nXSpan = (nXSpan < nXSpanEnd ? nXSpan : nXSpanEnd) - iSrcX; |
463 | |
|
464 | 0 | const size_t nXSpanSize = |
465 | 0 | CPLUnsanitizedMul(nXSpan, static_cast<size_t>(nPixelSpace)); |
466 | |
|
467 | 0 | bool bJustInitialize = |
468 | 0 | eRWFlag == GF_Write && nYOff <= nLBlockY * nBlockYSize && |
469 | 0 | nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize && |
470 | 0 | nXOff <= nLBlockX * nBlockXSize && |
471 | 0 | nXOff + nXSize >= nXRight; |
472 | | |
473 | | // Is this a partial tile at right and/or bottom edges of |
474 | | // the raster, and that is going to be completely written? |
475 | | // If so, do not load it from storage, but zero it so that |
476 | | // the content outsize of the validity area is initialized. |
477 | 0 | bool bMemZeroBuffer = false; |
478 | 0 | if (eRWFlag == GF_Write && !bJustInitialize && |
479 | 0 | nXOff <= nLBlockX * nBlockXSize && |
480 | 0 | nYOff <= nLBlockY * nBlockYSize && |
481 | 0 | (nXOff + nXSize >= nXRight || |
482 | | // cppcheck-suppress knownConditionTrueFalse |
483 | 0 | (nXOff + nXSize == GetXSize() && nXRight > GetXSize())) && |
484 | 0 | (nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize || |
485 | 0 | (nYOff + nYSize == GetYSize() && |
486 | 0 | nLBlockY * nBlockYSize > GetYSize() - nBlockYSize))) |
487 | 0 | { |
488 | 0 | bJustInitialize = true; |
489 | 0 | bMemZeroBuffer = true; |
490 | 0 | } |
491 | | |
492 | | /* -------------------------------------------------------------------- |
493 | | */ |
494 | | /* Ensure we have the appropriate block loaded. */ |
495 | | /* -------------------------------------------------------------------- |
496 | | */ |
497 | 0 | const GUInt32 nErrorCounter = CPLGetErrorCounter(); |
498 | 0 | poBlock = |
499 | 0 | GetLockedBlockRef(nLBlockX, nLBlockY, bJustInitialize); |
500 | 0 | if (!poBlock) |
501 | 0 | { |
502 | 0 | if (strstr(CPLGetLastErrorMsg(), "IReadBlock failed") == |
503 | 0 | nullptr) |
504 | 0 | { |
505 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
506 | 0 | "GetBlockRef failed at X block offset %d, " |
507 | 0 | "Y block offset %d%s", |
508 | 0 | nLBlockX, nLBlockY, |
509 | 0 | (nErrorCounter != CPLGetErrorCounter()) |
510 | 0 | ? CPLSPrintf(": %s", CPLGetLastErrorMsg()) |
511 | 0 | : ""); |
512 | 0 | } |
513 | 0 | return (CE_Failure); |
514 | 0 | } |
515 | | |
516 | 0 | if (eRWFlag == GF_Write) |
517 | 0 | poBlock->MarkDirty(); |
518 | |
|
519 | 0 | pabySrcBlock = static_cast<GByte *>(poBlock->GetDataRef()); |
520 | 0 | if (bMemZeroBuffer) |
521 | 0 | { |
522 | 0 | memset(pabySrcBlock, 0, |
523 | 0 | static_cast<GPtrDiff_t>(nBandDataSize) * |
524 | 0 | nBlockXSize * nBlockYSize); |
525 | 0 | } |
526 | | /* -------------------------------------------------------------------- |
527 | | */ |
528 | | /* Copy over this chunk of data. */ |
529 | | /* -------------------------------------------------------------------- |
530 | | */ |
531 | 0 | GPtrDiff_t iSrcOffset = |
532 | 0 | (static_cast<GPtrDiff_t>(iSrcX) - |
533 | 0 | static_cast<GPtrDiff_t>(nLBlockX * nBlockXSize) + |
534 | 0 | (static_cast<GPtrDiff_t>(iSrcY) - |
535 | 0 | static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) * |
536 | 0 | nBlockXSize) * |
537 | 0 | nBandDataSize; |
538 | | // Fill up as many rows as possible for the loaded block. |
539 | 0 | const int kmax = std::min(nBlockYSize - (iSrcY % nBlockYSize), |
540 | 0 | nBufYSize - iBufYOff); |
541 | 0 | for (int k = 0; k < kmax; k++) |
542 | 0 | { |
543 | 0 | if (eDataType == eBufType && nPixelSpace == nBufDataSize) |
544 | 0 | { |
545 | 0 | if (eRWFlag == GF_Read) |
546 | 0 | memcpy(static_cast<GByte *>(pData) + iBufOffset + |
547 | 0 | static_cast<GPtrDiff_t>(k) * nLineSpace, |
548 | 0 | pabySrcBlock + iSrcOffset, nXSpanSize); |
549 | 0 | else |
550 | 0 | memcpy(pabySrcBlock + iSrcOffset, |
551 | 0 | static_cast<GByte *>(pData) + iBufOffset + |
552 | 0 | static_cast<GPtrDiff_t>(k) * nLineSpace, |
553 | 0 | nXSpanSize); |
554 | 0 | } |
555 | 0 | else |
556 | 0 | { |
557 | | /* type to type conversion */ |
558 | 0 | if (eRWFlag == GF_Read) |
559 | 0 | GDALCopyWords64( |
560 | 0 | pabySrcBlock + iSrcOffset, eDataType, |
561 | 0 | nBandDataSize, |
562 | 0 | static_cast<GByte *>(pData) + iBufOffset + |
563 | 0 | static_cast<GPtrDiff_t>(k) * nLineSpace, |
564 | 0 | eBufType, static_cast<int>(nPixelSpace), |
565 | 0 | nXSpan); |
566 | 0 | else |
567 | 0 | GDALCopyWords64( |
568 | 0 | static_cast<GByte *>(pData) + iBufOffset + |
569 | 0 | static_cast<GPtrDiff_t>(k) * nLineSpace, |
570 | 0 | eBufType, static_cast<int>(nPixelSpace), |
571 | 0 | pabySrcBlock + iSrcOffset, eDataType, |
572 | 0 | nBandDataSize, nXSpan); |
573 | 0 | } |
574 | |
|
575 | 0 | iSrcOffset += |
576 | 0 | static_cast<GPtrDiff_t>(nBlockXSize) * nBandDataSize; |
577 | 0 | } |
578 | |
|
579 | 0 | iBufOffset = |
580 | 0 | CPLUnsanitizedAdd<GPtrDiff_t>(iBufOffset, nXSpanSize); |
581 | 0 | nLBlockX++; |
582 | 0 | iSrcX += nXSpan; |
583 | |
|
584 | 0 | poBlock->DropLock(); |
585 | 0 | poBlock = nullptr; |
586 | 0 | } |
587 | | |
588 | | /* Compute the increment to go on a block boundary */ |
589 | 0 | const int nYInc = nBlockYSize - (iSrcY % nBlockYSize); |
590 | |
|
591 | 0 | if (psExtraArg->pfnProgress != nullptr && |
592 | 0 | !psExtraArg->pfnProgress( |
593 | 0 | 1.0 * std::min(nBufYSize, iBufYOff + nYInc) / nBufYSize, "", |
594 | 0 | psExtraArg->pProgressData)) |
595 | 0 | { |
596 | 0 | return CE_Failure; |
597 | 0 | } |
598 | | |
599 | 0 | iBufYOff += nYInc; |
600 | 0 | if (iBufYOff >= nBufYSize) |
601 | 0 | break; |
602 | | // Only increment iSrcY after above loop end check, to avoid |
603 | | // potential int overflow. |
604 | 0 | iSrcY += nYInc; |
605 | 0 | } |
606 | | |
607 | 0 | return CE_None; |
608 | 0 | } |
609 | | |
610 | | /* ==================================================================== */ |
611 | | /* Loop reading required source blocks to satisfy output */ |
612 | | /* request. This is the most general implementation. */ |
613 | | /* ==================================================================== */ |
614 | | |
615 | 0 | double dfXOff = nXOff; |
616 | 0 | double dfYOff = nYOff; |
617 | 0 | double dfXSize = nXSize; |
618 | 0 | double dfYSize = nYSize; |
619 | 0 | if (psExtraArg->bFloatingPointWindowValidity) |
620 | 0 | { |
621 | 0 | dfXOff = psExtraArg->dfXOff; |
622 | 0 | dfYOff = psExtraArg->dfYOff; |
623 | 0 | dfXSize = psExtraArg->dfXSize; |
624 | 0 | dfYSize = psExtraArg->dfYSize; |
625 | 0 | } |
626 | | |
627 | | /* -------------------------------------------------------------------- */ |
628 | | /* Compute stepping increment. */ |
629 | | /* -------------------------------------------------------------------- */ |
630 | 0 | const double dfSrcXInc = dfXSize / static_cast<double>(nBufXSize); |
631 | 0 | const double dfSrcYInc = dfYSize / static_cast<double>(nBufYSize); |
632 | 0 | CPLErr eErr = CE_None; |
633 | |
|
634 | 0 | if (eRWFlag == GF_Write) |
635 | 0 | { |
636 | | /* -------------------------------------------------------------------- |
637 | | */ |
638 | | /* Write case */ |
639 | | /* Loop over raster window computing source locations in the buffer. |
640 | | */ |
641 | | /* -------------------------------------------------------------------- |
642 | | */ |
643 | 0 | GByte *pabyDstBlock = nullptr; |
644 | 0 | int nLBlockX = -1; |
645 | 0 | int nLBlockY = -1; |
646 | |
|
647 | 0 | for (int iDstY = nYOff; iDstY < nYOff + nYSize; iDstY++) |
648 | 0 | { |
649 | 0 | const int iBufYOff = static_cast<int>((iDstY - nYOff) / dfSrcYInc); |
650 | |
|
651 | 0 | for (int iDstX = nXOff; iDstX < nXOff + nXSize; iDstX++) |
652 | 0 | { |
653 | 0 | const int iBufXOff = |
654 | 0 | static_cast<int>((iDstX - nXOff) / dfSrcXInc); |
655 | 0 | GPtrDiff_t iBufOffset = |
656 | 0 | static_cast<GPtrDiff_t>(iBufYOff) * |
657 | 0 | static_cast<GPtrDiff_t>(nLineSpace) + |
658 | 0 | iBufXOff * static_cast<GPtrDiff_t>(nPixelSpace); |
659 | | |
660 | | // FIXME: this code likely doesn't work if the dirty block gets |
661 | | // flushed to disk before being completely written. |
662 | | // In the meantime, bJustInitialize should probably be set to |
663 | | // FALSE even if it is not ideal performance wise, and for |
664 | | // lossy compression. |
665 | | |
666 | | /* -------------------------------------------------------------------- |
667 | | */ |
668 | | /* Ensure we have the appropriate block loaded. */ |
669 | | /* -------------------------------------------------------------------- |
670 | | */ |
671 | 0 | if (iDstX < nLBlockX * nBlockXSize || |
672 | 0 | iDstX - nBlockXSize >= nLBlockX * nBlockXSize || |
673 | 0 | iDstY < nLBlockY * nBlockYSize || |
674 | 0 | iDstY - nBlockYSize >= nLBlockY * nBlockYSize) |
675 | 0 | { |
676 | 0 | nLBlockX = iDstX / nBlockXSize; |
677 | 0 | nLBlockY = iDstY / nBlockYSize; |
678 | |
|
679 | 0 | const bool bJustInitialize = |
680 | 0 | nYOff <= nLBlockY * nBlockYSize && |
681 | 0 | nYOff + nYSize - nBlockYSize >= |
682 | 0 | nLBlockY * nBlockYSize && |
683 | 0 | nXOff <= nLBlockX * nBlockXSize && |
684 | 0 | nXOff + nXSize - nBlockXSize >= nLBlockX * nBlockXSize; |
685 | | /*bool bMemZeroBuffer = FALSE; |
686 | | if( !bJustInitialize && |
687 | | nXOff <= nLBlockX * nBlockXSize && |
688 | | nYOff <= nLBlockY * nBlockYSize && |
689 | | (nXOff + nXSize >= (nLBlockX+1) * nBlockXSize || |
690 | | (nXOff + nXSize == GetXSize() && |
691 | | (nLBlockX+1) * nBlockXSize > GetXSize())) && |
692 | | (nYOff + nYSize >= (nLBlockY+1) * nBlockYSize || |
693 | | (nYOff + nYSize == GetYSize() && |
694 | | (nLBlockY+1) * nBlockYSize > GetYSize())) ) |
695 | | { |
696 | | bJustInitialize = TRUE; |
697 | | bMemZeroBuffer = TRUE; |
698 | | }*/ |
699 | 0 | if (poBlock != nullptr) |
700 | 0 | poBlock->DropLock(); |
701 | |
|
702 | 0 | poBlock = |
703 | 0 | GetLockedBlockRef(nLBlockX, nLBlockY, bJustInitialize); |
704 | 0 | if (poBlock == nullptr) |
705 | 0 | { |
706 | 0 | return (CE_Failure); |
707 | 0 | } |
708 | | |
709 | 0 | poBlock->MarkDirty(); |
710 | |
|
711 | 0 | pabyDstBlock = static_cast<GByte *>(poBlock->GetDataRef()); |
712 | | /*if( bMemZeroBuffer ) |
713 | | { |
714 | | memset(pabyDstBlock, 0, |
715 | | static_cast<GPtrDiff_t>(nBandDataSize) * nBlockXSize |
716 | | * nBlockYSize); |
717 | | }*/ |
718 | 0 | } |
719 | | |
720 | | // To make Coverity happy. Should not happen by design. |
721 | 0 | if (pabyDstBlock == nullptr) |
722 | 0 | { |
723 | 0 | CPLAssert(false); |
724 | 0 | eErr = CE_Failure; |
725 | 0 | break; |
726 | 0 | } |
727 | | |
728 | | /* -------------------------------------------------------------------- |
729 | | */ |
730 | | /* Copy over this pixel of data. */ |
731 | | /* -------------------------------------------------------------------- |
732 | | */ |
733 | 0 | GPtrDiff_t iDstOffset = |
734 | 0 | (static_cast<GPtrDiff_t>(iDstX) - |
735 | 0 | static_cast<GPtrDiff_t>(nLBlockX) * nBlockXSize + |
736 | 0 | (static_cast<GPtrDiff_t>(iDstY) - |
737 | 0 | static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) * |
738 | 0 | nBlockXSize) * |
739 | 0 | nBandDataSize; |
740 | |
|
741 | 0 | if (eDataType == eBufType) |
742 | 0 | { |
743 | 0 | memcpy(pabyDstBlock + iDstOffset, |
744 | 0 | static_cast<GByte *>(pData) + iBufOffset, |
745 | 0 | nBandDataSize); |
746 | 0 | } |
747 | 0 | else |
748 | 0 | { |
749 | | /* type to type conversion ... ouch, this is expensive way |
750 | | of handling single words */ |
751 | 0 | GDALCopyWords64(static_cast<GByte *>(pData) + iBufOffset, |
752 | 0 | eBufType, 0, pabyDstBlock + iDstOffset, |
753 | 0 | eDataType, 0, 1); |
754 | 0 | } |
755 | 0 | } |
756 | | |
757 | 0 | if (psExtraArg->pfnProgress != nullptr && |
758 | 0 | !psExtraArg->pfnProgress(1.0 * (iDstY - nYOff + 1) / nYSize, "", |
759 | 0 | psExtraArg->pProgressData)) |
760 | 0 | { |
761 | 0 | eErr = CE_Failure; |
762 | 0 | break; |
763 | 0 | } |
764 | 0 | } |
765 | 0 | } |
766 | 0 | else |
767 | 0 | { |
768 | 0 | if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) |
769 | 0 | { |
770 | 0 | if ((psExtraArg->eResampleAlg == GRIORA_Cubic || |
771 | 0 | psExtraArg->eResampleAlg == GRIORA_CubicSpline || |
772 | 0 | psExtraArg->eResampleAlg == GRIORA_Bilinear || |
773 | 0 | psExtraArg->eResampleAlg == GRIORA_Lanczos) && |
774 | 0 | GetColorTable() != nullptr) |
775 | 0 | { |
776 | 0 | CPLError(CE_Warning, CPLE_NotSupported, |
777 | 0 | "Resampling method not supported on paletted band. " |
778 | 0 | "Falling back to nearest neighbour"); |
779 | 0 | } |
780 | 0 | else if (psExtraArg->eResampleAlg == GRIORA_Gauss && |
781 | 0 | GDALDataTypeIsComplex(eDataType)) |
782 | 0 | { |
783 | 0 | CPLError(CE_Warning, CPLE_NotSupported, |
784 | 0 | "Resampling method not supported on complex data type " |
785 | 0 | "band. Falling back to nearest neighbour"); |
786 | 0 | } |
787 | 0 | else |
788 | 0 | { |
789 | 0 | return RasterIOResampled(eRWFlag, nXOff, nYOff, nXSize, nYSize, |
790 | 0 | pData, nBufXSize, nBufYSize, eBufType, |
791 | 0 | nPixelSpace, nLineSpace, psExtraArg); |
792 | 0 | } |
793 | 0 | } |
794 | | |
795 | 0 | int nLimitBlockY = 0; |
796 | 0 | const bool bByteCopy = eDataType == eBufType && nBandDataSize == 1; |
797 | 0 | int nStartBlockX = -nBlockXSize; |
798 | 0 | constexpr double EPS = 1e-10; |
799 | 0 | int nLBlockY = -1; |
800 | 0 | const double dfSrcXStart = 0.5 * dfSrcXInc + dfXOff + EPS; |
801 | 0 | const bool bIntegerXFactor = |
802 | 0 | bUseIntegerRequestCoords && |
803 | 0 | static_cast<int>(dfSrcXInc) == dfSrcXInc && |
804 | 0 | static_cast<int>(dfSrcXInc) < INT_MAX / nBandDataSize; |
805 | | |
806 | | /* -------------------------------------------------------------------- |
807 | | */ |
808 | | /* Read case */ |
809 | | /* Loop over buffer computing source locations. */ |
810 | | /* -------------------------------------------------------------------- |
811 | | */ |
812 | 0 | for (int iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++) |
813 | 0 | { |
814 | | // Add small epsilon to avoid some numeric precision issues. |
815 | 0 | const double dfSrcY = (iBufYOff + 0.5) * dfSrcYInc + dfYOff + EPS; |
816 | 0 | const int iSrcY = static_cast<int>(std::min( |
817 | 0 | std::max(0.0, dfSrcY), static_cast<double>(nRasterYSize - 1))); |
818 | |
|
819 | 0 | GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) * |
820 | 0 | static_cast<GPtrDiff_t>(nLineSpace); |
821 | |
|
822 | 0 | if (iSrcY >= nLimitBlockY) |
823 | 0 | { |
824 | 0 | nLBlockY = iSrcY / nBlockYSize; |
825 | 0 | nLimitBlockY = nLBlockY * nBlockYSize; |
826 | 0 | if (nLimitBlockY < INT_MAX - nBlockYSize) |
827 | 0 | nLimitBlockY += nBlockYSize; |
828 | 0 | else |
829 | 0 | nLimitBlockY = INT_MAX; |
830 | | // Make sure a new block is loaded. |
831 | 0 | nStartBlockX = -nBlockXSize; |
832 | 0 | } |
833 | 0 | else if (static_cast<int>(dfSrcXStart) < nStartBlockX) |
834 | 0 | { |
835 | | // Make sure a new block is loaded. |
836 | 0 | nStartBlockX = -nBlockXSize; |
837 | 0 | } |
838 | |
|
839 | 0 | GPtrDiff_t iSrcOffsetCst = (iSrcY - nLBlockY * nBlockYSize) * |
840 | 0 | static_cast<GPtrDiff_t>(nBlockXSize); |
841 | |
|
842 | 0 | if (bIntegerXFactor) |
843 | 0 | { |
844 | 0 | int iSrcX = static_cast<int>(dfSrcXStart); |
845 | 0 | const int nSrcXInc = static_cast<int>(dfSrcXInc); |
846 | 0 | GByte *pabyDstData = static_cast<GByte *>(pData) + iBufOffset; |
847 | 0 | bool bRet = false; |
848 | 0 | if (bByteCopy) |
849 | 0 | { |
850 | 0 | bRet = DownsamplingIntegerXFactor<true, 1>( |
851 | 0 | this, iSrcX, nSrcXInc, iSrcOffsetCst, pabyDstData, |
852 | 0 | static_cast<int>(nPixelSpace), nBufXSize, GDT_UInt8, |
853 | 0 | GDT_UInt8, nStartBlockX, nBlockXSize, poBlock, |
854 | 0 | nLBlockY); |
855 | 0 | } |
856 | 0 | else if (eDataType == eBufType) |
857 | 0 | { |
858 | 0 | switch (nBandDataSize) |
859 | 0 | { |
860 | 0 | case 2: |
861 | 0 | bRet = DownsamplingIntegerXFactor<true, 2>( |
862 | 0 | this, iSrcX, nSrcXInc, iSrcOffsetCst, |
863 | 0 | pabyDstData, static_cast<int>(nPixelSpace), |
864 | 0 | nBufXSize, eDataType, eDataType, nStartBlockX, |
865 | 0 | nBlockXSize, poBlock, nLBlockY); |
866 | 0 | break; |
867 | 0 | case 4: |
868 | 0 | bRet = DownsamplingIntegerXFactor<true, 4>( |
869 | 0 | this, iSrcX, nSrcXInc, iSrcOffsetCst, |
870 | 0 | pabyDstData, static_cast<int>(nPixelSpace), |
871 | 0 | nBufXSize, eDataType, eDataType, nStartBlockX, |
872 | 0 | nBlockXSize, poBlock, nLBlockY); |
873 | 0 | break; |
874 | 0 | case 8: |
875 | 0 | bRet = DownsamplingIntegerXFactor<true, 8>( |
876 | 0 | this, iSrcX, nSrcXInc, iSrcOffsetCst, |
877 | 0 | pabyDstData, static_cast<int>(nPixelSpace), |
878 | 0 | nBufXSize, eDataType, eDataType, nStartBlockX, |
879 | 0 | nBlockXSize, poBlock, nLBlockY); |
880 | 0 | break; |
881 | 0 | case 16: |
882 | 0 | bRet = DownsamplingIntegerXFactor<true, 16>( |
883 | 0 | this, iSrcX, nSrcXInc, iSrcOffsetCst, |
884 | 0 | pabyDstData, static_cast<int>(nPixelSpace), |
885 | 0 | nBufXSize, eDataType, eDataType, nStartBlockX, |
886 | 0 | nBlockXSize, poBlock, nLBlockY); |
887 | 0 | break; |
888 | 0 | default: |
889 | 0 | CPLAssert(false); |
890 | 0 | break; |
891 | 0 | } |
892 | 0 | } |
893 | 0 | else |
894 | 0 | { |
895 | 0 | bRet = DownsamplingIntegerXFactor<false, 0>( |
896 | 0 | this, iSrcX, nSrcXInc, iSrcOffsetCst, pabyDstData, |
897 | 0 | static_cast<int>(nPixelSpace), nBufXSize, eDataType, |
898 | 0 | eBufType, nStartBlockX, nBlockXSize, poBlock, nLBlockY); |
899 | 0 | } |
900 | 0 | if (!bRet) |
901 | 0 | eErr = CE_Failure; |
902 | 0 | } |
903 | 0 | else |
904 | 0 | { |
905 | 0 | double dfSrcX = dfSrcXStart; |
906 | 0 | for (int iBufXOff = 0; iBufXOff < nBufXSize; |
907 | 0 | iBufXOff++, dfSrcX += dfSrcXInc) |
908 | 0 | { |
909 | | // TODO?: try to avoid the clamping for most iterations |
910 | 0 | const int iSrcX = static_cast<int>( |
911 | 0 | std::min(std::max(0.0, dfSrcX), |
912 | 0 | static_cast<double>(nRasterXSize - 1))); |
913 | | |
914 | | /* -------------------------------------------------------------------- |
915 | | */ |
916 | | /* Ensure we have the appropriate block loaded. */ |
917 | | /* -------------------------------------------------------------------- |
918 | | */ |
919 | 0 | if (iSrcX >= nBlockXSize + nStartBlockX) |
920 | 0 | { |
921 | 0 | const int nLBlockX = iSrcX / nBlockXSize; |
922 | 0 | nStartBlockX = nLBlockX * nBlockXSize; |
923 | |
|
924 | 0 | if (poBlock != nullptr) |
925 | 0 | poBlock->DropLock(); |
926 | |
|
927 | 0 | poBlock = GetLockedBlockRef(nLBlockX, nLBlockY, FALSE); |
928 | 0 | if (poBlock == nullptr) |
929 | 0 | { |
930 | 0 | eErr = CE_Failure; |
931 | 0 | break; |
932 | 0 | } |
933 | | |
934 | 0 | pabySrcBlock = |
935 | 0 | static_cast<GByte *>(poBlock->GetDataRef()); |
936 | 0 | } |
937 | 0 | const GPtrDiff_t nDiffX = |
938 | 0 | static_cast<GPtrDiff_t>(iSrcX - nStartBlockX); |
939 | | |
940 | | /* -------------------------------------------------------------------- |
941 | | */ |
942 | | /* Copy over this pixel of data. */ |
943 | | /* -------------------------------------------------------------------- |
944 | | */ |
945 | |
|
946 | 0 | if (bByteCopy) |
947 | 0 | { |
948 | 0 | GPtrDiff_t iSrcOffset = nDiffX + iSrcOffsetCst; |
949 | 0 | static_cast<GByte *>(pData)[iBufOffset] = |
950 | 0 | pabySrcBlock[iSrcOffset]; |
951 | 0 | } |
952 | 0 | else if (eDataType == eBufType) |
953 | 0 | { |
954 | 0 | GPtrDiff_t iSrcOffset = |
955 | 0 | (nDiffX + iSrcOffsetCst) * nBandDataSize; |
956 | 0 | memcpy(static_cast<GByte *>(pData) + iBufOffset, |
957 | 0 | pabySrcBlock + iSrcOffset, nBandDataSize); |
958 | 0 | } |
959 | 0 | else |
960 | 0 | { |
961 | | // Type to type conversion ... |
962 | 0 | GPtrDiff_t iSrcOffset = |
963 | 0 | (nDiffX + iSrcOffsetCst) * nBandDataSize; |
964 | 0 | GDALCopyWords64(pabySrcBlock + iSrcOffset, eDataType, 0, |
965 | 0 | static_cast<GByte *>(pData) + |
966 | 0 | iBufOffset, |
967 | 0 | eBufType, 0, 1); |
968 | 0 | } |
969 | |
|
970 | 0 | iBufOffset += static_cast<int>(nPixelSpace); |
971 | 0 | } |
972 | 0 | } |
973 | 0 | if (eErr == CE_Failure) |
974 | 0 | break; |
975 | | |
976 | 0 | if (psExtraArg->pfnProgress != nullptr && |
977 | 0 | !psExtraArg->pfnProgress(1.0 * (iBufYOff + 1) / nBufYSize, "", |
978 | 0 | psExtraArg->pProgressData)) |
979 | 0 | { |
980 | 0 | eErr = CE_Failure; |
981 | 0 | break; |
982 | 0 | } |
983 | 0 | } |
984 | 0 | } |
985 | | |
986 | 0 | if (poBlock != nullptr) |
987 | 0 | poBlock->DropLock(); |
988 | |
|
989 | 0 | return eErr; |
990 | 0 | } |
991 | | |
992 | | /************************************************************************/ |
993 | | /* GDALRasterIOTransformer() */ |
994 | | /************************************************************************/ |
995 | | |
996 | | struct GDALRasterIOTransformerStruct |
997 | | { |
998 | | double dfXOff; |
999 | | double dfYOff; |
1000 | | double dfXRatioDstToSrc; |
1001 | | double dfYRatioDstToSrc; |
1002 | | }; |
1003 | | |
1004 | | static int GDALRasterIOTransformer(void *pTransformerArg, int bDstToSrc, |
1005 | | int nPointCount, double *x, double *y, |
1006 | | double * /* z */, int *panSuccess) |
1007 | 0 | { |
1008 | 0 | GDALRasterIOTransformerStruct *psParams = |
1009 | 0 | static_cast<GDALRasterIOTransformerStruct *>(pTransformerArg); |
1010 | 0 | if (bDstToSrc) |
1011 | 0 | { |
1012 | 0 | for (int i = 0; i < nPointCount; i++) |
1013 | 0 | { |
1014 | 0 | x[i] = x[i] * psParams->dfXRatioDstToSrc + psParams->dfXOff; |
1015 | 0 | y[i] = y[i] * psParams->dfYRatioDstToSrc + psParams->dfYOff; |
1016 | 0 | panSuccess[i] = TRUE; |
1017 | 0 | } |
1018 | 0 | } |
1019 | 0 | else |
1020 | 0 | { |
1021 | 0 | for (int i = 0; i < nPointCount; i++) |
1022 | 0 | { |
1023 | 0 | x[i] = (x[i] - psParams->dfXOff) / psParams->dfXRatioDstToSrc; |
1024 | 0 | y[i] = (y[i] - psParams->dfYOff) / psParams->dfYRatioDstToSrc; |
1025 | 0 | panSuccess[i] = TRUE; |
1026 | 0 | } |
1027 | 0 | } |
1028 | 0 | return TRUE; |
1029 | 0 | } |
1030 | | |
1031 | | /************************************************************************/ |
1032 | | /* RasterIOResampled() */ |
1033 | | /************************************************************************/ |
1034 | | |
1035 | | //! @cond Doxygen_Suppress |
1036 | | CPLErr GDALRasterBand::RasterIOResampled( |
1037 | | GDALRWFlag /* eRWFlag */, int nXOff, int nYOff, int nXSize, int nYSize, |
1038 | | void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, |
1039 | | GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg) |
1040 | 0 | { |
1041 | | // Determine if we use warping resampling or overview resampling |
1042 | 0 | const bool bUseWarp = |
1043 | 0 | (GDALDataTypeIsComplex(eDataType) && |
1044 | 0 | psExtraArg->eResampleAlg != GRIORA_NearestNeighbour && |
1045 | 0 | psExtraArg->eResampleAlg != GRIORA_Mode); |
1046 | |
|
1047 | 0 | double dfXOff = nXOff; |
1048 | 0 | double dfYOff = nYOff; |
1049 | 0 | double dfXSize = nXSize; |
1050 | 0 | double dfYSize = nYSize; |
1051 | 0 | if (psExtraArg->bFloatingPointWindowValidity) |
1052 | 0 | { |
1053 | 0 | dfXOff = psExtraArg->dfXOff; |
1054 | 0 | dfYOff = psExtraArg->dfYOff; |
1055 | 0 | dfXSize = psExtraArg->dfXSize; |
1056 | 0 | dfYSize = psExtraArg->dfYSize; |
1057 | 0 | } |
1058 | |
|
1059 | 0 | const double dfXRatioDstToSrc = dfXSize / nBufXSize; |
1060 | 0 | const double dfYRatioDstToSrc = dfYSize / nBufYSize; |
1061 | | |
1062 | | // Determine the coordinates in the "virtual" output raster to see |
1063 | | // if there are not integers, in which case we will use them as a shift |
1064 | | // so that subwindow extracts give the exact same results as entire raster |
1065 | | // scaling. |
1066 | 0 | double dfDestXOff = dfXOff / dfXRatioDstToSrc; |
1067 | 0 | bool bHasXOffVirtual = false; |
1068 | 0 | int nDestXOffVirtual = 0; |
1069 | 0 | if (fabs(dfDestXOff - static_cast<int>(dfDestXOff + 0.5)) < 1e-8) |
1070 | 0 | { |
1071 | 0 | bHasXOffVirtual = true; |
1072 | 0 | dfXOff = nXOff; |
1073 | 0 | nDestXOffVirtual = static_cast<int>(dfDestXOff + 0.5); |
1074 | 0 | } |
1075 | |
|
1076 | 0 | double dfDestYOff = dfYOff / dfYRatioDstToSrc; |
1077 | 0 | bool bHasYOffVirtual = false; |
1078 | 0 | int nDestYOffVirtual = 0; |
1079 | 0 | if (fabs(dfDestYOff - static_cast<int>(dfDestYOff + 0.5)) < 1e-8) |
1080 | 0 | { |
1081 | 0 | bHasYOffVirtual = true; |
1082 | 0 | dfYOff = nYOff; |
1083 | 0 | nDestYOffVirtual = static_cast<int>(dfDestYOff + 0.5); |
1084 | 0 | } |
1085 | | |
1086 | | // Create a MEM dataset that wraps the output buffer. |
1087 | 0 | GDALDataset *poMEMDS; |
1088 | 0 | void *pTempBuffer = nullptr; |
1089 | 0 | GSpacing nPSMem = nPixelSpace; |
1090 | 0 | GSpacing nLSMem = nLineSpace; |
1091 | 0 | void *pDataMem = pData; |
1092 | 0 | GDALDataType eDTMem = eBufType; |
1093 | 0 | if (eBufType != eDataType && !GDAL_GET_OPERATE_IN_BUF_TYPE(*psExtraArg)) |
1094 | 0 | { |
1095 | 0 | nPSMem = GDALGetDataTypeSizeBytes(eDataType); |
1096 | 0 | nLSMem = nPSMem * nBufXSize; |
1097 | 0 | pTempBuffer = |
1098 | 0 | VSI_MALLOC2_VERBOSE(nBufYSize, static_cast<size_t>(nLSMem)); |
1099 | 0 | if (pTempBuffer == nullptr) |
1100 | 0 | return CE_Failure; |
1101 | 0 | pDataMem = pTempBuffer; |
1102 | 0 | eDTMem = eDataType; |
1103 | 0 | } |
1104 | | |
1105 | 0 | poMEMDS = |
1106 | 0 | MEMDataset::Create("", nDestXOffVirtual + nBufXSize, |
1107 | 0 | nDestYOffVirtual + nBufYSize, 0, eDTMem, nullptr); |
1108 | 0 | GByte *pabyData = static_cast<GByte *>(pDataMem) - |
1109 | 0 | nPSMem * nDestXOffVirtual - nLSMem * nDestYOffVirtual; |
1110 | 0 | GDALRasterBandH hMEMBand = MEMCreateRasterBandEx( |
1111 | 0 | poMEMDS, 1, pabyData, eDTMem, nPSMem, nLSMem, false); |
1112 | 0 | poMEMDS->SetBand(1, GDALRasterBand::FromHandle(hMEMBand)); |
1113 | |
|
1114 | 0 | const char *pszNBITS = |
1115 | 0 | GetMetadataItem(GDALMD_NBITS, GDAL_MDD_IMAGE_STRUCTURE); |
1116 | 0 | const int nNBITS = pszNBITS ? atoi(pszNBITS) : 0; |
1117 | 0 | if (pszNBITS) |
1118 | 0 | GDALRasterBand::FromHandle(hMEMBand)->SetMetadataItem( |
1119 | 0 | GDALMD_NBITS, pszNBITS, GDAL_MDD_IMAGE_STRUCTURE); |
1120 | |
|
1121 | 0 | CPLErr eErr = CE_None; |
1122 | | |
1123 | | // Do the resampling. |
1124 | 0 | if (bUseWarp) |
1125 | 0 | { |
1126 | 0 | int bHasNoData = FALSE; |
1127 | 0 | double dfNoDataValue = GetNoDataValue(&bHasNoData); |
1128 | |
|
1129 | 0 | VRTDatasetH hVRTDS = nullptr; |
1130 | 0 | GDALRasterBandH hVRTBand = nullptr; |
1131 | 0 | if (GetDataset() == nullptr) |
1132 | 0 | { |
1133 | | /* Create VRT dataset that wraps the whole dataset */ |
1134 | 0 | hVRTDS = VRTCreate(nRasterXSize, nRasterYSize); |
1135 | 0 | VRTAddBand(hVRTDS, eDataType, nullptr); |
1136 | 0 | hVRTBand = GDALGetRasterBand(hVRTDS, 1); |
1137 | 0 | VRTAddSimpleSource(hVRTBand, this, 0, 0, nRasterXSize, nRasterYSize, |
1138 | 0 | 0, 0, nRasterXSize, nRasterYSize, nullptr, |
1139 | 0 | VRT_NODATA_UNSET); |
1140 | | |
1141 | | /* Add a mask band if needed */ |
1142 | 0 | if (GetMaskFlags() != GMF_ALL_VALID) |
1143 | 0 | { |
1144 | 0 | GDALDataset::FromHandle(hVRTDS)->CreateMaskBand(0); |
1145 | 0 | VRTSourcedRasterBand *poVRTMaskBand = |
1146 | 0 | reinterpret_cast<VRTSourcedRasterBand *>( |
1147 | 0 | reinterpret_cast<GDALRasterBand *>(hVRTBand) |
1148 | 0 | ->GetMaskBand()); |
1149 | 0 | poVRTMaskBand->AddMaskBandSource(this, 0, 0, nRasterXSize, |
1150 | 0 | nRasterYSize, 0, 0, |
1151 | 0 | nRasterXSize, nRasterYSize); |
1152 | 0 | } |
1153 | 0 | } |
1154 | |
|
1155 | 0 | GDALWarpOptions *psWarpOptions = GDALCreateWarpOptions(); |
1156 | 0 | switch (psExtraArg->eResampleAlg) |
1157 | 0 | { |
1158 | 0 | case GRIORA_NearestNeighbour: |
1159 | 0 | psWarpOptions->eResampleAlg = GRA_NearestNeighbour; |
1160 | 0 | break; |
1161 | 0 | case GRIORA_Bilinear: |
1162 | 0 | psWarpOptions->eResampleAlg = GRA_Bilinear; |
1163 | 0 | break; |
1164 | 0 | case GRIORA_Cubic: |
1165 | 0 | psWarpOptions->eResampleAlg = GRA_Cubic; |
1166 | 0 | break; |
1167 | 0 | case GRIORA_CubicSpline: |
1168 | 0 | psWarpOptions->eResampleAlg = GRA_CubicSpline; |
1169 | 0 | break; |
1170 | 0 | case GRIORA_Lanczos: |
1171 | 0 | psWarpOptions->eResampleAlg = GRA_Lanczos; |
1172 | 0 | break; |
1173 | 0 | case GRIORA_Average: |
1174 | 0 | psWarpOptions->eResampleAlg = GRA_Average; |
1175 | 0 | break; |
1176 | 0 | case GRIORA_RMS: |
1177 | 0 | psWarpOptions->eResampleAlg = GRA_RMS; |
1178 | 0 | break; |
1179 | 0 | case GRIORA_Mode: |
1180 | 0 | psWarpOptions->eResampleAlg = GRA_Mode; |
1181 | 0 | break; |
1182 | 0 | default: |
1183 | 0 | CPLAssert(false); |
1184 | 0 | psWarpOptions->eResampleAlg = GRA_NearestNeighbour; |
1185 | 0 | break; |
1186 | 0 | } |
1187 | 0 | psWarpOptions->hSrcDS = hVRTDS ? hVRTDS : GetDataset(); |
1188 | 0 | psWarpOptions->hDstDS = poMEMDS; |
1189 | 0 | psWarpOptions->nBandCount = 1; |
1190 | 0 | int nSrcBandNumber = hVRTDS ? 1 : nBand; |
1191 | 0 | int nDstBandNumber = 1; |
1192 | 0 | psWarpOptions->panSrcBands = &nSrcBandNumber; |
1193 | 0 | psWarpOptions->panDstBands = &nDstBandNumber; |
1194 | 0 | psWarpOptions->pfnProgress = psExtraArg->pfnProgress |
1195 | 0 | ? psExtraArg->pfnProgress |
1196 | 0 | : GDALDummyProgress; |
1197 | 0 | psWarpOptions->pProgressArg = psExtraArg->pProgressData; |
1198 | 0 | psWarpOptions->pfnTransformer = GDALRasterIOTransformer; |
1199 | 0 | if (bHasNoData) |
1200 | 0 | { |
1201 | 0 | psWarpOptions->papszWarpOptions = CSLSetNameValue( |
1202 | 0 | psWarpOptions->papszWarpOptions, "INIT_DEST", "NO_DATA"); |
1203 | 0 | if (psWarpOptions->padfSrcNoDataReal == nullptr) |
1204 | 0 | { |
1205 | 0 | psWarpOptions->padfSrcNoDataReal = |
1206 | 0 | static_cast<double *>(CPLMalloc(sizeof(double))); |
1207 | 0 | psWarpOptions->padfSrcNoDataReal[0] = dfNoDataValue; |
1208 | 0 | } |
1209 | |
|
1210 | 0 | if (psWarpOptions->padfDstNoDataReal == nullptr) |
1211 | 0 | { |
1212 | 0 | psWarpOptions->padfDstNoDataReal = |
1213 | 0 | static_cast<double *>(CPLMalloc(sizeof(double))); |
1214 | 0 | psWarpOptions->padfDstNoDataReal[0] = dfNoDataValue; |
1215 | 0 | } |
1216 | 0 | } |
1217 | |
|
1218 | 0 | GDALRasterIOTransformerStruct sTransformer; |
1219 | 0 | sTransformer.dfXOff = bHasXOffVirtual ? 0 : dfXOff; |
1220 | 0 | sTransformer.dfYOff = bHasYOffVirtual ? 0 : dfYOff; |
1221 | 0 | sTransformer.dfXRatioDstToSrc = dfXRatioDstToSrc; |
1222 | 0 | sTransformer.dfYRatioDstToSrc = dfYRatioDstToSrc; |
1223 | 0 | psWarpOptions->pTransformerArg = &sTransformer; |
1224 | |
|
1225 | 0 | GDALWarpOperationH hWarpOperation = |
1226 | 0 | GDALCreateWarpOperation(psWarpOptions); |
1227 | 0 | eErr = GDALChunkAndWarpImage(hWarpOperation, nDestXOffVirtual, |
1228 | 0 | nDestYOffVirtual, nBufXSize, nBufYSize); |
1229 | 0 | GDALDestroyWarpOperation(hWarpOperation); |
1230 | |
|
1231 | 0 | psWarpOptions->panSrcBands = nullptr; |
1232 | 0 | psWarpOptions->panDstBands = nullptr; |
1233 | 0 | GDALDestroyWarpOptions(psWarpOptions); |
1234 | |
|
1235 | 0 | if (hVRTDS) |
1236 | 0 | GDALClose(hVRTDS); |
1237 | 0 | } |
1238 | 0 | else |
1239 | 0 | { |
1240 | 0 | const char *pszResampling = |
1241 | 0 | GDALRasterIOGetResampleAlg(psExtraArg->eResampleAlg); |
1242 | 0 | int nKernelRadius = 0; |
1243 | 0 | GDALResampleFunction pfnResampleFunc = |
1244 | 0 | GDALGetResampleFunction(pszResampling, &nKernelRadius); |
1245 | 0 | CPLAssert(pfnResampleFunc); |
1246 | 0 | GDALDataType eWrkDataType = |
1247 | 0 | GDALGetOvrWorkDataType(pszResampling, eDataType); |
1248 | 0 | int nHasNoData = 0; |
1249 | 0 | double dfNoDataValue = GetNoDataValue(&nHasNoData); |
1250 | 0 | const bool bHasNoData = CPL_TO_BOOL(nHasNoData); |
1251 | 0 | if (!bHasNoData) |
1252 | 0 | dfNoDataValue = 0.0; |
1253 | |
|
1254 | 0 | int nDstBlockXSize = nBufXSize; |
1255 | 0 | int nDstBlockYSize = nBufYSize; |
1256 | 0 | int nFullResXChunk = 0; |
1257 | 0 | int nFullResYChunk = 0; |
1258 | 0 | while (true) |
1259 | 0 | { |
1260 | 0 | nFullResXChunk = static_cast<int>(std::min<double>( |
1261 | 0 | 3 + nDstBlockXSize * dfXRatioDstToSrc, nRasterXSize)); |
1262 | 0 | nFullResYChunk = static_cast<int>(std::min<double>( |
1263 | 0 | 3 + nDstBlockYSize * dfYRatioDstToSrc, nRasterYSize)); |
1264 | 0 | if ((nDstBlockXSize == 1 && nDstBlockYSize == 1) || |
1265 | 0 | (static_cast<GIntBig>(nFullResXChunk) * nFullResYChunk <= |
1266 | 0 | 1024 * 1024)) |
1267 | 0 | break; |
1268 | | // When operating on the full width of a raster whose block width is |
1269 | | // the raster width, prefer doing chunks in height. |
1270 | 0 | if (nFullResXChunk >= nXSize && nXSize == nBlockXSize && |
1271 | 0 | nDstBlockYSize > 1) |
1272 | 0 | nDstBlockYSize /= 2; |
1273 | | /* Otherwise cut the maximal dimension */ |
1274 | 0 | else if (nDstBlockXSize > 1 && |
1275 | 0 | (nFullResXChunk > nFullResYChunk || nDstBlockYSize == 1)) |
1276 | 0 | nDstBlockXSize /= 2; |
1277 | 0 | else |
1278 | 0 | nDstBlockYSize /= 2; |
1279 | 0 | } |
1280 | |
|
1281 | 0 | const int nOvrXFactor = |
1282 | 0 | std::max(1, static_cast<int>(0.5 + dfXRatioDstToSrc)); |
1283 | 0 | const int nOvrYFactor = |
1284 | 0 | std::max(1, static_cast<int>(0.5 + dfYRatioDstToSrc)); |
1285 | 0 | const int nFullResXSizeQueried = static_cast<int>( |
1286 | 0 | std::min<int64_t>(nFullResXChunk + static_cast<int64_t>(2) * |
1287 | 0 | nKernelRadius * nOvrXFactor, |
1288 | 0 | nRasterXSize)); |
1289 | 0 | const int nFullResYSizeQueried = static_cast<int>( |
1290 | 0 | std::min<int64_t>(nFullResYChunk + static_cast<int64_t>(2) * |
1291 | 0 | nKernelRadius * nOvrYFactor, |
1292 | 0 | nRasterYSize)); |
1293 | |
|
1294 | 0 | void *pChunk = |
1295 | 0 | VSI_MALLOC3_VERBOSE(GDALGetDataTypeSizeBytes(eWrkDataType), |
1296 | 0 | nFullResXSizeQueried, nFullResYSizeQueried); |
1297 | 0 | GByte *pabyChunkNoDataMask = nullptr; |
1298 | |
|
1299 | 0 | GDALRasterBand *poMaskBand = GetMaskBand(); |
1300 | 0 | int l_nMaskFlags = GetMaskFlags(); |
1301 | |
|
1302 | 0 | bool bUseNoDataMask = ((l_nMaskFlags & GMF_ALL_VALID) == 0); |
1303 | 0 | if (bUseNoDataMask) |
1304 | 0 | { |
1305 | 0 | pabyChunkNoDataMask = static_cast<GByte *>(VSI_MALLOC2_VERBOSE( |
1306 | 0 | nFullResXSizeQueried, nFullResYSizeQueried)); |
1307 | 0 | } |
1308 | 0 | if (pChunk == nullptr || |
1309 | 0 | (bUseNoDataMask && pabyChunkNoDataMask == nullptr)) |
1310 | 0 | { |
1311 | 0 | GDALClose(poMEMDS); |
1312 | 0 | CPLFree(pChunk); |
1313 | 0 | CPLFree(pabyChunkNoDataMask); |
1314 | 0 | VSIFree(pTempBuffer); |
1315 | 0 | return CE_Failure; |
1316 | 0 | } |
1317 | | |
1318 | 0 | const int64_t nTotalBlocks = |
1319 | 0 | static_cast<int64_t>(cpl::div_round_up(nBufXSize, nDstBlockXSize)) * |
1320 | 0 | cpl::div_round_up(nBufYSize, nDstBlockYSize); |
1321 | 0 | int64_t nBlocksDone = 0; |
1322 | |
|
1323 | 0 | for (int nDstYOff = 0; nDstYOff < nBufYSize && eErr == CE_None; |
1324 | 0 | nDstYOff += nDstBlockYSize) |
1325 | 0 | { |
1326 | 0 | int nDstYCount; |
1327 | 0 | if (nDstYOff + nDstBlockYSize <= nBufYSize) |
1328 | 0 | nDstYCount = nDstBlockYSize; |
1329 | 0 | else |
1330 | 0 | nDstYCount = nBufYSize - nDstYOff; |
1331 | |
|
1332 | 0 | int nChunkYOff = |
1333 | 0 | nYOff + static_cast<int>(nDstYOff * dfYRatioDstToSrc); |
1334 | 0 | int nChunkYOff2 = nYOff + 1 + |
1335 | 0 | static_cast<int>(ceil((nDstYOff + nDstYCount) * |
1336 | 0 | dfYRatioDstToSrc)); |
1337 | 0 | if (nChunkYOff2 > nRasterYSize) |
1338 | 0 | nChunkYOff2 = nRasterYSize; |
1339 | 0 | int nYCount = nChunkYOff2 - nChunkYOff; |
1340 | 0 | CPLAssert(nYCount <= nFullResYChunk); |
1341 | | |
1342 | 0 | int nChunkYOffQueried = nChunkYOff - nKernelRadius * nOvrYFactor; |
1343 | 0 | int nChunkYSizeQueried = nYCount + 2 * nKernelRadius * nOvrYFactor; |
1344 | 0 | if (nChunkYOffQueried < 0) |
1345 | 0 | { |
1346 | 0 | nChunkYSizeQueried += nChunkYOffQueried; |
1347 | 0 | nChunkYOffQueried = 0; |
1348 | 0 | } |
1349 | 0 | if (nChunkYSizeQueried + nChunkYOffQueried > nRasterYSize) |
1350 | 0 | nChunkYSizeQueried = nRasterYSize - nChunkYOffQueried; |
1351 | 0 | CPLAssert(nChunkYSizeQueried <= nFullResYSizeQueried); |
1352 | | |
1353 | 0 | int nDstXOff = 0; |
1354 | 0 | for (nDstXOff = 0; nDstXOff < nBufXSize && eErr == CE_None; |
1355 | 0 | nDstXOff += nDstBlockXSize) |
1356 | 0 | { |
1357 | 0 | int nDstXCount = 0; |
1358 | 0 | if (nDstXOff + nDstBlockXSize <= nBufXSize) |
1359 | 0 | nDstXCount = nDstBlockXSize; |
1360 | 0 | else |
1361 | 0 | nDstXCount = nBufXSize - nDstXOff; |
1362 | |
|
1363 | 0 | int nChunkXOff = |
1364 | 0 | nXOff + static_cast<int>(nDstXOff * dfXRatioDstToSrc); |
1365 | 0 | int nChunkXOff2 = |
1366 | 0 | nXOff + 1 + |
1367 | 0 | static_cast<int>( |
1368 | 0 | ceil((nDstXOff + nDstXCount) * dfXRatioDstToSrc)); |
1369 | 0 | if (nChunkXOff2 > nRasterXSize) |
1370 | 0 | nChunkXOff2 = nRasterXSize; |
1371 | 0 | int nXCount = nChunkXOff2 - nChunkXOff; |
1372 | 0 | CPLAssert(nXCount <= nFullResXChunk); |
1373 | | |
1374 | 0 | int nChunkXOffQueried = |
1375 | 0 | nChunkXOff - nKernelRadius * nOvrXFactor; |
1376 | 0 | int nChunkXSizeQueried = |
1377 | 0 | nXCount + 2 * nKernelRadius * nOvrXFactor; |
1378 | 0 | if (nChunkXOffQueried < 0) |
1379 | 0 | { |
1380 | 0 | nChunkXSizeQueried += nChunkXOffQueried; |
1381 | 0 | nChunkXOffQueried = 0; |
1382 | 0 | } |
1383 | 0 | if (nChunkXSizeQueried + nChunkXOffQueried > nRasterXSize) |
1384 | 0 | nChunkXSizeQueried = nRasterXSize - nChunkXOffQueried; |
1385 | 0 | CPLAssert(nChunkXSizeQueried <= nFullResXSizeQueried); |
1386 | | |
1387 | | // Read the source buffers. |
1388 | 0 | eErr = RasterIO(GF_Read, nChunkXOffQueried, nChunkYOffQueried, |
1389 | 0 | nChunkXSizeQueried, nChunkYSizeQueried, pChunk, |
1390 | 0 | nChunkXSizeQueried, nChunkYSizeQueried, |
1391 | 0 | eWrkDataType, 0, 0, nullptr); |
1392 | |
|
1393 | 0 | bool bSkipResample = false; |
1394 | 0 | bool bNoDataMaskFullyOpaque = false; |
1395 | 0 | if (eErr == CE_None && bUseNoDataMask) |
1396 | 0 | { |
1397 | 0 | eErr = poMaskBand->RasterIO( |
1398 | 0 | GF_Read, nChunkXOffQueried, nChunkYOffQueried, |
1399 | 0 | nChunkXSizeQueried, nChunkYSizeQueried, |
1400 | 0 | pabyChunkNoDataMask, nChunkXSizeQueried, |
1401 | 0 | nChunkYSizeQueried, GDT_UInt8, 0, 0, nullptr); |
1402 | | |
1403 | | /* Optimizations if mask if fully opaque or transparent */ |
1404 | 0 | int nPixels = nChunkXSizeQueried * nChunkYSizeQueried; |
1405 | 0 | GByte bVal = pabyChunkNoDataMask[0]; |
1406 | 0 | int i = 1; |
1407 | 0 | for (; i < nPixels; i++) |
1408 | 0 | { |
1409 | 0 | if (pabyChunkNoDataMask[i] != bVal) |
1410 | 0 | break; |
1411 | 0 | } |
1412 | 0 | if (i == nPixels) |
1413 | 0 | { |
1414 | 0 | if (bVal == 0) |
1415 | 0 | { |
1416 | 0 | for (int j = 0; j < nDstYCount; j++) |
1417 | 0 | { |
1418 | 0 | GDALCopyWords64(&dfNoDataValue, GDT_Float64, 0, |
1419 | 0 | static_cast<GByte *>(pDataMem) + |
1420 | 0 | nLSMem * (j + nDstYOff) + |
1421 | 0 | nDstXOff * nPSMem, |
1422 | 0 | eDTMem, |
1423 | 0 | static_cast<int>(nPSMem), |
1424 | 0 | nDstXCount); |
1425 | 0 | } |
1426 | 0 | bSkipResample = true; |
1427 | 0 | } |
1428 | 0 | else |
1429 | 0 | { |
1430 | 0 | bNoDataMaskFullyOpaque = true; |
1431 | 0 | } |
1432 | 0 | } |
1433 | 0 | } |
1434 | |
|
1435 | 0 | if (!bSkipResample && eErr == CE_None) |
1436 | 0 | { |
1437 | 0 | const bool bPropagateNoData = false; |
1438 | 0 | void *pDstBuffer = nullptr; |
1439 | 0 | GDALDataType eDstBufferDataType = GDT_Unknown; |
1440 | 0 | GDALRasterBand *poMEMBand = |
1441 | 0 | GDALRasterBand::FromHandle(hMEMBand); |
1442 | 0 | GDALOverviewResampleArgs args; |
1443 | 0 | args.eSrcDataType = eDataType; |
1444 | 0 | args.eOvrDataType = poMEMBand->GetRasterDataType(); |
1445 | 0 | args.nOvrXSize = poMEMBand->GetXSize(); |
1446 | 0 | args.nOvrYSize = poMEMBand->GetYSize(); |
1447 | 0 | args.nOvrNBITS = nNBITS; |
1448 | 0 | args.dfXRatioDstToSrc = dfXRatioDstToSrc; |
1449 | 0 | args.dfYRatioDstToSrc = dfYRatioDstToSrc; |
1450 | 0 | args.dfSrcXDelta = |
1451 | 0 | dfXOff - nXOff; /* == 0 if bHasXOffVirtual */ |
1452 | 0 | args.dfSrcYDelta = |
1453 | 0 | dfYOff - nYOff; /* == 0 if bHasYOffVirtual */ |
1454 | 0 | args.eWrkDataType = eWrkDataType; |
1455 | 0 | args.pabyChunkNodataMask = |
1456 | 0 | bNoDataMaskFullyOpaque ? nullptr : pabyChunkNoDataMask; |
1457 | 0 | args.nChunkXOff = |
1458 | 0 | nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff); |
1459 | 0 | args.nChunkXSize = nChunkXSizeQueried; |
1460 | 0 | args.nChunkYOff = |
1461 | 0 | nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff); |
1462 | 0 | args.nChunkYSize = nChunkYSizeQueried; |
1463 | 0 | args.nDstXOff = nDstXOff + nDestXOffVirtual; |
1464 | 0 | args.nDstXOff2 = nDstXOff + nDestXOffVirtual + nDstXCount; |
1465 | 0 | args.nDstYOff = nDstYOff + nDestYOffVirtual; |
1466 | 0 | args.nDstYOff2 = nDstYOff + nDestYOffVirtual + nDstYCount; |
1467 | 0 | args.pszResampling = pszResampling; |
1468 | 0 | args.bHasNoData = bHasNoData; |
1469 | 0 | args.dfNoDataValue = dfNoDataValue; |
1470 | 0 | args.poColorTable = GetColorTable(); |
1471 | 0 | args.bPropagateNoData = bPropagateNoData; |
1472 | 0 | eErr = pfnResampleFunc(args, pChunk, &pDstBuffer, |
1473 | 0 | &eDstBufferDataType); |
1474 | 0 | if (eErr == CE_None) |
1475 | 0 | { |
1476 | 0 | eErr = poMEMBand->RasterIO( |
1477 | 0 | GF_Write, nDstXOff + nDestXOffVirtual, |
1478 | 0 | nDstYOff + nDestYOffVirtual, nDstXCount, nDstYCount, |
1479 | 0 | pDstBuffer, nDstXCount, nDstYCount, |
1480 | 0 | eDstBufferDataType, 0, 0, nullptr); |
1481 | 0 | } |
1482 | 0 | CPLFree(pDstBuffer); |
1483 | 0 | } |
1484 | |
|
1485 | 0 | nBlocksDone++; |
1486 | 0 | if (eErr == CE_None && psExtraArg->pfnProgress != nullptr && |
1487 | 0 | !psExtraArg->pfnProgress( |
1488 | 0 | static_cast<double>(nBlocksDone) / |
1489 | 0 | static_cast<double>(nTotalBlocks), |
1490 | 0 | "", psExtraArg->pProgressData)) |
1491 | 0 | { |
1492 | 0 | eErr = CE_Failure; |
1493 | 0 | } |
1494 | 0 | } |
1495 | 0 | } |
1496 | | |
1497 | 0 | CPLFree(pChunk); |
1498 | 0 | CPLFree(pabyChunkNoDataMask); |
1499 | 0 | } |
1500 | | |
1501 | 0 | if (pTempBuffer) |
1502 | 0 | { |
1503 | 0 | CPL_IGNORE_RET_VAL(poMEMDS->GetRasterBand(1)->RasterIO( |
1504 | 0 | GF_Read, nDestXOffVirtual, nDestYOffVirtual, nBufXSize, nBufYSize, |
1505 | 0 | pData, nBufXSize, nBufYSize, eBufType, nPixelSpace, nLineSpace, |
1506 | 0 | nullptr)); |
1507 | 0 | } |
1508 | 0 | GDALClose(poMEMDS); |
1509 | 0 | VSIFree(pTempBuffer); |
1510 | |
|
1511 | 0 | return eErr; |
1512 | 0 | } |
1513 | | |
1514 | | /************************************************************************/ |
1515 | | /* RasterIOResampled() */ |
1516 | | /************************************************************************/ |
1517 | | |
1518 | | CPLErr GDALDataset::RasterIOResampled( |
1519 | | GDALRWFlag /* eRWFlag */, int nXOff, int nYOff, int nXSize, int nYSize, |
1520 | | void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, |
1521 | | int nBandCount, const int *panBandMap, GSpacing nPixelSpace, |
1522 | | GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg) |
1523 | | |
1524 | 0 | { |
1525 | | #if 0 |
1526 | | // Determine if we use warping resampling or overview resampling |
1527 | | bool bUseWarp = false; |
1528 | | if( GDALDataTypeIsComplex( eDataType ) ) |
1529 | | bUseWarp = true; |
1530 | | #endif |
1531 | |
|
1532 | 0 | double dfXOff = nXOff; |
1533 | 0 | double dfYOff = nYOff; |
1534 | 0 | double dfXSize = nXSize; |
1535 | 0 | double dfYSize = nYSize; |
1536 | 0 | if (psExtraArg->bFloatingPointWindowValidity) |
1537 | 0 | { |
1538 | 0 | dfXOff = psExtraArg->dfXOff; |
1539 | 0 | dfYOff = psExtraArg->dfYOff; |
1540 | 0 | dfXSize = psExtraArg->dfXSize; |
1541 | 0 | dfYSize = psExtraArg->dfYSize; |
1542 | 0 | } |
1543 | |
|
1544 | 0 | const double dfXRatioDstToSrc = dfXSize / nBufXSize; |
1545 | 0 | const double dfYRatioDstToSrc = dfYSize / nBufYSize; |
1546 | | |
1547 | | // Determine the coordinates in the "virtual" output raster to see |
1548 | | // if there are not integers, in which case we will use them as a shift |
1549 | | // so that subwindow extracts give the exact same results as entire raster |
1550 | | // scaling. |
1551 | 0 | double dfDestXOff = dfXOff / dfXRatioDstToSrc; |
1552 | 0 | bool bHasXOffVirtual = false; |
1553 | 0 | int nDestXOffVirtual = 0; |
1554 | 0 | if (fabs(dfDestXOff - static_cast<int>(dfDestXOff + 0.5)) < 1e-8) |
1555 | 0 | { |
1556 | 0 | bHasXOffVirtual = true; |
1557 | 0 | dfXOff = nXOff; |
1558 | 0 | nDestXOffVirtual = static_cast<int>(dfDestXOff + 0.5); |
1559 | 0 | } |
1560 | |
|
1561 | 0 | double dfDestYOff = dfYOff / dfYRatioDstToSrc; |
1562 | 0 | bool bHasYOffVirtual = false; |
1563 | 0 | int nDestYOffVirtual = 0; |
1564 | 0 | if (fabs(dfDestYOff - static_cast<int>(dfDestYOff + 0.5)) < 1e-8) |
1565 | 0 | { |
1566 | 0 | bHasYOffVirtual = true; |
1567 | 0 | dfYOff = nYOff; |
1568 | 0 | nDestYOffVirtual = static_cast<int>(dfDestYOff + 0.5); |
1569 | 0 | } |
1570 | | |
1571 | | // Create a MEM dataset that wraps the output buffer. |
1572 | 0 | std::unique_ptr<void, VSIFreeReleaser> pTempBuffer; |
1573 | 0 | GSpacing nPSMem = nPixelSpace; |
1574 | 0 | GSpacing nLSMem = nLineSpace; |
1575 | 0 | GSpacing nBandSpaceMEM = nBandSpace; |
1576 | 0 | void *pDataMem = pData; |
1577 | 0 | GDALDataType eDTMem = eBufType; |
1578 | 0 | GDALRasterBand *poFirstSrcBand = GetRasterBand(panBandMap[0]); |
1579 | 0 | const GDALDataType eDataType = poFirstSrcBand->GetRasterDataType(); |
1580 | 0 | if (eBufType != eDataType && !GDAL_GET_OPERATE_IN_BUF_TYPE(*psExtraArg)) |
1581 | 0 | { |
1582 | 0 | nPSMem = GDALGetDataTypeSizeBytes(eDataType); |
1583 | 0 | nLSMem = nPSMem * nBufXSize; |
1584 | 0 | nBandSpaceMEM = nLSMem * nBandCount; |
1585 | 0 | pTempBuffer.reset(VSI_MALLOC3_VERBOSE(nBandCount, nBufYSize, |
1586 | 0 | static_cast<size_t>(nLSMem))); |
1587 | 0 | if (pTempBuffer == nullptr) |
1588 | 0 | return CE_Failure; |
1589 | 0 | pDataMem = pTempBuffer.get(); |
1590 | 0 | eDTMem = eDataType; |
1591 | 0 | } |
1592 | | |
1593 | 0 | auto poMEMDS = std::unique_ptr<GDALDataset>( |
1594 | 0 | MEMDataset::Create("", nDestXOffVirtual + nBufXSize, |
1595 | 0 | nDestYOffVirtual + nBufYSize, 0, eDTMem, nullptr)); |
1596 | | #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND |
1597 | | std::vector<GDALRasterBand *> apoDstBands(nBandCount); |
1598 | | #endif |
1599 | 0 | int nNBITS = 0; |
1600 | 0 | for (int i = 0; i < nBandCount; i++) |
1601 | 0 | { |
1602 | 0 | GByte *const pBandData = static_cast<GByte *>(pDataMem) - |
1603 | 0 | nPSMem * nDestXOffVirtual - |
1604 | 0 | nLSMem * nDestYOffVirtual + nBandSpaceMEM * i; |
1605 | 0 | auto poMEMBand = GDALRasterBand::FromHandle(MEMCreateRasterBandEx( |
1606 | 0 | poMEMDS.get(), i + 1, pBandData, eDTMem, nPSMem, nLSMem, false)); |
1607 | 0 | poMEMDS->SetBand(i + 1, poMEMBand); |
1608 | |
|
1609 | 0 | GDALRasterBand *poSrcBand = GetRasterBand(panBandMap[i]); |
1610 | | #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND |
1611 | | apoDstBands[i] = poMEMBand; |
1612 | | #endif |
1613 | 0 | const char *pszNBITS = |
1614 | 0 | poSrcBand->GetMetadataItem(GDALMD_NBITS, GDAL_MDD_IMAGE_STRUCTURE); |
1615 | 0 | if (pszNBITS) |
1616 | 0 | { |
1617 | 0 | nNBITS = atoi(pszNBITS); |
1618 | 0 | poMEMDS->GetRasterBand(i + 1)->SetMetadataItem( |
1619 | 0 | GDALMD_NBITS, pszNBITS, GDAL_MDD_IMAGE_STRUCTURE); |
1620 | 0 | } |
1621 | 0 | } |
1622 | |
|
1623 | 0 | CPLErr eErr = CE_None; |
1624 | | |
1625 | | // TODO(schwehr): Why disabled? Why not just delete? |
1626 | | // Looks like this code was initially added as disable by copying |
1627 | | // from RasterIO here: |
1628 | | // https://trac.osgeo.org/gdal/changeset/29572 |
1629 | | #if 0 |
1630 | | // Do the resampling. |
1631 | | if( bUseWarp ) |
1632 | | { |
1633 | | VRTDatasetH hVRTDS = nullptr; |
1634 | | GDALRasterBandH hVRTBand = nullptr; |
1635 | | if( GetDataset() == nullptr ) |
1636 | | { |
1637 | | /* Create VRT dataset that wraps the whole dataset */ |
1638 | | hVRTDS = VRTCreate(nRasterXSize, nRasterYSize); |
1639 | | VRTAddBand( hVRTDS, eDataType, nullptr ); |
1640 | | hVRTBand = GDALGetRasterBand(hVRTDS, 1); |
1641 | | VRTAddSimpleSource( (VRTSourcedRasterBandH)hVRTBand, |
1642 | | (GDALRasterBandH)this, |
1643 | | 0, 0, |
1644 | | nRasterXSize, nRasterYSize, |
1645 | | 0, 0, |
1646 | | nRasterXSize, nRasterYSize, |
1647 | | nullptr, VRT_NODATA_UNSET ); |
1648 | | |
1649 | | /* Add a mask band if needed */ |
1650 | | if( GetMaskFlags() != GMF_ALL_VALID ) |
1651 | | { |
1652 | | ((GDALDataset*)hVRTDS)->CreateMaskBand(0); |
1653 | | VRTSourcedRasterBand* poVRTMaskBand = |
1654 | | (VRTSourcedRasterBand*)(((GDALRasterBand*)hVRTBand)->GetMaskBand()); |
1655 | | poVRTMaskBand-> |
1656 | | AddMaskBandSource( this, |
1657 | | 0, 0, |
1658 | | nRasterXSize, nRasterYSize, |
1659 | | 0, 0, |
1660 | | nRasterXSize, nRasterYSize); |
1661 | | } |
1662 | | } |
1663 | | |
1664 | | GDALWarpOptions* psWarpOptions = GDALCreateWarpOptions(); |
1665 | | psWarpOptions->eResampleAlg = (GDALResampleAlg)psExtraArg->eResampleAlg; |
1666 | | psWarpOptions->hSrcDS = (GDALDatasetH) (hVRTDS ? hVRTDS : GetDataset()); |
1667 | | psWarpOptions->hDstDS = (GDALDatasetH) poMEMDS; |
1668 | | psWarpOptions->nBandCount = 1; |
1669 | | int nSrcBandNumber = (hVRTDS ? 1 : nBand); |
1670 | | int nDstBandNumber = 1; |
1671 | | psWarpOptions->panSrcBands = &nSrcBandNumber; |
1672 | | psWarpOptions->panDstBands = &nDstBandNumber; |
1673 | | psWarpOptions->pfnProgress = psExtraArg->pfnProgress ? |
1674 | | psExtraArg->pfnProgress : GDALDummyProgress; |
1675 | | psWarpOptions->pProgressArg = psExtraArg->pProgressData; |
1676 | | psWarpOptions->pfnTransformer = GDALRasterIOTransformer; |
1677 | | GDALRasterIOTransformerStruct sTransformer; |
1678 | | sTransformer.dfXOff = bHasXOffVirtual ? 0 : dfXOff; |
1679 | | sTransformer.dfYOff = bHasYOffVirtual ? 0 : dfYOff; |
1680 | | sTransformer.dfXRatioDstToSrc = dfXRatioDstToSrc; |
1681 | | sTransformer.dfYRatioDstToSrc = dfYRatioDstToSrc; |
1682 | | psWarpOptions->pTransformerArg = &sTransformer; |
1683 | | |
1684 | | GDALWarpOperationH hWarpOperation = GDALCreateWarpOperation(psWarpOptions); |
1685 | | eErr = GDALChunkAndWarpImage( hWarpOperation, |
1686 | | nDestXOffVirtual, nDestYOffVirtual, |
1687 | | nBufXSize, nBufYSize ); |
1688 | | GDALDestroyWarpOperation( hWarpOperation ); |
1689 | | |
1690 | | psWarpOptions->panSrcBands = nullptr; |
1691 | | psWarpOptions->panDstBands = nullptr; |
1692 | | GDALDestroyWarpOptions( psWarpOptions ); |
1693 | | |
1694 | | if( hVRTDS ) |
1695 | | GDALClose(hVRTDS); |
1696 | | } |
1697 | | else |
1698 | | #endif |
1699 | 0 | { |
1700 | 0 | const char *pszResampling = |
1701 | 0 | GDALRasterIOGetResampleAlg(psExtraArg->eResampleAlg); |
1702 | |
|
1703 | 0 | int nBlockXSize, nBlockYSize; |
1704 | 0 | poFirstSrcBand->GetBlockSize(&nBlockXSize, &nBlockYSize); |
1705 | |
|
1706 | 0 | int nKernelRadius; |
1707 | 0 | GDALResampleFunction pfnResampleFunc = |
1708 | 0 | GDALGetResampleFunction(pszResampling, &nKernelRadius); |
1709 | 0 | CPLAssert(pfnResampleFunc); |
1710 | | #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND |
1711 | | GDALResampleFunctionMultiBands pfnResampleFuncMultiBands = |
1712 | | GDALGetResampleFunctionMultiBands(pszResampling, &nKernelRadius); |
1713 | | #endif |
1714 | 0 | GDALDataType eWrkDataType = |
1715 | 0 | GDALGetOvrWorkDataType(pszResampling, eDataType); |
1716 | |
|
1717 | 0 | int nDstBlockXSize = nBufXSize; |
1718 | 0 | int nDstBlockYSize = nBufYSize; |
1719 | 0 | int nFullResXChunk, nFullResYChunk; |
1720 | 0 | while (true) |
1721 | 0 | { |
1722 | 0 | nFullResXChunk = static_cast<int>(std::min<double>( |
1723 | 0 | 3 + nDstBlockXSize * dfXRatioDstToSrc, nRasterXSize)); |
1724 | 0 | nFullResYChunk = static_cast<int>(std::min<double>( |
1725 | 0 | 3 + nDstBlockYSize * dfYRatioDstToSrc, nRasterYSize)); |
1726 | 0 | if ((nDstBlockXSize == 1 && nDstBlockYSize == 1) || |
1727 | 0 | (static_cast<GIntBig>(nFullResXChunk) * nFullResYChunk <= |
1728 | 0 | 1024 * 1024)) |
1729 | 0 | break; |
1730 | | // When operating on the full width of a raster whose block width is |
1731 | | // the raster width, prefer doing chunks in height. |
1732 | 0 | if (nFullResXChunk >= nXSize && nXSize == nBlockXSize && |
1733 | 0 | nDstBlockYSize > 1) |
1734 | 0 | nDstBlockYSize /= 2; |
1735 | | /* Otherwise cut the maximal dimension */ |
1736 | 0 | else if (nDstBlockXSize > 1 && |
1737 | 0 | (nFullResXChunk > nFullResYChunk || nDstBlockYSize == 1)) |
1738 | 0 | nDstBlockXSize /= 2; |
1739 | 0 | else |
1740 | 0 | nDstBlockYSize /= 2; |
1741 | 0 | } |
1742 | |
|
1743 | 0 | const int nOvrFactor = |
1744 | 0 | std::max(1, std::max(static_cast<int>(0.5 + dfXRatioDstToSrc), |
1745 | 0 | static_cast<int>(0.5 + dfYRatioDstToSrc))); |
1746 | 0 | const int nFullResXSizeQueried = static_cast<int>( |
1747 | 0 | std::min<int64_t>(nFullResXChunk + static_cast<int64_t>(2) * |
1748 | 0 | nKernelRadius * nOvrFactor, |
1749 | 0 | nRasterXSize)); |
1750 | 0 | const int nFullResYSizeQueried = static_cast<int>( |
1751 | 0 | std::min<int64_t>(nFullResYChunk + static_cast<int64_t>(2) * |
1752 | 0 | nKernelRadius * nOvrFactor, |
1753 | 0 | nRasterYSize)); |
1754 | |
|
1755 | 0 | void *pChunk = VSI_MALLOC3_VERBOSE( |
1756 | 0 | cpl::fits_on<int>(GDALGetDataTypeSizeBytes(eWrkDataType) * |
1757 | 0 | nBandCount), |
1758 | 0 | nFullResXSizeQueried, nFullResYSizeQueried); |
1759 | 0 | GByte *pabyChunkNoDataMask = nullptr; |
1760 | |
|
1761 | 0 | GDALRasterBand *poMaskBand = poFirstSrcBand->GetMaskBand(); |
1762 | 0 | int nMaskFlags = poFirstSrcBand->GetMaskFlags(); |
1763 | |
|
1764 | 0 | bool bUseNoDataMask = ((nMaskFlags & GMF_ALL_VALID) == 0); |
1765 | 0 | if (bUseNoDataMask) |
1766 | 0 | { |
1767 | 0 | pabyChunkNoDataMask = static_cast<GByte *>(VSI_MALLOC2_VERBOSE( |
1768 | 0 | nFullResXSizeQueried, nFullResYSizeQueried)); |
1769 | 0 | } |
1770 | 0 | if (pChunk == nullptr || |
1771 | 0 | (bUseNoDataMask && pabyChunkNoDataMask == nullptr)) |
1772 | 0 | { |
1773 | 0 | CPLFree(pChunk); |
1774 | 0 | CPLFree(pabyChunkNoDataMask); |
1775 | 0 | return CE_Failure; |
1776 | 0 | } |
1777 | | |
1778 | 0 | const int64_t nTotalBlocks = |
1779 | 0 | static_cast<int64_t>(cpl::div_round_up(nBufXSize, nDstBlockXSize)) * |
1780 | 0 | cpl::div_round_up(nBufYSize, nDstBlockYSize); |
1781 | 0 | int64_t nBlocksDone = 0; |
1782 | |
|
1783 | 0 | for (int nDstYOff = 0; nDstYOff < nBufYSize && eErr == CE_None; |
1784 | 0 | nDstYOff += nDstBlockYSize) |
1785 | 0 | { |
1786 | 0 | int nDstYCount; |
1787 | 0 | if (nDstYOff + nDstBlockYSize <= nBufYSize) |
1788 | 0 | nDstYCount = nDstBlockYSize; |
1789 | 0 | else |
1790 | 0 | nDstYCount = nBufYSize - nDstYOff; |
1791 | |
|
1792 | 0 | int nChunkYOff = |
1793 | 0 | nYOff + static_cast<int>(nDstYOff * dfYRatioDstToSrc); |
1794 | 0 | int nChunkYOff2 = nYOff + 1 + |
1795 | 0 | static_cast<int>(ceil((nDstYOff + nDstYCount) * |
1796 | 0 | dfYRatioDstToSrc)); |
1797 | 0 | if (nChunkYOff2 > nRasterYSize) |
1798 | 0 | nChunkYOff2 = nRasterYSize; |
1799 | 0 | int nYCount = nChunkYOff2 - nChunkYOff; |
1800 | 0 | CPLAssert(nYCount <= nFullResYChunk); |
1801 | | |
1802 | 0 | int nChunkYOffQueried = nChunkYOff - nKernelRadius * nOvrFactor; |
1803 | 0 | int nChunkYSizeQueried = nYCount + 2 * nKernelRadius * nOvrFactor; |
1804 | 0 | if (nChunkYOffQueried < 0) |
1805 | 0 | { |
1806 | 0 | nChunkYSizeQueried += nChunkYOffQueried; |
1807 | 0 | nChunkYOffQueried = 0; |
1808 | 0 | } |
1809 | 0 | if (nChunkYSizeQueried + nChunkYOffQueried > nRasterYSize) |
1810 | 0 | nChunkYSizeQueried = nRasterYSize - nChunkYOffQueried; |
1811 | 0 | CPLAssert(nChunkYSizeQueried <= nFullResYSizeQueried); |
1812 | | |
1813 | 0 | int nDstXOff; |
1814 | 0 | for (nDstXOff = 0; nDstXOff < nBufXSize && eErr == CE_None; |
1815 | 0 | nDstXOff += nDstBlockXSize) |
1816 | 0 | { |
1817 | 0 | int nDstXCount; |
1818 | 0 | if (nDstXOff + nDstBlockXSize <= nBufXSize) |
1819 | 0 | nDstXCount = nDstBlockXSize; |
1820 | 0 | else |
1821 | 0 | nDstXCount = nBufXSize - nDstXOff; |
1822 | |
|
1823 | 0 | int nChunkXOff = |
1824 | 0 | nXOff + static_cast<int>(nDstXOff * dfXRatioDstToSrc); |
1825 | 0 | int nChunkXOff2 = |
1826 | 0 | nXOff + 1 + |
1827 | 0 | static_cast<int>( |
1828 | 0 | ceil((nDstXOff + nDstXCount) * dfXRatioDstToSrc)); |
1829 | 0 | if (nChunkXOff2 > nRasterXSize) |
1830 | 0 | nChunkXOff2 = nRasterXSize; |
1831 | 0 | int nXCount = nChunkXOff2 - nChunkXOff; |
1832 | 0 | CPLAssert(nXCount <= nFullResXChunk); |
1833 | | |
1834 | 0 | int nChunkXOffQueried = nChunkXOff - nKernelRadius * nOvrFactor; |
1835 | 0 | int nChunkXSizeQueried = |
1836 | 0 | nXCount + 2 * nKernelRadius * nOvrFactor; |
1837 | 0 | if (nChunkXOffQueried < 0) |
1838 | 0 | { |
1839 | 0 | nChunkXSizeQueried += nChunkXOffQueried; |
1840 | 0 | nChunkXOffQueried = 0; |
1841 | 0 | } |
1842 | 0 | if (nChunkXSizeQueried + nChunkXOffQueried > nRasterXSize) |
1843 | 0 | nChunkXSizeQueried = nRasterXSize - nChunkXOffQueried; |
1844 | 0 | CPLAssert(nChunkXSizeQueried <= nFullResXSizeQueried); |
1845 | | |
1846 | 0 | bool bSkipResample = false; |
1847 | 0 | bool bNoDataMaskFullyOpaque = false; |
1848 | 0 | if (eErr == CE_None && bUseNoDataMask) |
1849 | 0 | { |
1850 | 0 | eErr = poMaskBand->RasterIO( |
1851 | 0 | GF_Read, nChunkXOffQueried, nChunkYOffQueried, |
1852 | 0 | nChunkXSizeQueried, nChunkYSizeQueried, |
1853 | 0 | pabyChunkNoDataMask, nChunkXSizeQueried, |
1854 | 0 | nChunkYSizeQueried, GDT_UInt8, 0, 0, nullptr); |
1855 | | |
1856 | | /* Optimizations if mask if fully opaque or transparent */ |
1857 | 0 | const int nPixels = nChunkXSizeQueried * nChunkYSizeQueried; |
1858 | 0 | const GByte bVal = pabyChunkNoDataMask[0]; |
1859 | 0 | int i = 1; // Used after for. |
1860 | 0 | for (; i < nPixels; i++) |
1861 | 0 | { |
1862 | 0 | if (pabyChunkNoDataMask[i] != bVal) |
1863 | 0 | break; |
1864 | 0 | } |
1865 | 0 | if (i == nPixels) |
1866 | 0 | { |
1867 | 0 | if (bVal == 0) |
1868 | 0 | { |
1869 | 0 | GByte abyZero[16] = {0}; |
1870 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
1871 | 0 | { |
1872 | 0 | for (int j = 0; j < nDstYCount; j++) |
1873 | 0 | { |
1874 | 0 | GDALCopyWords64( |
1875 | 0 | abyZero, GDT_UInt8, 0, |
1876 | 0 | static_cast<GByte *>(pDataMem) + |
1877 | 0 | iBand * nBandSpaceMEM + |
1878 | 0 | nLSMem * (j + nDstYOff) + |
1879 | 0 | nDstXOff * nPSMem, |
1880 | 0 | eBufType, static_cast<int>(nPSMem), |
1881 | 0 | nDstXCount); |
1882 | 0 | } |
1883 | 0 | } |
1884 | 0 | bSkipResample = true; |
1885 | 0 | } |
1886 | 0 | else |
1887 | 0 | { |
1888 | 0 | bNoDataMaskFullyOpaque = true; |
1889 | 0 | } |
1890 | 0 | } |
1891 | 0 | } |
1892 | |
|
1893 | 0 | if (!bSkipResample && eErr == CE_None) |
1894 | 0 | { |
1895 | | /* Read the source buffers */ |
1896 | 0 | eErr = RasterIO( |
1897 | 0 | GF_Read, nChunkXOffQueried, nChunkYOffQueried, |
1898 | 0 | nChunkXSizeQueried, nChunkYSizeQueried, pChunk, |
1899 | 0 | nChunkXSizeQueried, nChunkYSizeQueried, eWrkDataType, |
1900 | 0 | nBandCount, panBandMap, 0, 0, 0, nullptr); |
1901 | 0 | } |
1902 | |
|
1903 | | #ifdef GDAL_ENABLE_RESAMPLING_MULTIBAND |
1904 | | if (pfnResampleFuncMultiBands && !bSkipResample && |
1905 | | eErr == CE_None) |
1906 | | { |
1907 | | eErr = pfnResampleFuncMultiBands( |
1908 | | dfXRatioDstToSrc, dfYRatioDstToSrc, |
1909 | | dfXOff - nXOff, /* == 0 if bHasXOffVirtual */ |
1910 | | dfYOff - nYOff, /* == 0 if bHasYOffVirtual */ |
1911 | | eWrkDataType, (GByte *)pChunk, nBandCount, |
1912 | | bNoDataMaskFullyOpaque ? nullptr : pabyChunkNoDataMask, |
1913 | | nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff), |
1914 | | nChunkXSizeQueried, |
1915 | | nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff), |
1916 | | nChunkYSizeQueried, nDstXOff + nDestXOffVirtual, |
1917 | | nDstXOff + nDestXOffVirtual + nDstXCount, |
1918 | | nDstYOff + nDestYOffVirtual, |
1919 | | nDstYOff + nDestYOffVirtual + nDstYCount, |
1920 | | apoDstBands.data(), pszResampling, FALSE /*bHasNoData*/, |
1921 | | 0.0 /* dfNoDataValue */, nullptr /* color table*/, |
1922 | | eDataType); |
1923 | | } |
1924 | | else |
1925 | | #endif |
1926 | 0 | { |
1927 | 0 | size_t nChunkBandOffset = |
1928 | 0 | static_cast<size_t>(nChunkXSizeQueried) * |
1929 | 0 | nChunkYSizeQueried * |
1930 | 0 | GDALGetDataTypeSizeBytes(eWrkDataType); |
1931 | 0 | for (int i = 0; |
1932 | 0 | i < nBandCount && !bSkipResample && eErr == CE_None; |
1933 | 0 | i++) |
1934 | 0 | { |
1935 | 0 | const bool bPropagateNoData = false; |
1936 | 0 | void *pDstBuffer = nullptr; |
1937 | 0 | GDALDataType eDstBufferDataType = GDT_Unknown; |
1938 | 0 | GDALRasterBand *poMEMBand = |
1939 | 0 | poMEMDS->GetRasterBand(i + 1); |
1940 | 0 | GDALOverviewResampleArgs args; |
1941 | 0 | args.eSrcDataType = eDataType; |
1942 | 0 | args.eOvrDataType = poMEMBand->GetRasterDataType(); |
1943 | 0 | args.nOvrXSize = poMEMBand->GetXSize(); |
1944 | 0 | args.nOvrYSize = poMEMBand->GetYSize(); |
1945 | 0 | args.nOvrNBITS = nNBITS; |
1946 | 0 | args.dfXRatioDstToSrc = dfXRatioDstToSrc; |
1947 | 0 | args.dfYRatioDstToSrc = dfYRatioDstToSrc; |
1948 | 0 | args.dfSrcXDelta = |
1949 | 0 | dfXOff - nXOff; /* == 0 if bHasXOffVirtual */ |
1950 | 0 | args.dfSrcYDelta = |
1951 | 0 | dfYOff - nYOff; /* == 0 if bHasYOffVirtual */ |
1952 | 0 | args.eWrkDataType = eWrkDataType; |
1953 | 0 | args.pabyChunkNodataMask = bNoDataMaskFullyOpaque |
1954 | 0 | ? nullptr |
1955 | 0 | : pabyChunkNoDataMask; |
1956 | 0 | args.nChunkXOff = |
1957 | 0 | nChunkXOffQueried - (bHasXOffVirtual ? 0 : nXOff); |
1958 | 0 | args.nChunkXSize = nChunkXSizeQueried; |
1959 | 0 | args.nChunkYOff = |
1960 | 0 | nChunkYOffQueried - (bHasYOffVirtual ? 0 : nYOff); |
1961 | 0 | args.nChunkYSize = nChunkYSizeQueried; |
1962 | 0 | args.nDstXOff = nDstXOff + nDestXOffVirtual; |
1963 | 0 | args.nDstXOff2 = |
1964 | 0 | nDstXOff + nDestXOffVirtual + nDstXCount; |
1965 | 0 | args.nDstYOff = nDstYOff + nDestYOffVirtual; |
1966 | 0 | args.nDstYOff2 = |
1967 | 0 | nDstYOff + nDestYOffVirtual + nDstYCount; |
1968 | 0 | args.pszResampling = pszResampling; |
1969 | 0 | args.bHasNoData = false; |
1970 | 0 | args.dfNoDataValue = 0.0; |
1971 | 0 | args.poColorTable = nullptr; |
1972 | 0 | args.bPropagateNoData = bPropagateNoData; |
1973 | |
|
1974 | 0 | eErr = |
1975 | 0 | pfnResampleFunc(args, |
1976 | 0 | reinterpret_cast<GByte *>(pChunk) + |
1977 | 0 | i * nChunkBandOffset, |
1978 | 0 | &pDstBuffer, &eDstBufferDataType); |
1979 | 0 | if (eErr == CE_None) |
1980 | 0 | { |
1981 | 0 | eErr = poMEMBand->RasterIO( |
1982 | 0 | GF_Write, nDstXOff + nDestXOffVirtual, |
1983 | 0 | nDstYOff + nDestYOffVirtual, nDstXCount, |
1984 | 0 | nDstYCount, pDstBuffer, nDstXCount, nDstYCount, |
1985 | 0 | eDstBufferDataType, 0, 0, nullptr); |
1986 | 0 | } |
1987 | 0 | CPLFree(pDstBuffer); |
1988 | 0 | } |
1989 | 0 | } |
1990 | |
|
1991 | 0 | nBlocksDone++; |
1992 | 0 | if (eErr == CE_None && psExtraArg->pfnProgress != nullptr && |
1993 | 0 | !psExtraArg->pfnProgress( |
1994 | 0 | static_cast<double>(nBlocksDone) / |
1995 | 0 | static_cast<double>(nTotalBlocks), |
1996 | 0 | "", psExtraArg->pProgressData)) |
1997 | 0 | { |
1998 | 0 | eErr = CE_Failure; |
1999 | 0 | } |
2000 | 0 | } |
2001 | 0 | } |
2002 | | |
2003 | 0 | CPLFree(pChunk); |
2004 | 0 | CPLFree(pabyChunkNoDataMask); |
2005 | 0 | } |
2006 | | |
2007 | 0 | if (pTempBuffer) |
2008 | 0 | { |
2009 | 0 | CPL_IGNORE_RET_VAL(poMEMDS->RasterIO( |
2010 | 0 | GF_Read, nDestXOffVirtual, nDestYOffVirtual, nBufXSize, nBufYSize, |
2011 | 0 | pData, nBufXSize, nBufYSize, eBufType, nBandCount, nullptr, |
2012 | 0 | nPixelSpace, nLineSpace, nBandSpace, nullptr)); |
2013 | 0 | } |
2014 | |
|
2015 | 0 | return eErr; |
2016 | 0 | } |
2017 | | |
2018 | | //! @endcond |
2019 | | |
2020 | | /************************************************************************/ |
2021 | | /* GDALSwapWords() */ |
2022 | | /************************************************************************/ |
2023 | | |
2024 | | /** |
2025 | | * Byte swap words in-place. |
2026 | | * |
2027 | | * This function will byte swap a set of 2, 4 or 8 byte words "in place" in |
2028 | | * a memory array. No assumption is made that the words being swapped are |
2029 | | * word aligned in memory. Use the CPL_LSB and CPL_MSB macros from cpl_port.h |
2030 | | * to determine if the current platform is big endian or little endian. Use |
2031 | | * The macros like CPL_SWAP32() to byte swap single values without the overhead |
2032 | | * of a function call. |
2033 | | * |
2034 | | * @param pData pointer to start of data buffer. |
2035 | | * @param nWordSize size of words being swapped in bytes. Normally 2, 4 or 8. |
2036 | | * @param nWordCount the number of words to be swapped in this call. |
2037 | | * @param nWordSkip the byte offset from the start of one word to the start of |
2038 | | * the next. For packed buffers this is the same as nWordSize. |
2039 | | */ |
2040 | | |
2041 | | void CPL_STDCALL GDALSwapWords(void *pData, int nWordSize, int nWordCount, |
2042 | | int nWordSkip) |
2043 | | |
2044 | 0 | { |
2045 | 0 | if (nWordCount > 0) |
2046 | 0 | VALIDATE_POINTER0(pData, "GDALSwapWords"); |
2047 | | |
2048 | 0 | GByte *pabyData = static_cast<GByte *>(pData); |
2049 | |
|
2050 | 0 | switch (nWordSize) |
2051 | 0 | { |
2052 | 0 | case 1: |
2053 | 0 | break; |
2054 | | |
2055 | 0 | case 2: |
2056 | 0 | CPLAssert(nWordSkip >= 2 || nWordCount == 1); |
2057 | 0 | for (int i = 0; i < nWordCount; i++) |
2058 | 0 | { |
2059 | 0 | CPL_SWAP16PTR(pabyData); |
2060 | 0 | pabyData += nWordSkip; |
2061 | 0 | } |
2062 | 0 | break; |
2063 | | |
2064 | 0 | case 4: |
2065 | 0 | CPLAssert(nWordSkip >= 4 || nWordCount == 1); |
2066 | 0 | if (CPL_IS_ALIGNED(pabyData, 4) && (nWordSkip % 4) == 0) |
2067 | 0 | { |
2068 | 0 | for (int i = 0; i < nWordCount; i++) |
2069 | 0 | { |
2070 | 0 | *reinterpret_cast<GUInt32 *>(pabyData) = CPL_SWAP32( |
2071 | 0 | *reinterpret_cast<const GUInt32 *>(pabyData)); |
2072 | 0 | pabyData += nWordSkip; |
2073 | 0 | } |
2074 | 0 | } |
2075 | 0 | else |
2076 | 0 | { |
2077 | 0 | for (int i = 0; i < nWordCount; i++) |
2078 | 0 | { |
2079 | 0 | CPL_SWAP32PTR(pabyData); |
2080 | 0 | pabyData += nWordSkip; |
2081 | 0 | } |
2082 | 0 | } |
2083 | 0 | break; |
2084 | | |
2085 | 0 | case 8: |
2086 | 0 | CPLAssert(nWordSkip >= 8 || nWordCount == 1); |
2087 | 0 | if (CPL_IS_ALIGNED(pabyData, 8) && (nWordSkip % 8) == 0) |
2088 | 0 | { |
2089 | 0 | for (int i = 0; i < nWordCount; i++) |
2090 | 0 | { |
2091 | 0 | *reinterpret_cast<GUInt64 *>(pabyData) = CPL_SWAP64( |
2092 | 0 | *reinterpret_cast<const GUInt64 *>(pabyData)); |
2093 | 0 | pabyData += nWordSkip; |
2094 | 0 | } |
2095 | 0 | } |
2096 | 0 | else |
2097 | 0 | { |
2098 | 0 | for (int i = 0; i < nWordCount; i++) |
2099 | 0 | { |
2100 | 0 | CPL_SWAP64PTR(pabyData); |
2101 | 0 | pabyData += nWordSkip; |
2102 | 0 | } |
2103 | 0 | } |
2104 | 0 | break; |
2105 | | |
2106 | 0 | default: |
2107 | 0 | CPLAssert(false); |
2108 | 0 | } |
2109 | 0 | } |
2110 | | |
2111 | | /************************************************************************/ |
2112 | | /* GDALSwapWordsEx() */ |
2113 | | /************************************************************************/ |
2114 | | |
2115 | | /** |
2116 | | * Byte swap words in-place. |
2117 | | * |
2118 | | * This function will byte swap a set of 2, 4 or 8 byte words "in place" in |
2119 | | * a memory array. No assumption is made that the words being swapped are |
2120 | | * word aligned in memory. Use the CPL_LSB and CPL_MSB macros from cpl_port.h |
2121 | | * to determine if the current platform is big endian or little endian. Use |
2122 | | * The macros like CPL_SWAP32() to byte swap single values without the overhead |
2123 | | * of a function call. |
2124 | | * |
2125 | | * @param pData pointer to start of data buffer. |
2126 | | * @param nWordSize size of words being swapped in bytes. Normally 2, 4 or 8. |
2127 | | * @param nWordCount the number of words to be swapped in this call. |
2128 | | * @param nWordSkip the byte offset from the start of one word to the start of |
2129 | | * the next. For packed buffers this is the same as nWordSize. |
2130 | | */ |
2131 | | void CPL_STDCALL GDALSwapWordsEx(void *pData, int nWordSize, size_t nWordCount, |
2132 | | int nWordSkip) |
2133 | 0 | { |
2134 | 0 | GByte *pabyData = static_cast<GByte *>(pData); |
2135 | 0 | while (nWordCount) |
2136 | 0 | { |
2137 | | // Pick-up a multiple of 8 as max chunk size. |
2138 | 0 | const int nWordCountSmall = |
2139 | 0 | (nWordCount > (1 << 30)) ? (1 << 30) : static_cast<int>(nWordCount); |
2140 | 0 | GDALSwapWords(pabyData, nWordSize, nWordCountSmall, nWordSkip); |
2141 | 0 | pabyData += static_cast<size_t>(nWordSkip) * nWordCountSmall; |
2142 | 0 | nWordCount -= nWordCountSmall; |
2143 | 0 | } |
2144 | 0 | } |
2145 | | |
2146 | | // Place the new GDALCopyWords helpers in an anonymous namespace |
2147 | | namespace |
2148 | | { |
2149 | | |
2150 | | /************************************************************************/ |
2151 | | /* GDALCopyWordsT() */ |
2152 | | /************************************************************************/ |
2153 | | /** |
2154 | | * Template function, used to copy data from pSrcData into buffer |
2155 | | * pDstData, with stride nSrcPixelStride in the source data and |
2156 | | * stride nDstPixelStride in the destination data. This template can |
2157 | | * deal with the case where the input data type is real or complex and |
2158 | | * the output is real. |
2159 | | * |
2160 | | * @param pSrcData the source data buffer |
2161 | | * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels |
2162 | | * of interest. |
2163 | | * @param pDstData the destination buffer. |
2164 | | * @param nDstPixelStride the stride in the buffer pDstData for pixels of |
2165 | | * interest. |
2166 | | * @param nWordCount the total number of pixel words to copy |
2167 | | * |
2168 | | * @code |
2169 | | * // Assume an input buffer of type GUInt16 named pBufferIn |
2170 | | * GByte *pBufferOut = new GByte[numBytesOut]; |
2171 | | * GDALCopyWordsT<GUInt16, GByte>(pSrcData, 2, pDstData, 1, numBytesOut); |
2172 | | * @endcode |
2173 | | * @note |
2174 | | * This is a private function, and should not be exposed outside of |
2175 | | * rasterio.cpp. External users should call the GDALCopyWords driver function. |
2176 | | */ |
2177 | | |
2178 | | template <class Tin, class Tout> |
2179 | | static void inline GDALCopyWordsGenericT(const Tin *const CPL_RESTRICT pSrcData, |
2180 | | int nSrcPixelStride, |
2181 | | Tout *const CPL_RESTRICT pDstData, |
2182 | | int nDstPixelStride, |
2183 | | GPtrDiff_t nWordCount) |
2184 | 0 | { |
2185 | 0 | decltype(nWordCount) nDstOffset = 0; |
2186 | |
|
2187 | 0 | const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData); |
2188 | 0 | char *const pDstDataPtr = reinterpret_cast<char *>(pDstData); |
2189 | 0 | for (decltype(nWordCount) n = 0; n < nWordCount; n++) |
2190 | 0 | { |
2191 | 0 | const Tin tValue = |
2192 | 0 | *reinterpret_cast<const Tin *>(pSrcDataPtr + (n * nSrcPixelStride)); |
2193 | 0 | Tout *const pOutPixel = |
2194 | 0 | reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset); |
2195 | |
|
2196 | 0 | GDALCopyWord(tValue, *pOutPixel); |
2197 | |
|
2198 | 0 | nDstOffset += nDstPixelStride; |
2199 | 0 | } |
2200 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, unsigned char>(unsigned char const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, signed char>(unsigned char const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, unsigned short>(unsigned char const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, short>(unsigned char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, unsigned int>(unsigned char const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, int>(unsigned char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, unsigned long>(unsigned char const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, long>(unsigned char const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, cpl::Float16>(unsigned char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, float>(unsigned char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned char, double>(unsigned char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, unsigned char>(signed char const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, signed char>(signed char const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, unsigned short>(signed char const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, short>(signed char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, unsigned int>(signed char const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, int>(signed char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, unsigned long>(signed char const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, long>(signed char const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, cpl::Float16>(signed char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, float>(signed char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<signed char, double>(signed char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, unsigned char>(unsigned short const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, signed char>(unsigned short const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, unsigned short>(unsigned short const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, short>(unsigned short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, unsigned int>(unsigned short const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, int>(unsigned short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, unsigned long>(unsigned short const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, long>(unsigned short const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, cpl::Float16>(unsigned short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, float>(unsigned short const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned short, double>(unsigned short const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, unsigned char>(short const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, signed char>(short const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, unsigned short>(short const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, short>(short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, unsigned int>(short const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, int>(short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, unsigned long>(short const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, long>(short const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, cpl::Float16>(short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, float>(short const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<short, double>(short const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, unsigned char>(unsigned int const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, signed char>(unsigned int const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, unsigned short>(unsigned int const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, short>(unsigned int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, unsigned int>(unsigned int const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, int>(unsigned int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, unsigned long>(unsigned int const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, long>(unsigned int const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, cpl::Float16>(unsigned int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, float>(unsigned int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned int, double>(unsigned int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, unsigned char>(int const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, signed char>(int const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, unsigned short>(int const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, short>(int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, unsigned int>(int const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, int>(int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, unsigned long>(int const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, long>(int const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, cpl::Float16>(int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, float>(int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<int, double>(int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, unsigned char>(unsigned long const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, signed char>(unsigned long const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, unsigned short>(unsigned long const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, short>(unsigned long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, unsigned int>(unsigned long const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, int>(unsigned long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, unsigned long>(unsigned long const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, long>(unsigned long const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, cpl::Float16>(unsigned long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, float>(unsigned long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<unsigned long, double>(unsigned long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, unsigned char>(long const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, signed char>(long const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, unsigned short>(long const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, short>(long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, unsigned int>(long const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, int>(long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, unsigned long>(long const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, long>(long const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, cpl::Float16>(long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, float>(long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<long, double>(long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, unsigned char>(cpl::Float16 const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, signed char>(cpl::Float16 const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, unsigned short>(cpl::Float16 const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, short>(cpl::Float16 const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, unsigned int>(cpl::Float16 const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, int>(cpl::Float16 const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, unsigned long>(cpl::Float16 const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, long>(cpl::Float16 const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<cpl::Float16, cpl::Float16>(cpl::Float16 const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<float, unsigned int>(float const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<float, unsigned long>(float const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<float, long>(float const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<float, float>(float const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, signed char>(double const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, short>(double const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, unsigned int>(double const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, int>(double const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, unsigned long>(double const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, long>(double const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsGenericT<double, double>(double const*, int, double*, int, long long) |
2201 | | |
2202 | | template <class Tin, class Tout> |
2203 | | static void CPL_NOINLINE GDALCopyWordsT(const Tin *const CPL_RESTRICT pSrcData, |
2204 | | int nSrcPixelStride, |
2205 | | Tout *const CPL_RESTRICT pDstData, |
2206 | | int nDstPixelStride, |
2207 | | GPtrDiff_t nWordCount) |
2208 | 0 | { |
2209 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, nDstPixelStride, |
2210 | 0 | nWordCount); |
2211 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned char, unsigned char>(unsigned char const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned char, unsigned long>(unsigned char const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned char, long>(unsigned char const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned char, cpl::Float16>(unsigned char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, signed char>(signed char const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, unsigned short>(signed char const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, short>(signed char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, unsigned int>(signed char const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, int>(signed char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, unsigned long>(signed char const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, long>(signed char const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, cpl::Float16>(signed char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, float>(signed char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<signed char, double>(signed char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, signed char>(unsigned short const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, unsigned short>(unsigned short const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, unsigned int>(unsigned short const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, int>(unsigned short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, unsigned long>(unsigned short const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, long>(unsigned short const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned short, cpl::Float16>(unsigned short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, signed char>(short const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, short>(short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, unsigned int>(short const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, int>(short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, unsigned long>(short const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, long>(short const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<short, cpl::Float16>(short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, unsigned char>(unsigned int const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, signed char>(unsigned int const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, unsigned short>(unsigned int const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, short>(unsigned int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, unsigned int>(unsigned int const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, unsigned long>(unsigned int const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, long>(unsigned int const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, cpl::Float16>(unsigned int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, float>(unsigned int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned int, double>(unsigned int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, signed char>(int const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, int>(int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, unsigned long>(int const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, long>(int const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, cpl::Float16>(int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, float>(int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<int, double>(int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, unsigned char>(unsigned long const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, signed char>(unsigned long const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, unsigned short>(unsigned long const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, short>(unsigned long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, unsigned int>(unsigned long const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, int>(unsigned long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, unsigned long>(unsigned long const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, long>(unsigned long const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, cpl::Float16>(unsigned long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, float>(unsigned long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<unsigned long, double>(unsigned long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, unsigned char>(long const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, signed char>(long const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, unsigned short>(long const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, short>(long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, unsigned int>(long const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, int>(long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, unsigned long>(long const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, long>(long const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, cpl::Float16>(long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, float>(long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<long, double>(long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, unsigned char>(cpl::Float16 const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, signed char>(cpl::Float16 const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, unsigned short>(cpl::Float16 const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, short>(cpl::Float16 const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, unsigned int>(cpl::Float16 const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, int>(cpl::Float16 const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, unsigned long>(cpl::Float16 const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, long>(cpl::Float16 const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<cpl::Float16, cpl::Float16>(cpl::Float16 const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<float, unsigned int>(float const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<float, unsigned long>(float const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<float, long>(float const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<float, float>(float const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, signed char>(double const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, short>(double const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, unsigned int>(double const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, int>(double const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, unsigned long>(double const*, int, unsigned long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, long>(double const*, int, long*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT<double, double>(double const*, int, double*, int, long long) |
2212 | | |
2213 | | template <class Tin, class Tout> |
2214 | | static void inline GDALCopyWordsT_8atatime( |
2215 | | const Tin *const CPL_RESTRICT pSrcData, int nSrcPixelStride, |
2216 | | Tout *const CPL_RESTRICT pDstData, int nDstPixelStride, |
2217 | | GPtrDiff_t nWordCount) |
2218 | 0 | { |
2219 | 0 | decltype(nWordCount) nDstOffset = 0; |
2220 | |
|
2221 | 0 | const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData); |
2222 | 0 | char *const pDstDataPtr = reinterpret_cast<char *>(pDstData); |
2223 | 0 | decltype(nWordCount) n = 0; |
2224 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(Tin)) && |
2225 | 0 | nDstPixelStride == static_cast<int>(sizeof(Tout))) |
2226 | 0 | { |
2227 | 0 | for (; n < nWordCount - 7; n += 8) |
2228 | 0 | { |
2229 | 0 | const Tin *pInValues = reinterpret_cast<const Tin *>( |
2230 | 0 | pSrcDataPtr + (n * nSrcPixelStride)); |
2231 | 0 | Tout *const pOutPixels = |
2232 | 0 | reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset); |
2233 | |
|
2234 | 0 | GDALCopy8Words(pInValues, pOutPixels); |
2235 | |
|
2236 | 0 | nDstOffset += 8 * nDstPixelStride; |
2237 | 0 | } |
2238 | 0 | } |
2239 | 0 | for (; n < nWordCount; n++) |
2240 | 0 | { |
2241 | 0 | const Tin tValue = |
2242 | 0 | *reinterpret_cast<const Tin *>(pSrcDataPtr + (n * nSrcPixelStride)); |
2243 | 0 | Tout *const pOutPixel = |
2244 | 0 | reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset); |
2245 | |
|
2246 | 0 | GDALCopyWord(tValue, *pOutPixel); |
2247 | |
|
2248 | 0 | nDstOffset += nDstPixelStride; |
2249 | 0 | } |
2250 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<cpl::Float16, float>(cpl::Float16 const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<cpl::Float16, double>(cpl::Float16 const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, unsigned char>(float const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, signed char>(float const*, int, signed char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, unsigned short>(float const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, short>(float const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, int>(float const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, cpl::Float16>(float const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<float, double>(float const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<double, unsigned char>(double const*, int, unsigned char*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<double, unsigned short>(double const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<double, cpl::Float16>(double const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsT_8atatime<double, float>(double const*, int, float*, int, long long) |
2251 | | |
2252 | | #ifdef HAVE_SSE2 |
2253 | | |
2254 | | template <class Tout> |
2255 | | void GDALCopyWordsByteTo16Bit(const GByte *const CPL_RESTRICT pSrcData, |
2256 | | int nSrcPixelStride, |
2257 | | Tout *const CPL_RESTRICT pDstData, |
2258 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2259 | 0 | { |
2260 | 0 | static_assert(std::is_integral<Tout>::value && |
2261 | 0 | sizeof(Tout) == sizeof(uint16_t), |
2262 | 0 | "Bad Tout"); |
2263 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2264 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2265 | 0 | { |
2266 | 0 | decltype(nWordCount) n = 0; |
2267 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2268 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2269 | 0 | reinterpret_cast<GByte *>(pDstData); |
2270 | 0 | for (; n < nWordCount - 15; n += 16) |
2271 | 0 | { |
2272 | 0 | __m128i xmm = _mm_loadu_si128( |
2273 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2274 | 0 | __m128i xmm0 = _mm_unpacklo_epi8(xmm, xmm_zero); |
2275 | 0 | __m128i xmm1 = _mm_unpackhi_epi8(xmm, xmm_zero); |
2276 | 0 | _mm_storeu_si128( |
2277 | 0 | reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 2), xmm0); |
2278 | 0 | _mm_storeu_si128( |
2279 | 0 | reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 2 + 16), xmm1); |
2280 | 0 | } |
2281 | 0 | #if defined(__clang__) |
2282 | 0 | #pragma clang loop vectorize(disable) |
2283 | 0 | #endif |
2284 | 0 | for (; n < nWordCount; n++) |
2285 | 0 | { |
2286 | 0 | pDstData[n] = pSrcData[n]; |
2287 | 0 | } |
2288 | 0 | } |
2289 | 0 | else |
2290 | 0 | { |
2291 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2292 | 0 | nDstPixelStride, nWordCount); |
2293 | 0 | } |
2294 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsByteTo16Bit<unsigned short>(unsigned char const*, int, unsigned short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsByteTo16Bit<short>(unsigned char const*, int, short*, int, long long) |
2295 | | |
2296 | | template <> |
2297 | | CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData, |
2298 | | int nSrcPixelStride, |
2299 | | GUInt16 *const CPL_RESTRICT pDstData, |
2300 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2301 | 0 | { |
2302 | 0 | GDALCopyWordsByteTo16Bit(pSrcData, nSrcPixelStride, pDstData, |
2303 | 0 | nDstPixelStride, nWordCount); |
2304 | 0 | } |
2305 | | |
2306 | | template <> |
2307 | | CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData, |
2308 | | int nSrcPixelStride, |
2309 | | GInt16 *const CPL_RESTRICT pDstData, |
2310 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2311 | 0 | { |
2312 | 0 | GDALCopyWordsByteTo16Bit(pSrcData, nSrcPixelStride, pDstData, |
2313 | 0 | nDstPixelStride, nWordCount); |
2314 | 0 | } |
2315 | | |
2316 | | template <class Tout> |
2317 | | void GDALCopyWordsByteTo32Bit(const GByte *const CPL_RESTRICT pSrcData, |
2318 | | int nSrcPixelStride, |
2319 | | Tout *const CPL_RESTRICT pDstData, |
2320 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2321 | 0 | { |
2322 | 0 | static_assert(std::is_integral<Tout>::value && |
2323 | 0 | sizeof(Tout) == sizeof(uint32_t), |
2324 | 0 | "Bad Tout"); |
2325 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2326 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2327 | 0 | { |
2328 | 0 | decltype(nWordCount) n = 0; |
2329 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2330 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2331 | 0 | reinterpret_cast<GByte *>(pDstData); |
2332 | 0 | for (; n < nWordCount - 15; n += 16) |
2333 | 0 | { |
2334 | 0 | __m128i xmm = _mm_loadu_si128( |
2335 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2336 | 0 | __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero); |
2337 | 0 | __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero); |
2338 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero); |
2339 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero); |
2340 | 0 | __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero); |
2341 | 0 | __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero); |
2342 | 0 | _mm_storeu_si128( |
2343 | 0 | reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4), xmm0); |
2344 | 0 | _mm_storeu_si128( |
2345 | 0 | reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 16), xmm1); |
2346 | 0 | _mm_storeu_si128( |
2347 | 0 | reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 32), xmm2); |
2348 | 0 | _mm_storeu_si128( |
2349 | 0 | reinterpret_cast<__m128i *>(pabyDstDataPtr + n * 4 + 48), xmm3); |
2350 | 0 | } |
2351 | 0 | #if defined(__clang__) |
2352 | 0 | #pragma clang loop vectorize(disable) |
2353 | 0 | #endif |
2354 | 0 | for (; n < nWordCount; n++) |
2355 | 0 | { |
2356 | 0 | pDstData[n] = pSrcData[n]; |
2357 | 0 | } |
2358 | 0 | } |
2359 | 0 | else |
2360 | 0 | { |
2361 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2362 | 0 | nDstPixelStride, nWordCount); |
2363 | 0 | } |
2364 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsByteTo32Bit<unsigned int>(unsigned char const*, int, unsigned int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsByteTo32Bit<int>(unsigned char const*, int, int*, int, long long) |
2365 | | |
2366 | | template <> |
2367 | | CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData, |
2368 | | int nSrcPixelStride, |
2369 | | GUInt32 *const CPL_RESTRICT pDstData, |
2370 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2371 | 0 | { |
2372 | 0 | GDALCopyWordsByteTo32Bit(pSrcData, nSrcPixelStride, pDstData, |
2373 | 0 | nDstPixelStride, nWordCount); |
2374 | 0 | } |
2375 | | |
2376 | | template <> |
2377 | | CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData, |
2378 | | int nSrcPixelStride, |
2379 | | GInt32 *const CPL_RESTRICT pDstData, |
2380 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2381 | 0 | { |
2382 | 0 | GDALCopyWordsByteTo32Bit(pSrcData, nSrcPixelStride, pDstData, |
2383 | 0 | nDstPixelStride, nWordCount); |
2384 | 0 | } |
2385 | | |
2386 | | template <> |
2387 | | CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData, |
2388 | | int nSrcPixelStride, |
2389 | | float *const CPL_RESTRICT pDstData, |
2390 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2391 | 0 | { |
2392 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2393 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2394 | 0 | { |
2395 | 0 | decltype(nWordCount) n = 0; |
2396 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2397 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2398 | 0 | reinterpret_cast<GByte *>(pDstData); |
2399 | 0 | for (; n < nWordCount - 15; n += 16) |
2400 | 0 | { |
2401 | 0 | __m128i xmm = _mm_loadu_si128( |
2402 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2403 | 0 | __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero); |
2404 | 0 | __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero); |
2405 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero); |
2406 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero); |
2407 | 0 | __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero); |
2408 | 0 | __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero); |
2409 | 0 | __m128 xmm0_f = _mm_cvtepi32_ps(xmm0); |
2410 | 0 | __m128 xmm1_f = _mm_cvtepi32_ps(xmm1); |
2411 | 0 | __m128 xmm2_f = _mm_cvtepi32_ps(xmm2); |
2412 | 0 | __m128 xmm3_f = _mm_cvtepi32_ps(xmm3); |
2413 | 0 | _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4), |
2414 | 0 | xmm0_f); |
2415 | 0 | _mm_storeu_ps( |
2416 | 0 | reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f); |
2417 | 0 | _mm_storeu_ps( |
2418 | 0 | reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 32), xmm2_f); |
2419 | 0 | _mm_storeu_ps( |
2420 | 0 | reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 48), xmm3_f); |
2421 | 0 | } |
2422 | 0 | #if defined(__clang__) |
2423 | 0 | #pragma clang loop vectorize(disable) |
2424 | 0 | #endif |
2425 | 0 | for (; n < nWordCount; n++) |
2426 | 0 | { |
2427 | 0 | pDstData[n] = pSrcData[n]; |
2428 | 0 | } |
2429 | 0 | } |
2430 | 0 | else |
2431 | 0 | { |
2432 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2433 | 0 | nDstPixelStride, nWordCount); |
2434 | 0 | } |
2435 | 0 | } |
2436 | | |
2437 | | template <> |
2438 | | CPL_NOINLINE void GDALCopyWordsT(const GByte *const CPL_RESTRICT pSrcData, |
2439 | | int nSrcPixelStride, |
2440 | | double *const CPL_RESTRICT pDstData, |
2441 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2442 | 0 | { |
2443 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2444 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2445 | 0 | { |
2446 | 0 | decltype(nWordCount) n = 0; |
2447 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2448 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2449 | 0 | reinterpret_cast<GByte *>(pDstData); |
2450 | 0 | for (; n < nWordCount - 15; n += 16) |
2451 | 0 | { |
2452 | 0 | __m128i xmm = _mm_loadu_si128( |
2453 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2454 | 0 | __m128i xmm_low = _mm_unpacklo_epi8(xmm, xmm_zero); |
2455 | 0 | __m128i xmm_high = _mm_unpackhi_epi8(xmm, xmm_zero); |
2456 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm_low, xmm_zero); |
2457 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm_low, xmm_zero); |
2458 | 0 | __m128i xmm2 = _mm_unpacklo_epi16(xmm_high, xmm_zero); |
2459 | 0 | __m128i xmm3 = _mm_unpackhi_epi16(xmm_high, xmm_zero); |
2460 | |
|
2461 | | #if defined(__AVX2__) && defined(slightly_slower_than_SSE2) |
2462 | | _mm256_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8), |
2463 | | _mm256_cvtepi32_pd(xmm0)); |
2464 | | _mm256_storeu_pd( |
2465 | | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32), |
2466 | | _mm256_cvtepi32_pd(xmm1)); |
2467 | | _mm256_storeu_pd( |
2468 | | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 64), |
2469 | | _mm256_cvtepi32_pd(xmm2)); |
2470 | | _mm256_storeu_pd( |
2471 | | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 96), |
2472 | | _mm256_cvtepi32_pd(xmm3)); |
2473 | | #else |
2474 | 0 | __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0); |
2475 | 0 | __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1); |
2476 | 0 | __m128d xmm2_low_d = _mm_cvtepi32_pd(xmm2); |
2477 | 0 | __m128d xmm3_low_d = _mm_cvtepi32_pd(xmm3); |
2478 | 0 | xmm0 = _mm_srli_si128(xmm0, 8); |
2479 | 0 | xmm1 = _mm_srli_si128(xmm1, 8); |
2480 | 0 | xmm2 = _mm_srli_si128(xmm2, 8); |
2481 | 0 | xmm3 = _mm_srli_si128(xmm3, 8); |
2482 | 0 | __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0); |
2483 | 0 | __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1); |
2484 | 0 | __m128d xmm2_high_d = _mm_cvtepi32_pd(xmm2); |
2485 | 0 | __m128d xmm3_high_d = _mm_cvtepi32_pd(xmm3); |
2486 | |
|
2487 | 0 | _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8), |
2488 | 0 | xmm0_low_d); |
2489 | 0 | _mm_storeu_pd( |
2490 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16), |
2491 | 0 | xmm0_high_d); |
2492 | 0 | _mm_storeu_pd( |
2493 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32), |
2494 | 0 | xmm1_low_d); |
2495 | 0 | _mm_storeu_pd( |
2496 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48), |
2497 | 0 | xmm1_high_d); |
2498 | 0 | _mm_storeu_pd( |
2499 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 64), |
2500 | 0 | xmm2_low_d); |
2501 | 0 | _mm_storeu_pd( |
2502 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 80), |
2503 | 0 | xmm2_high_d); |
2504 | 0 | _mm_storeu_pd( |
2505 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 96), |
2506 | 0 | xmm3_low_d); |
2507 | 0 | _mm_storeu_pd( |
2508 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 112), |
2509 | 0 | xmm3_high_d); |
2510 | 0 | #endif |
2511 | 0 | } |
2512 | 0 | #if defined(__clang__) |
2513 | 0 | #pragma clang loop vectorize(disable) |
2514 | 0 | #endif |
2515 | 0 | for (; n < nWordCount; n++) |
2516 | 0 | { |
2517 | 0 | pDstData[n] = pSrcData[n]; |
2518 | 0 | } |
2519 | 0 | } |
2520 | 0 | else |
2521 | 0 | { |
2522 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2523 | 0 | nDstPixelStride, nWordCount); |
2524 | 0 | } |
2525 | 0 | } |
2526 | | |
2527 | | template <> |
2528 | | CPL_NOINLINE void GDALCopyWordsT(const uint8_t *const CPL_RESTRICT pSrcData, |
2529 | | int nSrcPixelStride, |
2530 | | int8_t *const CPL_RESTRICT pDstData, |
2531 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2532 | 0 | { |
2533 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2534 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2535 | 0 | { |
2536 | 0 | decltype(nWordCount) n = 0; |
2537 | 0 | const __m128i xmm_127 = _mm_set1_epi8(127); |
2538 | 0 | for (; n < nWordCount - 31; n += 32) |
2539 | 0 | { |
2540 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2541 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2542 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2543 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 16)); |
2544 | 0 | xmm0 = _mm_min_epu8(xmm0, xmm_127); |
2545 | 0 | xmm1 = _mm_min_epu8(xmm1, xmm_127); |
2546 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2547 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16), |
2548 | 0 | xmm1); |
2549 | 0 | } |
2550 | 0 | #if defined(__clang__) |
2551 | 0 | #pragma clang loop vectorize(disable) |
2552 | 0 | #endif |
2553 | 0 | for (; n < nWordCount; n++) |
2554 | 0 | { |
2555 | 0 | pDstData[n] = static_cast<int8_t>(std::min<int>(pSrcData[n], 127)); |
2556 | 0 | } |
2557 | 0 | } |
2558 | 0 | else |
2559 | 0 | { |
2560 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2561 | 0 | nDstPixelStride, nWordCount); |
2562 | 0 | } |
2563 | 0 | } |
2564 | | |
2565 | | template <> |
2566 | | CPL_NOINLINE void GDALCopyWordsT(const int8_t *const CPL_RESTRICT pSrcData, |
2567 | | int nSrcPixelStride, |
2568 | | uint8_t *const CPL_RESTRICT pDstData, |
2569 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2570 | 0 | { |
2571 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2572 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2573 | 0 | { |
2574 | 0 | decltype(nWordCount) n = 0; |
2575 | 0 | #if !(defined(__SSE4_1__) || defined(__AVX__) || \ |
2576 | 0 | defined(USE_NEON_OPTIMIZATIONS)) |
2577 | 0 | const __m128i xmm_INT8_to_UINT8 = _mm_set1_epi8(-128); |
2578 | 0 | #endif |
2579 | 0 | for (; n < nWordCount - 31; n += 32) |
2580 | 0 | { |
2581 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2582 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2583 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2584 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 16)); |
2585 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2586 | | xmm0 = _mm_max_epi8(xmm0, _mm_setzero_si128()); |
2587 | | xmm1 = _mm_max_epi8(xmm1, _mm_setzero_si128()); |
2588 | | #else |
2589 | 0 | xmm0 = _mm_add_epi8(xmm0, xmm_INT8_to_UINT8); |
2590 | 0 | xmm1 = _mm_add_epi8(xmm1, xmm_INT8_to_UINT8); |
2591 | 0 | xmm0 = _mm_max_epu8(xmm0, xmm_INT8_to_UINT8); |
2592 | 0 | xmm1 = _mm_max_epu8(xmm1, xmm_INT8_to_UINT8); |
2593 | 0 | xmm0 = _mm_sub_epi8(xmm0, xmm_INT8_to_UINT8); |
2594 | 0 | xmm1 = _mm_sub_epi8(xmm1, xmm_INT8_to_UINT8); |
2595 | 0 | #endif |
2596 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2597 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16), |
2598 | 0 | xmm1); |
2599 | 0 | } |
2600 | 0 | #if defined(__clang__) |
2601 | 0 | #pragma clang loop vectorize(disable) |
2602 | 0 | #endif |
2603 | 0 | for (; n < nWordCount; n++) |
2604 | 0 | { |
2605 | 0 | pDstData[n] = static_cast<uint8_t>(std::max<int>(pSrcData[n], 0)); |
2606 | 0 | } |
2607 | 0 | } |
2608 | 0 | else |
2609 | 0 | { |
2610 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2611 | 0 | nDstPixelStride, nWordCount); |
2612 | 0 | } |
2613 | 0 | } |
2614 | | |
2615 | | template <> |
2616 | | CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData, |
2617 | | int nSrcPixelStride, |
2618 | | uint8_t *const CPL_RESTRICT pDstData, |
2619 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2620 | 0 | { |
2621 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2622 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2623 | 0 | { |
2624 | 0 | decltype(nWordCount) n = 0; |
2625 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2626 | | const auto xmm_MAX_INT16 = _mm_set1_epi16(32767); |
2627 | | #else |
2628 | | // In SSE2, min_epu16 does not exist, so shift from |
2629 | | // UInt16 to SInt16 to be able to use min_epi16 |
2630 | 0 | const __m128i xmm_UINT16_to_INT16 = _mm_set1_epi16(-32768); |
2631 | 0 | const __m128i xmm_m255_shifted = _mm_set1_epi16(255 - 32768); |
2632 | 0 | #endif |
2633 | 0 | for (; n < nWordCount - 15; n += 16) |
2634 | 0 | { |
2635 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2636 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2637 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2638 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
2639 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2640 | | xmm0 = _mm_min_epu16(xmm0, xmm_MAX_INT16); |
2641 | | xmm1 = _mm_min_epu16(xmm1, xmm_MAX_INT16); |
2642 | | #else |
2643 | 0 | xmm0 = _mm_add_epi16(xmm0, xmm_UINT16_to_INT16); |
2644 | 0 | xmm1 = _mm_add_epi16(xmm1, xmm_UINT16_to_INT16); |
2645 | 0 | xmm0 = _mm_min_epi16(xmm0, xmm_m255_shifted); |
2646 | 0 | xmm1 = _mm_min_epi16(xmm1, xmm_m255_shifted); |
2647 | 0 | xmm0 = _mm_sub_epi16(xmm0, xmm_UINT16_to_INT16); |
2648 | 0 | xmm1 = _mm_sub_epi16(xmm1, xmm_UINT16_to_INT16); |
2649 | 0 | #endif |
2650 | 0 | xmm0 = _mm_packus_epi16(xmm0, xmm1); |
2651 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2652 | 0 | } |
2653 | 0 | #if defined(__clang__) |
2654 | 0 | #pragma clang loop vectorize(disable) |
2655 | 0 | #endif |
2656 | 0 | for (; n < nWordCount; n++) |
2657 | 0 | { |
2658 | 0 | pDstData[n] = static_cast<uint8_t>(std::min<int>(pSrcData[n], 255)); |
2659 | 0 | } |
2660 | 0 | } |
2661 | 0 | else |
2662 | 0 | { |
2663 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2664 | 0 | nDstPixelStride, nWordCount); |
2665 | 0 | } |
2666 | 0 | } |
2667 | | |
2668 | | template <> |
2669 | | CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData, |
2670 | | int nSrcPixelStride, |
2671 | | int16_t *const CPL_RESTRICT pDstData, |
2672 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2673 | 0 | { |
2674 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2675 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2676 | 0 | { |
2677 | 0 | decltype(nWordCount) n = 0; |
2678 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2679 | | const __m128i xmm_MAX_INT16 = _mm_set1_epi16(32767); |
2680 | | #else |
2681 | | // In SSE2, min_epu16 does not exist, so shift from |
2682 | | // UInt16 to SInt16 to be able to use min_epi16 |
2683 | 0 | const __m128i xmm_UINT16_to_INT16 = _mm_set1_epi16(-32768); |
2684 | 0 | const __m128i xmm_32767_shifted = _mm_set1_epi16(32767 - 32768); |
2685 | 0 | #endif |
2686 | 0 | for (; n < nWordCount - 15; n += 16) |
2687 | 0 | { |
2688 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2689 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2690 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2691 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
2692 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2693 | | xmm0 = _mm_min_epu16(xmm0, xmm_MAX_INT16); |
2694 | | xmm1 = _mm_min_epu16(xmm1, xmm_MAX_INT16); |
2695 | | #else |
2696 | 0 | xmm0 = _mm_add_epi16(xmm0, xmm_UINT16_to_INT16); |
2697 | 0 | xmm1 = _mm_add_epi16(xmm1, xmm_UINT16_to_INT16); |
2698 | 0 | xmm0 = _mm_min_epi16(xmm0, xmm_32767_shifted); |
2699 | 0 | xmm1 = _mm_min_epi16(xmm1, xmm_32767_shifted); |
2700 | 0 | xmm0 = _mm_sub_epi16(xmm0, xmm_UINT16_to_INT16); |
2701 | 0 | xmm1 = _mm_sub_epi16(xmm1, xmm_UINT16_to_INT16); |
2702 | 0 | #endif |
2703 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2704 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8), |
2705 | 0 | xmm1); |
2706 | 0 | } |
2707 | 0 | #if defined(__clang__) |
2708 | 0 | #pragma clang loop vectorize(disable) |
2709 | 0 | #endif |
2710 | 0 | for (; n < nWordCount; n++) |
2711 | 0 | { |
2712 | 0 | pDstData[n] = |
2713 | 0 | static_cast<int16_t>(std::min<int>(pSrcData[n], 32767)); |
2714 | 0 | } |
2715 | 0 | } |
2716 | 0 | else |
2717 | 0 | { |
2718 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2719 | 0 | nDstPixelStride, nWordCount); |
2720 | 0 | } |
2721 | 0 | } |
2722 | | |
2723 | | template <> |
2724 | | CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData, |
2725 | | int nSrcPixelStride, |
2726 | | uint16_t *const CPL_RESTRICT pDstData, |
2727 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2728 | 0 | { |
2729 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2730 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2731 | 0 | { |
2732 | 0 | decltype(nWordCount) n = 0; |
2733 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2734 | 0 | for (; n < nWordCount - 15; n += 16) |
2735 | 0 | { |
2736 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2737 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2738 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2739 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
2740 | 0 | xmm0 = _mm_max_epi16(xmm0, xmm_zero); |
2741 | 0 | xmm1 = _mm_max_epi16(xmm1, xmm_zero); |
2742 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2743 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8), |
2744 | 0 | xmm1); |
2745 | 0 | } |
2746 | 0 | #if defined(__clang__) |
2747 | 0 | #pragma clang loop vectorize(disable) |
2748 | 0 | #endif |
2749 | 0 | for (; n < nWordCount; n++) |
2750 | 0 | { |
2751 | 0 | pDstData[n] = static_cast<uint16_t>(std::max<int>(pSrcData[n], 0)); |
2752 | 0 | } |
2753 | 0 | } |
2754 | 0 | else |
2755 | 0 | { |
2756 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2757 | 0 | nDstPixelStride, nWordCount); |
2758 | 0 | } |
2759 | 0 | } |
2760 | | |
2761 | | template <> |
2762 | | CPL_NOINLINE void GDALCopyWordsT(const uint32_t *const CPL_RESTRICT pSrcData, |
2763 | | int nSrcPixelStride, |
2764 | | int32_t *const CPL_RESTRICT pDstData, |
2765 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2766 | 0 | { |
2767 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2768 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2769 | 0 | { |
2770 | 0 | decltype(nWordCount) n = 0; |
2771 | 0 | const __m128i xmm_MAX_INT = _mm_set1_epi32(INT_MAX); |
2772 | 0 | [[maybe_unused]] const __m128i bias = _mm_set1_epi32(INT_MIN); |
2773 | 0 | [[maybe_unused]] const __m128i xmm_MAX_INT_biased = |
2774 | 0 | _mm_xor_si128(xmm_MAX_INT, bias); |
2775 | 0 | for (; n < nWordCount - 7; n += 8) |
2776 | 0 | { |
2777 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2778 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2779 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2780 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 4)); |
2781 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2782 | | xmm0 = _mm_min_epu32(xmm0, xmm_MAX_INT); |
2783 | | xmm1 = _mm_min_epu32(xmm1, xmm_MAX_INT); |
2784 | | #else |
2785 | 0 | const __m128i xmm0_biased = _mm_xor_si128(xmm0, bias); |
2786 | 0 | const __m128i mask0 = |
2787 | 0 | _mm_cmplt_epi32(xmm0_biased, xmm_MAX_INT_biased); |
2788 | 0 | xmm0 = GDALIfThenElse(mask0, xmm0, xmm_MAX_INT); |
2789 | |
|
2790 | 0 | const __m128i xmm1_biased = _mm_xor_si128(xmm1, bias); |
2791 | 0 | const __m128i mask1 = |
2792 | 0 | _mm_cmplt_epi32(xmm1_biased, xmm_MAX_INT_biased); |
2793 | 0 | xmm1 = GDALIfThenElse(mask1, xmm1, xmm_MAX_INT); |
2794 | 0 | #endif |
2795 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2796 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 4), |
2797 | 0 | xmm1); |
2798 | 0 | } |
2799 | 0 | #if defined(__clang__) |
2800 | 0 | #pragma clang loop vectorize(disable) |
2801 | 0 | #endif |
2802 | 0 | for (; n < nWordCount; n++) |
2803 | 0 | { |
2804 | 0 | pDstData[n] = |
2805 | 0 | static_cast<int32_t>(std::min<uint32_t>(pSrcData[n], INT_MAX)); |
2806 | 0 | } |
2807 | 0 | } |
2808 | 0 | else |
2809 | 0 | { |
2810 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2811 | 0 | nDstPixelStride, nWordCount); |
2812 | 0 | } |
2813 | 0 | } |
2814 | | |
2815 | | template <> |
2816 | | CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData, |
2817 | | int nSrcPixelStride, |
2818 | | uint32_t *const CPL_RESTRICT pDstData, |
2819 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2820 | 0 | { |
2821 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2822 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2823 | 0 | { |
2824 | 0 | decltype(nWordCount) n = 0; |
2825 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2826 | 0 | for (; n < nWordCount - 7; n += 8) |
2827 | 0 | { |
2828 | 0 | __m128i xmm0 = _mm_loadu_si128( |
2829 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2830 | 0 | __m128i xmm1 = _mm_loadu_si128( |
2831 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 4)); |
2832 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
2833 | | xmm0 = _mm_max_epi32(xmm0, xmm_zero); |
2834 | | xmm1 = _mm_max_epi32(xmm1, xmm_zero); |
2835 | | #else |
2836 | 0 | const __m128i mask0 = _mm_cmpgt_epi32(xmm0, xmm_zero); |
2837 | 0 | const __m128i mask1 = _mm_cmpgt_epi32(xmm1, xmm_zero); |
2838 | 0 | xmm0 = _mm_and_si128(xmm0, mask0); |
2839 | 0 | xmm1 = _mm_and_si128(xmm1, mask1); |
2840 | 0 | #endif |
2841 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), xmm0); |
2842 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 4), |
2843 | 0 | xmm1); |
2844 | 0 | } |
2845 | 0 | #if defined(__clang__) |
2846 | 0 | #pragma clang loop vectorize(disable) |
2847 | 0 | #endif |
2848 | 0 | for (; n < nWordCount; n++) |
2849 | 0 | { |
2850 | 0 | pDstData[n] = static_cast<uint32_t>(std::max(pSrcData[n], 0)); |
2851 | 0 | } |
2852 | 0 | } |
2853 | 0 | else |
2854 | 0 | { |
2855 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2856 | 0 | nDstPixelStride, nWordCount); |
2857 | 0 | } |
2858 | 0 | } |
2859 | | |
2860 | | template <> |
2861 | | CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData, |
2862 | | int nSrcPixelStride, |
2863 | | float *const CPL_RESTRICT pDstData, |
2864 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2865 | 0 | { |
2866 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2867 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2868 | 0 | { |
2869 | 0 | decltype(nWordCount) n = 0; |
2870 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2871 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2872 | 0 | reinterpret_cast<GByte *>(pDstData); |
2873 | 0 | for (; n < nWordCount - 7; n += 8) |
2874 | 0 | { |
2875 | 0 | __m128i xmm = _mm_loadu_si128( |
2876 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2877 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm, xmm_zero); |
2878 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm, xmm_zero); |
2879 | 0 | __m128 xmm0_f = _mm_cvtepi32_ps(xmm0); |
2880 | 0 | __m128 xmm1_f = _mm_cvtepi32_ps(xmm1); |
2881 | 0 | _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4), |
2882 | 0 | xmm0_f); |
2883 | 0 | _mm_storeu_ps( |
2884 | 0 | reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f); |
2885 | 0 | } |
2886 | 0 | #if defined(__clang__) |
2887 | 0 | #pragma clang loop vectorize(disable) |
2888 | 0 | #endif |
2889 | 0 | for (; n < nWordCount; n++) |
2890 | 0 | { |
2891 | 0 | pDstData[n] = pSrcData[n]; |
2892 | 0 | } |
2893 | 0 | } |
2894 | 0 | else |
2895 | 0 | { |
2896 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2897 | 0 | nDstPixelStride, nWordCount); |
2898 | 0 | } |
2899 | 0 | } |
2900 | | |
2901 | | template <> |
2902 | | CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData, |
2903 | | int nSrcPixelStride, |
2904 | | float *const CPL_RESTRICT pDstData, |
2905 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2906 | 0 | { |
2907 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2908 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2909 | 0 | { |
2910 | 0 | decltype(nWordCount) n = 0; |
2911 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2912 | 0 | reinterpret_cast<GByte *>(pDstData); |
2913 | 0 | for (; n < nWordCount - 7; n += 8) |
2914 | 0 | { |
2915 | 0 | __m128i xmm = _mm_loadu_si128( |
2916 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2917 | 0 | const auto sign = _mm_srai_epi16(xmm, 15); |
2918 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm, sign); |
2919 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm, sign); |
2920 | 0 | __m128 xmm0_f = _mm_cvtepi32_ps(xmm0); |
2921 | 0 | __m128 xmm1_f = _mm_cvtepi32_ps(xmm1); |
2922 | 0 | _mm_storeu_ps(reinterpret_cast<float *>(pabyDstDataPtr + n * 4), |
2923 | 0 | xmm0_f); |
2924 | 0 | _mm_storeu_ps( |
2925 | 0 | reinterpret_cast<float *>(pabyDstDataPtr + n * 4 + 16), xmm1_f); |
2926 | 0 | } |
2927 | 0 | #if defined(__clang__) |
2928 | 0 | #pragma clang loop vectorize(disable) |
2929 | 0 | #endif |
2930 | 0 | for (; n < nWordCount; n++) |
2931 | 0 | { |
2932 | 0 | pDstData[n] = pSrcData[n]; |
2933 | 0 | } |
2934 | 0 | } |
2935 | 0 | else |
2936 | 0 | { |
2937 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2938 | 0 | nDstPixelStride, nWordCount); |
2939 | 0 | } |
2940 | 0 | } |
2941 | | |
2942 | | template <> |
2943 | | CPL_NOINLINE void GDALCopyWordsT(const uint16_t *const CPL_RESTRICT pSrcData, |
2944 | | int nSrcPixelStride, |
2945 | | double *const CPL_RESTRICT pDstData, |
2946 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
2947 | 0 | { |
2948 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
2949 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
2950 | 0 | { |
2951 | 0 | decltype(nWordCount) n = 0; |
2952 | 0 | const __m128i xmm_zero = _mm_setzero_si128(); |
2953 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
2954 | 0 | reinterpret_cast<GByte *>(pDstData); |
2955 | 0 | for (; n < nWordCount - 7; n += 8) |
2956 | 0 | { |
2957 | 0 | __m128i xmm = _mm_loadu_si128( |
2958 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
2959 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm, xmm_zero); |
2960 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm, xmm_zero); |
2961 | |
|
2962 | 0 | __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0); |
2963 | 0 | __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1); |
2964 | 0 | xmm0 = _mm_srli_si128(xmm0, 8); |
2965 | 0 | xmm1 = _mm_srli_si128(xmm1, 8); |
2966 | 0 | __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0); |
2967 | 0 | __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1); |
2968 | |
|
2969 | 0 | _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8), |
2970 | 0 | xmm0_low_d); |
2971 | 0 | _mm_storeu_pd( |
2972 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16), |
2973 | 0 | xmm0_high_d); |
2974 | 0 | _mm_storeu_pd( |
2975 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32), |
2976 | 0 | xmm1_low_d); |
2977 | 0 | _mm_storeu_pd( |
2978 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48), |
2979 | 0 | xmm1_high_d); |
2980 | 0 | } |
2981 | 0 | #if defined(__clang__) |
2982 | 0 | #pragma clang loop vectorize(disable) |
2983 | 0 | #endif |
2984 | 0 | for (; n < nWordCount; n++) |
2985 | 0 | { |
2986 | 0 | pDstData[n] = pSrcData[n]; |
2987 | 0 | } |
2988 | 0 | } |
2989 | 0 | else |
2990 | 0 | { |
2991 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
2992 | 0 | nDstPixelStride, nWordCount); |
2993 | 0 | } |
2994 | 0 | } |
2995 | | |
2996 | | template <> |
2997 | | CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData, |
2998 | | int nSrcPixelStride, |
2999 | | double *const CPL_RESTRICT pDstData, |
3000 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3001 | 0 | { |
3002 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
3003 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
3004 | 0 | { |
3005 | 0 | decltype(nWordCount) n = 0; |
3006 | 0 | GByte *CPL_RESTRICT pabyDstDataPtr = |
3007 | 0 | reinterpret_cast<GByte *>(pDstData); |
3008 | 0 | for (; n < nWordCount - 7; n += 8) |
3009 | 0 | { |
3010 | 0 | __m128i xmm = _mm_loadu_si128( |
3011 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
3012 | 0 | const auto sign = _mm_srai_epi16(xmm, 15); |
3013 | 0 | __m128i xmm0 = _mm_unpacklo_epi16(xmm, sign); |
3014 | 0 | __m128i xmm1 = _mm_unpackhi_epi16(xmm, sign); |
3015 | |
|
3016 | 0 | __m128d xmm0_low_d = _mm_cvtepi32_pd(xmm0); |
3017 | 0 | __m128d xmm1_low_d = _mm_cvtepi32_pd(xmm1); |
3018 | 0 | xmm0 = _mm_srli_si128(xmm0, 8); |
3019 | 0 | xmm1 = _mm_srli_si128(xmm1, 8); |
3020 | 0 | __m128d xmm0_high_d = _mm_cvtepi32_pd(xmm0); |
3021 | 0 | __m128d xmm1_high_d = _mm_cvtepi32_pd(xmm1); |
3022 | |
|
3023 | 0 | _mm_storeu_pd(reinterpret_cast<double *>(pabyDstDataPtr + n * 8), |
3024 | 0 | xmm0_low_d); |
3025 | 0 | _mm_storeu_pd( |
3026 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 16), |
3027 | 0 | xmm0_high_d); |
3028 | 0 | _mm_storeu_pd( |
3029 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 32), |
3030 | 0 | xmm1_low_d); |
3031 | 0 | _mm_storeu_pd( |
3032 | 0 | reinterpret_cast<double *>(pabyDstDataPtr + n * 8 + 48), |
3033 | 0 | xmm1_high_d); |
3034 | 0 | } |
3035 | 0 | #if defined(__clang__) |
3036 | 0 | #pragma clang loop vectorize(disable) |
3037 | 0 | #endif |
3038 | 0 | for (; n < nWordCount; n++) |
3039 | 0 | { |
3040 | 0 | pDstData[n] = pSrcData[n]; |
3041 | 0 | } |
3042 | 0 | } |
3043 | 0 | else |
3044 | 0 | { |
3045 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
3046 | 0 | nDstPixelStride, nWordCount); |
3047 | 0 | } |
3048 | 0 | } |
3049 | | |
3050 | | // ---- AVX2 helpers for int32 narrowing (runtime dispatch) ---- |
3051 | | |
3052 | | #if defined(HAVE_AVX2_DISPATCH) |
3053 | | #if !defined(_MSC_VER) |
3054 | | __attribute__((target("avx2"))) |
3055 | | #endif |
3056 | | static void GDALCopyWordsInt32ToUInt8_AVX2(const int32_t *CPL_RESTRICT pSrc, |
3057 | | uint8_t *CPL_RESTRICT pDst, |
3058 | | GPtrDiff_t nWordCount) |
3059 | 0 | { |
3060 | 0 | const __m256i permuteIdx = _mm256_setr_epi32(0, 4, 1, 5, 2, 6, 3, 7); |
3061 | 0 | GPtrDiff_t n = 0; |
3062 | 0 | for (; n < nWordCount - 31; n += 32) |
3063 | 0 | { |
3064 | 0 | __m256i v0 = |
3065 | 0 | _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n)); |
3066 | 0 | __m256i v1 = |
3067 | 0 | _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n + 8)); |
3068 | 0 | __m256i v2 = _mm256_loadu_si256( |
3069 | 0 | reinterpret_cast<const __m256i *>(pSrc + n + 16)); |
3070 | 0 | __m256i v3 = _mm256_loadu_si256( |
3071 | 0 | reinterpret_cast<const __m256i *>(pSrc + n + 24)); |
3072 | | // Clamp to [0, 255] |
3073 | | // Pack int32 -> int16 -> uint8, then fix cross-lane ordering |
3074 | 0 | __m256i ab16 = _mm256_packs_epi32(v0, v1); |
3075 | 0 | __m256i cd16 = _mm256_packs_epi32(v2, v3); |
3076 | 0 | __m256i bytes = _mm256_packus_epi16(ab16, cd16); |
3077 | 0 | bytes = _mm256_permutevar8x32_epi32(bytes, permuteIdx); |
3078 | 0 | _mm256_storeu_si256(reinterpret_cast<__m256i *>(pDst + n), bytes); |
3079 | 0 | } |
3080 | 0 | #if defined(__clang__) |
3081 | 0 | #pragma clang loop vectorize(disable) |
3082 | 0 | #endif |
3083 | 0 | for (; n < nWordCount; n++) |
3084 | 0 | { |
3085 | 0 | pDst[n] = static_cast<uint8_t>(std::clamp(pSrc[n], 0, 255)); |
3086 | 0 | } |
3087 | 0 | } |
3088 | | |
3089 | | #if !defined(_MSC_VER) |
3090 | | __attribute__((target("avx2"))) |
3091 | | #endif |
3092 | | static void GDALCopyWordsInt32ToUInt16_AVX2(const int32_t *CPL_RESTRICT pSrc, |
3093 | | uint16_t *CPL_RESTRICT pDst, |
3094 | | GPtrDiff_t nWordCount) |
3095 | 0 | { |
3096 | | // _mm256_packus_epi32(v0, v1) produces per-lane interleaved result: |
3097 | | // [v0_lo4, v1_lo4, v0_hi4, v1_hi4] (in uint16 pairs per 32-bit lane) |
3098 | | // Permute to deinterleave: all v0 values first, then all v1 values |
3099 | 0 | const __m256i permuteIdx = _mm256_setr_epi32(0, 1, 4, 5, 2, 3, 6, 7); |
3100 | 0 | GPtrDiff_t n = 0; |
3101 | 0 | for (; n < nWordCount - 15; n += 16) |
3102 | 0 | { |
3103 | 0 | __m256i v0 = |
3104 | 0 | _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n)); |
3105 | 0 | __m256i v1 = |
3106 | 0 | _mm256_loadu_si256(reinterpret_cast<const __m256i *>(pSrc + n + 8)); |
3107 | | // Clamp to [0, 65535]: _mm256_packus_epi32 saturates uint |
3108 | 0 | __m256i packed = _mm256_packus_epi32(v0, v1); |
3109 | | // Fix cross-lane interleave from packus |
3110 | 0 | packed = _mm256_permutevar8x32_epi32(packed, permuteIdx); |
3111 | 0 | _mm256_storeu_si256(reinterpret_cast<__m256i *>(pDst + n), packed); |
3112 | 0 | } |
3113 | 0 | #if defined(__clang__) |
3114 | 0 | #pragma clang loop vectorize(disable) |
3115 | 0 | #endif |
3116 | 0 | for (; n < nWordCount; n++) |
3117 | 0 | { |
3118 | 0 | pDst[n] = static_cast<uint16_t>(std::clamp(pSrc[n], 0, 65535)); |
3119 | 0 | } |
3120 | 0 | } |
3121 | | #endif // HAVE_AVX2_DISPATCH |
3122 | | |
3123 | | // ---- int32 -> uint8 with clamping to [0, 255] ---- |
3124 | | template <> |
3125 | | CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData, |
3126 | | int nSrcPixelStride, |
3127 | | uint8_t *const CPL_RESTRICT pDstData, |
3128 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3129 | 0 | { |
3130 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
3131 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
3132 | 0 | { |
3133 | 0 | #if defined(HAVE_AVX2_DISPATCH) |
3134 | 0 | if (CPLHaveRuntimeAVX2()) |
3135 | 0 | { |
3136 | 0 | GDALCopyWordsInt32ToUInt8_AVX2(pSrcData, pDstData, nWordCount); |
3137 | 0 | return; |
3138 | 0 | } |
3139 | 0 | #endif |
3140 | | |
3141 | | // SSE2 path: 16 pixels per iteration |
3142 | 0 | decltype(nWordCount) n = 0; |
3143 | 0 | for (; n < nWordCount - 15; n += 16) |
3144 | 0 | { |
3145 | 0 | __m128i v0 = _mm_loadu_si128( |
3146 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
3147 | 0 | __m128i v1 = _mm_loadu_si128( |
3148 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 4)); |
3149 | 0 | __m128i v2 = _mm_loadu_si128( |
3150 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
3151 | 0 | __m128i v3 = _mm_loadu_si128( |
3152 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 12)); |
3153 | | // Pack int32->int16 with signed saturation to [-32768,32767] range |
3154 | 0 | __m128i lo16 = _mm_packs_epi32(v0, v1); |
3155 | 0 | __m128i hi16 = _mm_packs_epi32(v2, v3); |
3156 | | // Pack int16->uint8 with unsigned saturation to [0,255] range |
3157 | 0 | __m128i bytes = _mm_packus_epi16(lo16, hi16); |
3158 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), bytes); |
3159 | 0 | } |
3160 | 0 | #if defined(__clang__) |
3161 | 0 | #pragma clang loop vectorize(disable) |
3162 | 0 | #endif |
3163 | 0 | for (; n < nWordCount; n++) |
3164 | 0 | { |
3165 | 0 | pDstData[n] = static_cast<uint8_t>(std::clamp(pSrcData[n], 0, 255)); |
3166 | 0 | } |
3167 | 0 | } |
3168 | 0 | else |
3169 | 0 | { |
3170 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
3171 | 0 | nDstPixelStride, nWordCount); |
3172 | 0 | } |
3173 | 0 | } |
3174 | | |
3175 | | // ---- int32 -> uint16 with clamping to [0, 65535] ---- |
3176 | | template <> |
3177 | | CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData, |
3178 | | int nSrcPixelStride, |
3179 | | uint16_t *const CPL_RESTRICT pDstData, |
3180 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3181 | 0 | { |
3182 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
3183 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
3184 | 0 | { |
3185 | 0 | #if defined(HAVE_AVX2_DISPATCH) |
3186 | 0 | if (CPLHaveRuntimeAVX2()) |
3187 | 0 | { |
3188 | 0 | GDALCopyWordsInt32ToUInt16_AVX2(pSrcData, pDstData, nWordCount); |
3189 | 0 | return; |
3190 | 0 | } |
3191 | 0 | #endif |
3192 | 0 | decltype(nWordCount) n = 0; |
3193 | 0 | for (; n < nWordCount - 15; n += 16) |
3194 | 0 | { |
3195 | 0 | __m128i v0 = _mm_loadu_si128( |
3196 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
3197 | 0 | __m128i v1 = _mm_loadu_si128( |
3198 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 4)); |
3199 | 0 | __m128i v2 = _mm_loadu_si128( |
3200 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
3201 | 0 | __m128i v3 = _mm_loadu_si128( |
3202 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 12)); |
3203 | 0 | const auto packed_lo = GDAL_mm_packus_epi32(v0, v1); |
3204 | 0 | const auto packed_hi = GDAL_mm_packus_epi32(v2, v3); |
3205 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), |
3206 | 0 | packed_lo); |
3207 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8), |
3208 | 0 | packed_hi); |
3209 | 0 | } |
3210 | 0 | #if defined(__clang__) |
3211 | 0 | #pragma clang loop vectorize(disable) |
3212 | 0 | #endif |
3213 | 0 | for (; n < nWordCount; n++) |
3214 | 0 | { |
3215 | 0 | pDstData[n] = |
3216 | 0 | static_cast<uint16_t>(std::clamp(pSrcData[n], 0, 65535)); |
3217 | 0 | } |
3218 | 0 | } |
3219 | 0 | else |
3220 | 0 | { |
3221 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
3222 | 0 | nDstPixelStride, nWordCount); |
3223 | 0 | } |
3224 | 0 | } |
3225 | | |
3226 | | // ---- int32 -> int16 with clamping to [-32768, 32767] ---- |
3227 | | template <> |
3228 | | CPL_NOINLINE void GDALCopyWordsT(const int32_t *const CPL_RESTRICT pSrcData, |
3229 | | int nSrcPixelStride, |
3230 | | int16_t *const CPL_RESTRICT pDstData, |
3231 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3232 | 0 | { |
3233 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
3234 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
3235 | 0 | { |
3236 | | // SSE2 path: 16 pixels per iteration |
3237 | 0 | decltype(nWordCount) n = 0; |
3238 | 0 | for (; n < nWordCount - 15; n += 16) |
3239 | 0 | { |
3240 | 0 | __m128i v0 = _mm_loadu_si128( |
3241 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
3242 | 0 | __m128i v1 = _mm_loadu_si128( |
3243 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 4)); |
3244 | 0 | __m128i v2 = _mm_loadu_si128( |
3245 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
3246 | 0 | __m128i v3 = _mm_loadu_si128( |
3247 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 12)); |
3248 | | // Pack int32->int16 with signed saturation to [-32768,32767] range |
3249 | 0 | __m128i packed_lo = _mm_packs_epi32(v0, v1); |
3250 | 0 | __m128i packed_hi = _mm_packs_epi32(v2, v3); |
3251 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), |
3252 | 0 | packed_lo); |
3253 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 8), |
3254 | 0 | packed_hi); |
3255 | 0 | } |
3256 | 0 | #if defined(__clang__) |
3257 | 0 | #pragma clang loop vectorize(disable) |
3258 | 0 | #endif |
3259 | 0 | for (; n < nWordCount; n++) |
3260 | 0 | { |
3261 | 0 | pDstData[n] = |
3262 | 0 | static_cast<int16_t>(std::clamp(pSrcData[n], -32768, 32767)); |
3263 | 0 | } |
3264 | 0 | } |
3265 | 0 | else |
3266 | 0 | { |
3267 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
3268 | 0 | nDstPixelStride, nWordCount); |
3269 | 0 | } |
3270 | 0 | } |
3271 | | |
3272 | | // ---- int16 -> uint8 with clamping to [0, 255] ---- |
3273 | | template <> |
3274 | | CPL_NOINLINE void GDALCopyWordsT(const int16_t *const CPL_RESTRICT pSrcData, |
3275 | | int nSrcPixelStride, |
3276 | | uint8_t *const CPL_RESTRICT pDstData, |
3277 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3278 | 0 | { |
3279 | 0 | if (nSrcPixelStride == static_cast<int>(sizeof(*pSrcData)) && |
3280 | 0 | nDstPixelStride == static_cast<int>(sizeof(*pDstData))) |
3281 | 0 | { |
3282 | | // SSE2 path: 32 pixels per iteration |
3283 | 0 | decltype(nWordCount) n = 0; |
3284 | 0 | for (; n < nWordCount - 31; n += 32) |
3285 | 0 | { |
3286 | 0 | __m128i v0 = _mm_loadu_si128( |
3287 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n)); |
3288 | 0 | __m128i v1 = _mm_loadu_si128( |
3289 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 8)); |
3290 | 0 | __m128i v2 = _mm_loadu_si128( |
3291 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 16)); |
3292 | 0 | __m128i v3 = _mm_loadu_si128( |
3293 | 0 | reinterpret_cast<const __m128i *>(pSrcData + n + 24)); |
3294 | | // Pack int16->uint8 with unsigned saturation to [0, 255] range |
3295 | 0 | __m128i packed_lo = _mm_packus_epi16(v0, v1); |
3296 | 0 | __m128i packed_hi = _mm_packus_epi16(v2, v3); |
3297 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n), |
3298 | 0 | packed_lo); |
3299 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDstData + n + 16), |
3300 | 0 | packed_hi); |
3301 | 0 | } |
3302 | 0 | #if defined(__clang__) |
3303 | 0 | #pragma clang loop vectorize(disable) |
3304 | 0 | #endif |
3305 | 0 | for (; n < nWordCount; n++) |
3306 | 0 | { |
3307 | 0 | pDstData[n] = |
3308 | 0 | static_cast<uint8_t>(std::clamp<int>(pSrcData[n], 0, 255)); |
3309 | 0 | } |
3310 | 0 | } |
3311 | 0 | else |
3312 | 0 | { |
3313 | 0 | GDALCopyWordsGenericT(pSrcData, nSrcPixelStride, pDstData, |
3314 | 0 | nDstPixelStride, nWordCount); |
3315 | 0 | } |
3316 | 0 | } |
3317 | | |
3318 | | #endif // HAVE_SSE2 |
3319 | | |
3320 | | template <> |
3321 | | CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData, |
3322 | | int nSrcPixelStride, |
3323 | | GByte *const CPL_RESTRICT pDstData, |
3324 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3325 | 0 | { |
3326 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3327 | 0 | nDstPixelStride, nWordCount); |
3328 | 0 | } |
3329 | | |
3330 | | template <> |
3331 | | CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData, |
3332 | | int nSrcPixelStride, |
3333 | | GUInt16 *const CPL_RESTRICT pDstData, |
3334 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3335 | 0 | { |
3336 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3337 | 0 | nDstPixelStride, nWordCount); |
3338 | 0 | } |
3339 | | |
3340 | | template <> |
3341 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3342 | | int nSrcPixelStride, |
3343 | | double *const CPL_RESTRICT pDstData, |
3344 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3345 | 0 | { |
3346 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3347 | 0 | nDstPixelStride, nWordCount); |
3348 | 0 | } |
3349 | | |
3350 | | template <> |
3351 | | CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData, |
3352 | | int nSrcPixelStride, |
3353 | | float *const CPL_RESTRICT pDstData, |
3354 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3355 | 0 | { |
3356 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3357 | 0 | nDstPixelStride, nWordCount); |
3358 | 0 | } |
3359 | | |
3360 | | template <> |
3361 | | CPL_NOINLINE void GDALCopyWordsT(const GFloat16 *const CPL_RESTRICT pSrcData, |
3362 | | int nSrcPixelStride, |
3363 | | float *const CPL_RESTRICT pDstData, |
3364 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3365 | 0 | { |
3366 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3367 | 0 | nDstPixelStride, nWordCount); |
3368 | 0 | } |
3369 | | |
3370 | | template <> |
3371 | | CPL_NOINLINE void GDALCopyWordsT(const GFloat16 *const CPL_RESTRICT pSrcData, |
3372 | | int nSrcPixelStride, |
3373 | | double *const CPL_RESTRICT pDstData, |
3374 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3375 | 0 | { |
3376 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3377 | 0 | nDstPixelStride, nWordCount); |
3378 | 0 | } |
3379 | | |
3380 | | template <> |
3381 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3382 | | int nSrcPixelStride, |
3383 | | GByte *const CPL_RESTRICT pDstData, |
3384 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3385 | 0 | { |
3386 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3387 | 0 | nDstPixelStride, nWordCount); |
3388 | 0 | } |
3389 | | |
3390 | | template <> |
3391 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3392 | | int nSrcPixelStride, |
3393 | | GInt8 *const CPL_RESTRICT pDstData, |
3394 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3395 | 0 | { |
3396 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3397 | 0 | nDstPixelStride, nWordCount); |
3398 | 0 | } |
3399 | | |
3400 | | template <> |
3401 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3402 | | int nSrcPixelStride, |
3403 | | GInt16 *const CPL_RESTRICT pDstData, |
3404 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3405 | 0 | { |
3406 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3407 | 0 | nDstPixelStride, nWordCount); |
3408 | 0 | } |
3409 | | |
3410 | | template <> |
3411 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3412 | | int nSrcPixelStride, |
3413 | | GUInt16 *const CPL_RESTRICT pDstData, |
3414 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3415 | 0 | { |
3416 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3417 | 0 | nDstPixelStride, nWordCount); |
3418 | 0 | } |
3419 | | |
3420 | | template <> |
3421 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3422 | | int nSrcPixelStride, |
3423 | | GInt32 *const CPL_RESTRICT pDstData, |
3424 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3425 | 0 | { |
3426 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3427 | 0 | nDstPixelStride, nWordCount); |
3428 | 0 | } |
3429 | | |
3430 | | template <> |
3431 | | CPL_NOINLINE void GDALCopyWordsT(const float *const CPL_RESTRICT pSrcData, |
3432 | | int nSrcPixelStride, |
3433 | | GFloat16 *const CPL_RESTRICT pDstData, |
3434 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3435 | 0 | { |
3436 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3437 | 0 | nDstPixelStride, nWordCount); |
3438 | 0 | } |
3439 | | |
3440 | | template <> |
3441 | | CPL_NOINLINE void GDALCopyWordsT(const double *const CPL_RESTRICT pSrcData, |
3442 | | int nSrcPixelStride, |
3443 | | GFloat16 *const CPL_RESTRICT pDstData, |
3444 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3445 | 0 | { |
3446 | 0 | GDALCopyWordsT_8atatime(pSrcData, nSrcPixelStride, pDstData, |
3447 | 0 | nDstPixelStride, nWordCount); |
3448 | 0 | } |
3449 | | |
3450 | | /************************************************************************/ |
3451 | | /* GDALCopyWordsComplexT() */ |
3452 | | /************************************************************************/ |
3453 | | /** |
3454 | | * Template function, used to copy data from pSrcData into buffer |
3455 | | * pDstData, with stride nSrcPixelStride in the source data and |
3456 | | * stride nDstPixelStride in the destination data. Deals with the |
3457 | | * complex case, where input is complex and output is complex. |
3458 | | * |
3459 | | * @param pSrcData the source data buffer |
3460 | | * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels |
3461 | | * of interest. |
3462 | | * @param pDstData the destination buffer. |
3463 | | * @param nDstPixelStride the stride in the buffer pDstData for pixels of |
3464 | | * interest. |
3465 | | * @param nWordCount the total number of pixel words to copy |
3466 | | * |
3467 | | */ |
3468 | | template <class Tin, class Tout> |
3469 | | inline void GDALCopyWordsComplexT(const Tin *const CPL_RESTRICT pSrcData, |
3470 | | int nSrcPixelStride, |
3471 | | Tout *const CPL_RESTRICT pDstData, |
3472 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3473 | 0 | { |
3474 | 0 | decltype(nWordCount) nDstOffset = 0; |
3475 | 0 | const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData); |
3476 | 0 | char *const pDstDataPtr = reinterpret_cast<char *>(pDstData); |
3477 | |
|
3478 | 0 | for (decltype(nWordCount) n = 0; n < nWordCount; n++) |
3479 | 0 | { |
3480 | 0 | const Tin *const pPixelIn = |
3481 | 0 | reinterpret_cast<const Tin *>(pSrcDataPtr + n * nSrcPixelStride); |
3482 | 0 | Tout *const pPixelOut = |
3483 | 0 | reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset); |
3484 | |
|
3485 | 0 | GDALCopyWord(pPixelIn[0], pPixelOut[0]); |
3486 | 0 | GDALCopyWord(pPixelIn[1], pPixelOut[1]); |
3487 | |
|
3488 | 0 | nDstOffset += nDstPixelStride; |
3489 | 0 | } |
3490 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned char, short>(unsigned char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned char, int>(unsigned char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned char, cpl::Float16>(unsigned char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned char, float>(unsigned char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned char, double>(unsigned char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<signed char, short>(signed char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<signed char, int>(signed char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<signed char, cpl::Float16>(signed char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<signed char, float>(signed char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<signed char, double>(signed char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned short, short>(unsigned short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned short, int>(unsigned short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned short, cpl::Float16>(unsigned short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned short, float>(unsigned short const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned short, double>(unsigned short const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<short, short>(short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<short, int>(short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<short, cpl::Float16>(short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<short, float>(short const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<short, double>(short const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned int, short>(unsigned int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned int, int>(unsigned int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned int, cpl::Float16>(unsigned int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned int, float>(unsigned int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned int, double>(unsigned int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<int, short>(int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<int, int>(int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<int, cpl::Float16>(int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<int, float>(int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<int, double>(int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned long, short>(unsigned long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned long, int>(unsigned long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned long, cpl::Float16>(unsigned long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned long, float>(unsigned long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<unsigned long, double>(unsigned long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<long, short>(long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<long, int>(long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<long, cpl::Float16>(long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<long, float>(long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<long, double>(long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<cpl::Float16, short>(cpl::Float16 const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<cpl::Float16, int>(cpl::Float16 const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<cpl::Float16, cpl::Float16>(cpl::Float16 const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<cpl::Float16, float>(cpl::Float16 const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<cpl::Float16, double>(cpl::Float16 const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<float, short>(float const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<float, int>(float const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<float, cpl::Float16>(float const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<float, float>(float const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<float, double>(float const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<double, short>(double const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<double, int>(double const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<double, cpl::Float16>(double const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<double, float>(double const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexT<double, double>(double const*, int, double*, int, long long) |
3491 | | |
3492 | | /************************************************************************/ |
3493 | | /* GDALCopyWordsComplexOutT() */ |
3494 | | /************************************************************************/ |
3495 | | /** |
3496 | | * Template function, used to copy data from pSrcData into buffer |
3497 | | * pDstData, with stride nSrcPixelStride in the source data and |
3498 | | * stride nDstPixelStride in the destination data. Deals with the |
3499 | | * case where the value is real coming in, but complex going out. |
3500 | | * |
3501 | | * @param pSrcData the source data buffer |
3502 | | * @param nSrcPixelStride the stride, in the buffer pSrcData for pixels |
3503 | | * of interest, in bytes. |
3504 | | * @param pDstData the destination buffer. |
3505 | | * @param nDstPixelStride the stride in the buffer pDstData for pixels of |
3506 | | * interest, in bytes. |
3507 | | * @param nWordCount the total number of pixel words to copy |
3508 | | * |
3509 | | */ |
3510 | | template <class Tin, class Tout> |
3511 | | inline void GDALCopyWordsComplexOutT(const Tin *const CPL_RESTRICT pSrcData, |
3512 | | int nSrcPixelStride, |
3513 | | Tout *const CPL_RESTRICT pDstData, |
3514 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
3515 | 0 | { |
3516 | 0 | decltype(nWordCount) nDstOffset = 0; |
3517 | |
|
3518 | 0 | const Tout tOutZero = static_cast<Tout>(0); |
3519 | |
|
3520 | 0 | const char *const pSrcDataPtr = reinterpret_cast<const char *>(pSrcData); |
3521 | 0 | char *const pDstDataPtr = reinterpret_cast<char *>(pDstData); |
3522 | |
|
3523 | 0 | for (decltype(nWordCount) n = 0; n < nWordCount; n++) |
3524 | 0 | { |
3525 | 0 | const Tin tValue = |
3526 | 0 | *reinterpret_cast<const Tin *>(pSrcDataPtr + n * nSrcPixelStride); |
3527 | 0 | Tout *const pPixelOut = |
3528 | 0 | reinterpret_cast<Tout *>(pDstDataPtr + nDstOffset); |
3529 | 0 | GDALCopyWord(tValue, *pPixelOut); |
3530 | |
|
3531 | 0 | pPixelOut[1] = tOutZero; |
3532 | |
|
3533 | 0 | nDstOffset += nDstPixelStride; |
3534 | 0 | } |
3535 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned char, short>(unsigned char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned char, int>(unsigned char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned char, cpl::Float16>(unsigned char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned char, float>(unsigned char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned char, double>(unsigned char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<signed char, short>(signed char const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<signed char, int>(signed char const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<signed char, cpl::Float16>(signed char const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<signed char, float>(signed char const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<signed char, double>(signed char const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned short, short>(unsigned short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned short, int>(unsigned short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned short, cpl::Float16>(unsigned short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned short, float>(unsigned short const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned short, double>(unsigned short const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<short, short>(short const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<short, int>(short const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<short, cpl::Float16>(short const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<short, float>(short const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<short, double>(short const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned int, short>(unsigned int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned int, int>(unsigned int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned int, cpl::Float16>(unsigned int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned int, float>(unsigned int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned int, double>(unsigned int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<int, short>(int const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<int, int>(int const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<int, cpl::Float16>(int const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<int, float>(int const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<int, double>(int const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned long, short>(unsigned long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned long, int>(unsigned long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned long, cpl::Float16>(unsigned long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned long, float>(unsigned long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<unsigned long, double>(unsigned long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<long, short>(long const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<long, int>(long const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<long, cpl::Float16>(long const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<long, float>(long const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<long, double>(long const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<cpl::Float16, short>(cpl::Float16 const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<cpl::Float16, int>(cpl::Float16 const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<cpl::Float16, cpl::Float16>(cpl::Float16 const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<cpl::Float16, float>(cpl::Float16 const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<cpl::Float16, double>(cpl::Float16 const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<float, short>(float const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<float, int>(float const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<float, cpl::Float16>(float const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<float, float>(float const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<float, double>(float const*, int, double*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<double, short>(double const*, int, short*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<double, int>(double const*, int, int*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<double, cpl::Float16>(double const*, int, cpl::Float16*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<double, float>(double const*, int, float*, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsComplexOutT<double, double>(double const*, int, double*, int, long long) |
3536 | | |
3537 | | /************************************************************************/ |
3538 | | /* GDALCopyWordsFromT() */ |
3539 | | /************************************************************************/ |
3540 | | /** |
3541 | | * Template driver function. Given the input type T, call the appropriate |
3542 | | * GDALCopyWordsT function template for the desired output type. You should |
3543 | | * never call this function directly (call GDALCopyWords instead). |
3544 | | * |
3545 | | * @param pSrcData source data buffer |
3546 | | * @param nSrcPixelStride pixel stride in input buffer, in pixel words |
3547 | | * @param bInComplex input is complex |
3548 | | * @param pDstData destination data buffer |
3549 | | * @param eDstType destination data type |
3550 | | * @param nDstPixelStride pixel stride in output buffer, in pixel words |
3551 | | * @param nWordCount number of pixel words to be copied |
3552 | | */ |
3553 | | template <class T> |
3554 | | inline void GDALCopyWordsFromT(const T *const CPL_RESTRICT pSrcData, |
3555 | | int nSrcPixelStride, bool bInComplex, |
3556 | | void *CPL_RESTRICT pDstData, |
3557 | | GDALDataType eDstType, int nDstPixelStride, |
3558 | | GPtrDiff_t nWordCount) |
3559 | 0 | { |
3560 | 0 | switch (eDstType) |
3561 | 0 | { |
3562 | 0 | case GDT_UInt8: |
3563 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3564 | 0 | static_cast<unsigned char *>(pDstData), |
3565 | 0 | nDstPixelStride, nWordCount); |
3566 | 0 | break; |
3567 | 0 | case GDT_Int8: |
3568 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3569 | 0 | static_cast<signed char *>(pDstData), |
3570 | 0 | nDstPixelStride, nWordCount); |
3571 | 0 | break; |
3572 | 0 | case GDT_UInt16: |
3573 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3574 | 0 | static_cast<unsigned short *>(pDstData), |
3575 | 0 | nDstPixelStride, nWordCount); |
3576 | 0 | break; |
3577 | 0 | case GDT_Int16: |
3578 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3579 | 0 | static_cast<short *>(pDstData), nDstPixelStride, |
3580 | 0 | nWordCount); |
3581 | 0 | break; |
3582 | 0 | case GDT_UInt32: |
3583 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3584 | 0 | static_cast<unsigned int *>(pDstData), |
3585 | 0 | nDstPixelStride, nWordCount); |
3586 | 0 | break; |
3587 | 0 | case GDT_Int32: |
3588 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3589 | 0 | static_cast<int *>(pDstData), nDstPixelStride, |
3590 | 0 | nWordCount); |
3591 | 0 | break; |
3592 | 0 | case GDT_UInt64: |
3593 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3594 | 0 | static_cast<std::uint64_t *>(pDstData), |
3595 | 0 | nDstPixelStride, nWordCount); |
3596 | 0 | break; |
3597 | 0 | case GDT_Int64: |
3598 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3599 | 0 | static_cast<std::int64_t *>(pDstData), |
3600 | 0 | nDstPixelStride, nWordCount); |
3601 | 0 | break; |
3602 | 0 | case GDT_Float16: |
3603 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3604 | 0 | static_cast<GFloat16 *>(pDstData), nDstPixelStride, |
3605 | 0 | nWordCount); |
3606 | 0 | break; |
3607 | 0 | case GDT_Float32: |
3608 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3609 | 0 | static_cast<float *>(pDstData), nDstPixelStride, |
3610 | 0 | nWordCount); |
3611 | 0 | break; |
3612 | 0 | case GDT_Float64: |
3613 | 0 | GDALCopyWordsT(pSrcData, nSrcPixelStride, |
3614 | 0 | static_cast<double *>(pDstData), nDstPixelStride, |
3615 | 0 | nWordCount); |
3616 | 0 | break; |
3617 | 0 | case GDT_CInt16: |
3618 | 0 | if (bInComplex) |
3619 | 0 | { |
3620 | 0 | GDALCopyWordsComplexT(pSrcData, nSrcPixelStride, |
3621 | 0 | static_cast<short *>(pDstData), |
3622 | 0 | nDstPixelStride, nWordCount); |
3623 | 0 | } |
3624 | 0 | else // input is not complex, so we need to promote to a complex |
3625 | | // buffer |
3626 | 0 | { |
3627 | 0 | GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride, |
3628 | 0 | static_cast<short *>(pDstData), |
3629 | 0 | nDstPixelStride, nWordCount); |
3630 | 0 | } |
3631 | 0 | break; |
3632 | 0 | case GDT_CInt32: |
3633 | 0 | if (bInComplex) |
3634 | 0 | { |
3635 | 0 | GDALCopyWordsComplexT(pSrcData, nSrcPixelStride, |
3636 | 0 | static_cast<int *>(pDstData), |
3637 | 0 | nDstPixelStride, nWordCount); |
3638 | 0 | } |
3639 | 0 | else // input is not complex, so we need to promote to a complex |
3640 | | // buffer |
3641 | 0 | { |
3642 | 0 | GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride, |
3643 | 0 | static_cast<int *>(pDstData), |
3644 | 0 | nDstPixelStride, nWordCount); |
3645 | 0 | } |
3646 | 0 | break; |
3647 | 0 | case GDT_CFloat16: |
3648 | 0 | if (bInComplex) |
3649 | 0 | { |
3650 | 0 | GDALCopyWordsComplexT(pSrcData, nSrcPixelStride, |
3651 | 0 | static_cast<GFloat16 *>(pDstData), |
3652 | 0 | nDstPixelStride, nWordCount); |
3653 | 0 | } |
3654 | 0 | else // input is not complex, so we need to promote to a complex |
3655 | | // buffer |
3656 | 0 | { |
3657 | 0 | GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride, |
3658 | 0 | static_cast<GFloat16 *>(pDstData), |
3659 | 0 | nDstPixelStride, nWordCount); |
3660 | 0 | } |
3661 | 0 | break; |
3662 | 0 | case GDT_CFloat32: |
3663 | 0 | if (bInComplex) |
3664 | 0 | { |
3665 | 0 | GDALCopyWordsComplexT(pSrcData, nSrcPixelStride, |
3666 | 0 | static_cast<float *>(pDstData), |
3667 | 0 | nDstPixelStride, nWordCount); |
3668 | 0 | } |
3669 | 0 | else // input is not complex, so we need to promote to a complex |
3670 | | // buffer |
3671 | 0 | { |
3672 | 0 | GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride, |
3673 | 0 | static_cast<float *>(pDstData), |
3674 | 0 | nDstPixelStride, nWordCount); |
3675 | 0 | } |
3676 | 0 | break; |
3677 | 0 | case GDT_CFloat64: |
3678 | 0 | if (bInComplex) |
3679 | 0 | { |
3680 | 0 | GDALCopyWordsComplexT(pSrcData, nSrcPixelStride, |
3681 | 0 | static_cast<double *>(pDstData), |
3682 | 0 | nDstPixelStride, nWordCount); |
3683 | 0 | } |
3684 | 0 | else // input is not complex, so we need to promote to a complex |
3685 | | // buffer |
3686 | 0 | { |
3687 | 0 | GDALCopyWordsComplexOutT(pSrcData, nSrcPixelStride, |
3688 | 0 | static_cast<double *>(pDstData), |
3689 | 0 | nDstPixelStride, nWordCount); |
3690 | 0 | } |
3691 | 0 | break; |
3692 | 0 | case GDT_Unknown: |
3693 | 0 | case GDT_TypeCount: |
3694 | 0 | CPLAssert(false); |
3695 | 0 | } |
3696 | 0 | } Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<unsigned char>(unsigned char const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<signed char>(signed char const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<unsigned short>(unsigned short const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<short>(short const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<unsigned int>(unsigned int const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<int>(int const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<unsigned long>(unsigned long const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<long>(long const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<cpl::Float16>(cpl::Float16 const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<float>(float const*, int, bool, void*, GDALDataType, int, long long) Unexecuted instantiation: rasterio.cpp:void (anonymous namespace)::GDALCopyWordsFromT<double>(double const*, int, bool, void*, GDALDataType, int, long long) |
3697 | | |
3698 | | } // end anonymous namespace |
3699 | | |
3700 | | /************************************************************************/ |
3701 | | /* GDALReplicateWord() */ |
3702 | | /************************************************************************/ |
3703 | | |
3704 | | template <class T> |
3705 | | inline void GDALReplicateWordT(void *pDstData, int nDstPixelStride, |
3706 | | GPtrDiff_t nWordCount) |
3707 | 0 | { |
3708 | 0 | const T valSet = *static_cast<const T *>(pDstData); |
3709 | 0 | if (nDstPixelStride == static_cast<int>(sizeof(T))) |
3710 | 0 | { |
3711 | 0 | T *pDstPtr = static_cast<T *>(pDstData) + 1; |
3712 | 0 | while (nWordCount >= 4) |
3713 | 0 | { |
3714 | 0 | nWordCount -= 4; |
3715 | 0 | pDstPtr[0] = valSet; |
3716 | 0 | pDstPtr[1] = valSet; |
3717 | 0 | pDstPtr[2] = valSet; |
3718 | 0 | pDstPtr[3] = valSet; |
3719 | 0 | pDstPtr += 4; |
3720 | 0 | } |
3721 | 0 | while (nWordCount > 0) |
3722 | 0 | { |
3723 | 0 | --nWordCount; |
3724 | 0 | *pDstPtr = valSet; |
3725 | 0 | pDstPtr++; |
3726 | 0 | } |
3727 | 0 | } |
3728 | 0 | else |
3729 | 0 | { |
3730 | 0 | GByte *pabyDstPtr = static_cast<GByte *>(pDstData) + nDstPixelStride; |
3731 | 0 | while (nWordCount > 0) |
3732 | 0 | { |
3733 | 0 | --nWordCount; |
3734 | 0 | *reinterpret_cast<T *>(pabyDstPtr) = valSet; |
3735 | 0 | pabyDstPtr += nDstPixelStride; |
3736 | 0 | } |
3737 | 0 | } |
3738 | 0 | } Unexecuted instantiation: void GDALReplicateWordT<unsigned short>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<short>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<unsigned int>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<int>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<unsigned long>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<long>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<cpl::Float16>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<float>(void*, int, long long) Unexecuted instantiation: void GDALReplicateWordT<double>(void*, int, long long) |
3739 | | |
3740 | | static void GDALReplicateWord(const void *CPL_RESTRICT pSrcData, |
3741 | | GDALDataType eSrcType, |
3742 | | void *CPL_RESTRICT pDstData, |
3743 | | GDALDataType eDstType, int nDstPixelStride, |
3744 | | GPtrDiff_t nWordCount) |
3745 | 0 | { |
3746 | | /* ----------------------------------------------------------------------- |
3747 | | */ |
3748 | | /* Special case when the source data is always the same value */ |
3749 | | /* (for VRTSourcedRasterBand::IRasterIO and |
3750 | | * VRTDerivedRasterBand::IRasterIO*/ |
3751 | | /* for example) */ |
3752 | | /* ----------------------------------------------------------------------- |
3753 | | */ |
3754 | | // Let the general translation case do the necessary conversions |
3755 | | // on the first destination element. |
3756 | 0 | GDALCopyWords64(pSrcData, eSrcType, 0, pDstData, eDstType, 0, 1); |
3757 | | |
3758 | | // Now copy the first element to the nWordCount - 1 following destination |
3759 | | // elements. |
3760 | 0 | nWordCount--; |
3761 | 0 | GByte *pabyDstWord = reinterpret_cast<GByte *>(pDstData) + nDstPixelStride; |
3762 | |
|
3763 | 0 | switch (eDstType) |
3764 | 0 | { |
3765 | 0 | case GDT_UInt8: |
3766 | 0 | case GDT_Int8: |
3767 | 0 | { |
3768 | 0 | if (nDstPixelStride == 1) |
3769 | 0 | { |
3770 | 0 | if (nWordCount > 0) |
3771 | 0 | memset(pabyDstWord, |
3772 | 0 | *reinterpret_cast<const GByte *>(pDstData), |
3773 | 0 | nWordCount); |
3774 | 0 | } |
3775 | 0 | else |
3776 | 0 | { |
3777 | 0 | GByte valSet = *reinterpret_cast<const GByte *>(pDstData); |
3778 | 0 | while (nWordCount > 0) |
3779 | 0 | { |
3780 | 0 | --nWordCount; |
3781 | 0 | *pabyDstWord = valSet; |
3782 | 0 | pabyDstWord += nDstPixelStride; |
3783 | 0 | } |
3784 | 0 | } |
3785 | 0 | break; |
3786 | 0 | } |
3787 | | |
3788 | 0 | #define CASE_DUPLICATE_SIMPLE(enum_type, c_type) \ |
3789 | 0 | case enum_type: \ |
3790 | 0 | { \ |
3791 | 0 | GDALReplicateWordT<c_type>(pDstData, nDstPixelStride, nWordCount); \ |
3792 | 0 | break; \ |
3793 | 0 | } |
3794 | | |
3795 | 0 | CASE_DUPLICATE_SIMPLE(GDT_UInt16, GUInt16) |
3796 | 0 | CASE_DUPLICATE_SIMPLE(GDT_Int16, GInt16) |
3797 | 0 | CASE_DUPLICATE_SIMPLE(GDT_UInt32, GUInt32) |
3798 | 0 | CASE_DUPLICATE_SIMPLE(GDT_Int32, GInt32) |
3799 | 0 | CASE_DUPLICATE_SIMPLE(GDT_UInt64, std::uint64_t) |
3800 | 0 | CASE_DUPLICATE_SIMPLE(GDT_Int64, std::int64_t) |
3801 | 0 | CASE_DUPLICATE_SIMPLE(GDT_Float16, GFloat16) |
3802 | 0 | CASE_DUPLICATE_SIMPLE(GDT_Float32, float) |
3803 | 0 | CASE_DUPLICATE_SIMPLE(GDT_Float64, double) |
3804 | | |
3805 | 0 | #define CASE_DUPLICATE_COMPLEX(enum_type, c_type) \ |
3806 | 0 | case enum_type: \ |
3807 | 0 | { \ |
3808 | 0 | c_type valSet1 = reinterpret_cast<const c_type *>(pDstData)[0]; \ |
3809 | 0 | c_type valSet2 = reinterpret_cast<const c_type *>(pDstData)[1]; \ |
3810 | 0 | while (nWordCount > 0) \ |
3811 | 0 | { \ |
3812 | 0 | --nWordCount; \ |
3813 | 0 | reinterpret_cast<c_type *>(pabyDstWord)[0] = valSet1; \ |
3814 | 0 | reinterpret_cast<c_type *>(pabyDstWord)[1] = valSet2; \ |
3815 | 0 | pabyDstWord += nDstPixelStride; \ |
3816 | 0 | } \ |
3817 | 0 | break; \ |
3818 | 0 | } |
3819 | | |
3820 | 0 | CASE_DUPLICATE_COMPLEX(GDT_CInt16, GInt16) |
3821 | 0 | CASE_DUPLICATE_COMPLEX(GDT_CInt32, GInt32) |
3822 | 0 | CASE_DUPLICATE_COMPLEX(GDT_CFloat16, GFloat16) |
3823 | 0 | CASE_DUPLICATE_COMPLEX(GDT_CFloat32, float) |
3824 | 0 | CASE_DUPLICATE_COMPLEX(GDT_CFloat64, double) |
3825 | | |
3826 | 0 | case GDT_Unknown: |
3827 | 0 | case GDT_TypeCount: |
3828 | 0 | CPLAssert(false); |
3829 | 0 | } |
3830 | 0 | } |
3831 | | |
3832 | | /************************************************************************/ |
3833 | | /* GDALUnrolledCopy() */ |
3834 | | /************************************************************************/ |
3835 | | |
3836 | | template <class T, int srcStride, int dstStride> |
3837 | | #if defined(__GNUC__) && defined(__AVX2__) |
3838 | | __attribute__((optimize("tree-vectorize"))) |
3839 | | #endif |
3840 | | static inline void GDALUnrolledCopyGeneric(T *CPL_RESTRICT pDest, |
3841 | | const T *CPL_RESTRICT pSrc, |
3842 | | GPtrDiff_t nIters) |
3843 | 0 | { |
3844 | 0 | #if !(defined(__GNUC__) && defined(__AVX2__)) |
3845 | 0 | if (nIters >= 16) |
3846 | 0 | { |
3847 | 0 | for (GPtrDiff_t i = nIters / 16; i != 0; i--) |
3848 | 0 | { |
3849 | 0 | pDest[0 * dstStride] = pSrc[0 * srcStride]; |
3850 | 0 | pDest[1 * dstStride] = pSrc[1 * srcStride]; |
3851 | 0 | pDest[2 * dstStride] = pSrc[2 * srcStride]; |
3852 | 0 | pDest[3 * dstStride] = pSrc[3 * srcStride]; |
3853 | 0 | pDest[4 * dstStride] = pSrc[4 * srcStride]; |
3854 | 0 | pDest[5 * dstStride] = pSrc[5 * srcStride]; |
3855 | 0 | pDest[6 * dstStride] = pSrc[6 * srcStride]; |
3856 | 0 | pDest[7 * dstStride] = pSrc[7 * srcStride]; |
3857 | 0 | pDest[8 * dstStride] = pSrc[8 * srcStride]; |
3858 | 0 | pDest[9 * dstStride] = pSrc[9 * srcStride]; |
3859 | 0 | pDest[10 * dstStride] = pSrc[10 * srcStride]; |
3860 | 0 | pDest[11 * dstStride] = pSrc[11 * srcStride]; |
3861 | 0 | pDest[12 * dstStride] = pSrc[12 * srcStride]; |
3862 | 0 | pDest[13 * dstStride] = pSrc[13 * srcStride]; |
3863 | 0 | pDest[14 * dstStride] = pSrc[14 * srcStride]; |
3864 | 0 | pDest[15 * dstStride] = pSrc[15 * srcStride]; |
3865 | 0 | pDest += 16 * dstStride; |
3866 | 0 | pSrc += 16 * srcStride; |
3867 | 0 | } |
3868 | 0 | nIters = nIters % 16; |
3869 | 0 | } |
3870 | | #else |
3871 | | #pragma GCC unroll 4 |
3872 | | #endif |
3873 | 0 | for (GPtrDiff_t i = 0; i < nIters; i++) |
3874 | 0 | { |
3875 | 0 | pDest[i * dstStride] = *pSrc; |
3876 | 0 | pSrc += srcStride; |
3877 | 0 | } |
3878 | 0 | } Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<unsigned char, 1, 2>(unsigned char*, unsigned char const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<unsigned char, 1, 3>(unsigned char*, unsigned char const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<unsigned char, 1, 4>(unsigned char*, unsigned char const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<short, 2, 1>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<short, 3, 1>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<short, 4, 1>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<short, 1, 2>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<short, 1, 3>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopyGeneric<short, 1, 4>(short*, short const*, long long) |
3879 | | |
3880 | | template <class T, int srcStride, int dstStride> |
3881 | | static inline void GDALUnrolledCopy(T *CPL_RESTRICT pDest, |
3882 | | const T *CPL_RESTRICT pSrc, |
3883 | | GPtrDiff_t nIters) |
3884 | 0 | { |
3885 | 0 | GDALUnrolledCopyGeneric<T, srcStride, dstStride>(pDest, pSrc, nIters); |
3886 | 0 | } Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<unsigned char, 1, 2>(unsigned char*, unsigned char const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<unsigned char, 1, 3>(unsigned char*, unsigned char const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<unsigned char, 1, 4>(unsigned char*, unsigned char const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<short, 2, 1>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<short, 3, 1>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<short, 4, 1>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<short, 1, 2>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<short, 1, 3>(short*, short const*, long long) Unexecuted instantiation: rasterio.cpp:void GDALUnrolledCopy<short, 1, 4>(short*, short const*, long long) |
3887 | | |
3888 | | #if defined(__AVX2__) && defined(HAVE_SSSE3_AT_COMPILE_TIME) && \ |
3889 | | (defined(__x86_64) || defined(_M_X64) || defined(USE_NEON_OPTIMIZATIONS)) |
3890 | | |
3891 | | template <> |
3892 | | void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest, |
3893 | | const GByte *CPL_RESTRICT pSrc, |
3894 | | GPtrDiff_t nIters) |
3895 | | { |
3896 | | if (nIters > 16) |
3897 | | { |
3898 | | // The SSSE3 variant is slightly faster than what the gcc autovectorizer |
3899 | | // generates |
3900 | | GDALUnrolledCopy_GByte_3_1_SSSE3(pDest, pSrc, nIters); |
3901 | | } |
3902 | | else |
3903 | | { |
3904 | | for (GPtrDiff_t i = 0; i < nIters; i++) |
3905 | | { |
3906 | | pDest[i] = *pSrc; |
3907 | | pSrc += 3; |
3908 | | } |
3909 | | } |
3910 | | } |
3911 | | |
3912 | | #elif defined(HAVE_SSE2) && !(defined(__GNUC__) && defined(__AVX2__)) |
3913 | | |
3914 | | template <> |
3915 | | void GDALUnrolledCopy<GByte, 2, 1>(GByte *CPL_RESTRICT pDest, |
3916 | | const GByte *CPL_RESTRICT pSrc, |
3917 | | GPtrDiff_t nIters) |
3918 | 0 | { |
3919 | 0 | decltype(nIters) i = 0; |
3920 | 0 | if (nIters > 16) |
3921 | 0 | { |
3922 | 0 | const __m128i xmm_mask = _mm_set1_epi16(0xff); |
3923 | | // If we were sure that there would always be 1 trailing byte, we could |
3924 | | // check against nIters - 15 |
3925 | 0 | for (; i < nIters - 16; i += 16) |
3926 | 0 | { |
3927 | 0 | __m128i xmm0 = |
3928 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0)); |
3929 | 0 | __m128i xmm1 = |
3930 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16)); |
3931 | | // Set higher 8bit of each int16 packed word to 0 |
3932 | 0 | xmm0 = _mm_and_si128(xmm0, xmm_mask); |
3933 | 0 | xmm1 = _mm_and_si128(xmm1, xmm_mask); |
3934 | | // Pack int16 to uint8 and merge back both vector |
3935 | 0 | xmm0 = _mm_packus_epi16(xmm0, xmm1); |
3936 | | |
3937 | | // Store result |
3938 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm0); |
3939 | |
|
3940 | 0 | pSrc += 2 * 16; |
3941 | 0 | } |
3942 | 0 | } |
3943 | 0 | for (; i < nIters; i++) |
3944 | 0 | { |
3945 | 0 | pDest[i] = *pSrc; |
3946 | 0 | pSrc += 2; |
3947 | 0 | } |
3948 | 0 | } |
3949 | | |
3950 | | static void GDALUnrolledCopy_GByte_3_1_SSE2(GByte *CPL_RESTRICT pDest, |
3951 | | const GByte *CPL_RESTRICT pSrc, |
3952 | | GPtrDiff_t nIters) |
3953 | 0 | { |
3954 | 0 | decltype(nIters) i = 0; |
3955 | 0 | const __m128i xmm_mask_ori = _mm_set_epi32(0, 0, 0, 255); |
3956 | | // If we were sure that there would always be 2 trailing bytes, we could |
3957 | | // check against nIters - 15 |
3958 | 0 | for (; i < nIters - 16; i += 16) |
3959 | 0 | { |
3960 | 0 | __m128i xmm0 = |
3961 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0)); |
3962 | 0 | __m128i xmm1 = |
3963 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16)); |
3964 | 0 | __m128i xmm2 = |
3965 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 32)); |
3966 | |
|
3967 | 0 | auto xmm_mask0 = xmm_mask_ori; |
3968 | 0 | auto xmm_mask1 = _mm_slli_si128(xmm_mask_ori, 6); |
3969 | 0 | auto xmm_mask2 = _mm_slli_si128(xmm_mask_ori, 11); |
3970 | |
|
3971 | 0 | auto xmm = _mm_and_si128(xmm0, xmm_mask0); |
3972 | 0 | auto xmm_res1 = _mm_and_si128(_mm_slli_si128(xmm1, 4), xmm_mask1); |
3973 | |
|
3974 | 0 | xmm_mask0 = _mm_slli_si128(xmm_mask0, 1); |
3975 | 0 | xmm_mask1 = _mm_slli_si128(xmm_mask1, 1); |
3976 | 0 | xmm0 = _mm_srli_si128(xmm0, 2); |
3977 | 0 | xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0)); |
3978 | 0 | xmm_res1 = _mm_or_si128( |
3979 | 0 | xmm_res1, _mm_and_si128(_mm_slli_si128(xmm1, 2), xmm_mask1)); |
3980 | |
|
3981 | 0 | xmm_mask0 = _mm_slli_si128(xmm_mask0, 1); |
3982 | 0 | xmm_mask1 = _mm_slli_si128(xmm_mask1, 1); |
3983 | 0 | xmm0 = _mm_srli_si128(xmm0, 2); |
3984 | 0 | xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0)); |
3985 | 0 | xmm_res1 = _mm_or_si128(xmm_res1, _mm_and_si128(xmm1, xmm_mask1)); |
3986 | |
|
3987 | 0 | xmm_mask0 = _mm_slli_si128(xmm_mask0, 1); |
3988 | 0 | xmm_mask1 = _mm_slli_si128(xmm_mask1, 1); |
3989 | 0 | xmm0 = _mm_srli_si128(xmm0, 2); |
3990 | 0 | xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0)); |
3991 | 0 | xmm_res1 = _mm_or_si128( |
3992 | 0 | xmm_res1, _mm_and_si128(_mm_srli_si128(xmm1, 2), xmm_mask1)); |
3993 | |
|
3994 | 0 | xmm_mask0 = _mm_slli_si128(xmm_mask0, 1); |
3995 | 0 | xmm_mask1 = _mm_slli_si128(xmm_mask1, 1); |
3996 | 0 | xmm0 = _mm_srli_si128(xmm0, 2); |
3997 | 0 | xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0)); |
3998 | 0 | xmm_res1 = _mm_or_si128( |
3999 | 0 | xmm_res1, _mm_and_si128(_mm_srli_si128(xmm1, 4), xmm_mask1)); |
4000 | 0 | xmm = _mm_or_si128(xmm, xmm_res1); |
4001 | |
|
4002 | 0 | xmm_mask0 = _mm_slli_si128(xmm_mask0, 1); |
4003 | 0 | xmm0 = _mm_srli_si128(xmm0, 2); |
4004 | 0 | xmm = _mm_or_si128(xmm, _mm_and_si128(xmm0, xmm_mask0)); |
4005 | |
|
4006 | 0 | xmm = _mm_or_si128(xmm, |
4007 | 0 | _mm_and_si128(_mm_slli_si128(xmm2, 10), xmm_mask2)); |
4008 | |
|
4009 | 0 | xmm_mask2 = _mm_slli_si128(xmm_mask2, 1); |
4010 | 0 | xmm = _mm_or_si128(xmm, |
4011 | 0 | _mm_and_si128(_mm_slli_si128(xmm2, 8), xmm_mask2)); |
4012 | |
|
4013 | 0 | xmm_mask2 = _mm_slli_si128(xmm_mask2, 1); |
4014 | 0 | xmm = _mm_or_si128(xmm, |
4015 | 0 | _mm_and_si128(_mm_slli_si128(xmm2, 6), xmm_mask2)); |
4016 | |
|
4017 | 0 | xmm_mask2 = _mm_slli_si128(xmm_mask2, 1); |
4018 | 0 | xmm = _mm_or_si128(xmm, |
4019 | 0 | _mm_and_si128(_mm_slli_si128(xmm2, 4), xmm_mask2)); |
4020 | |
|
4021 | 0 | xmm_mask2 = _mm_slli_si128(xmm_mask2, 1); |
4022 | 0 | xmm = _mm_or_si128(xmm, |
4023 | 0 | _mm_and_si128(_mm_slli_si128(xmm2, 2), xmm_mask2)); |
4024 | |
|
4025 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm); |
4026 | |
|
4027 | 0 | pSrc += 3 * 16; |
4028 | 0 | } |
4029 | 0 | for (; i < nIters; i++) |
4030 | 0 | { |
4031 | 0 | pDest[i] = *pSrc; |
4032 | 0 | pSrc += 3; |
4033 | 0 | } |
4034 | 0 | } |
4035 | | |
4036 | | #ifdef HAVE_SSSE3_AT_COMPILE_TIME |
4037 | | |
4038 | | template <> |
4039 | | void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest, |
4040 | | const GByte *CPL_RESTRICT pSrc, |
4041 | | GPtrDiff_t nIters) |
4042 | 0 | { |
4043 | 0 | if (nIters > 16) |
4044 | 0 | { |
4045 | 0 | if (CPLHaveRuntimeSSSE3()) |
4046 | 0 | { |
4047 | 0 | GDALUnrolledCopy_GByte_3_1_SSSE3(pDest, pSrc, nIters); |
4048 | 0 | } |
4049 | 0 | else |
4050 | 0 | { |
4051 | 0 | GDALUnrolledCopy_GByte_3_1_SSE2(pDest, pSrc, nIters); |
4052 | 0 | } |
4053 | 0 | } |
4054 | 0 | else |
4055 | 0 | { |
4056 | 0 | for (GPtrDiff_t i = 0; i < nIters; i++) |
4057 | 0 | { |
4058 | 0 | pDest[i] = *pSrc; |
4059 | 0 | pSrc += 3; |
4060 | 0 | } |
4061 | 0 | } |
4062 | 0 | } |
4063 | | |
4064 | | #else |
4065 | | |
4066 | | template <> |
4067 | | void GDALUnrolledCopy<GByte, 3, 1>(GByte *CPL_RESTRICT pDest, |
4068 | | const GByte *CPL_RESTRICT pSrc, |
4069 | | GPtrDiff_t nIters) |
4070 | | { |
4071 | | GDALUnrolledCopy_GByte_3_1_SSE2(pDest, pSrc, nIters); |
4072 | | } |
4073 | | #endif |
4074 | | |
4075 | | template <> |
4076 | | void GDALUnrolledCopy<GByte, 4, 1>(GByte *CPL_RESTRICT pDest, |
4077 | | const GByte *CPL_RESTRICT pSrc, |
4078 | | GPtrDiff_t nIters) |
4079 | 0 | { |
4080 | 0 | decltype(nIters) i = 0; |
4081 | 0 | if (nIters > 16) |
4082 | 0 | { |
4083 | 0 | const __m128i xmm_mask = _mm_set1_epi32(0xff); |
4084 | | // If we were sure that there would always be 3 trailing bytes, we could |
4085 | | // check against nIters - 15 |
4086 | 0 | for (; i < nIters - 16; i += 16) |
4087 | 0 | { |
4088 | 0 | __m128i xmm0 = |
4089 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 0)); |
4090 | 0 | __m128i xmm1 = |
4091 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 16)); |
4092 | 0 | __m128i xmm2 = |
4093 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 32)); |
4094 | 0 | __m128i xmm3 = |
4095 | 0 | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + 48)); |
4096 | | // Set higher 24bit of each int32 packed word to 0 |
4097 | 0 | xmm0 = _mm_and_si128(xmm0, xmm_mask); |
4098 | 0 | xmm1 = _mm_and_si128(xmm1, xmm_mask); |
4099 | 0 | xmm2 = _mm_and_si128(xmm2, xmm_mask); |
4100 | 0 | xmm3 = _mm_and_si128(xmm3, xmm_mask); |
4101 | | // Pack int32 to int16 |
4102 | 0 | xmm0 = _mm_packs_epi32(xmm0, xmm1); |
4103 | 0 | xmm2 = _mm_packs_epi32(xmm2, xmm3); |
4104 | | // Pack int16 to uint8 |
4105 | 0 | xmm0 = _mm_packus_epi16(xmm0, xmm2); |
4106 | | |
4107 | | // Store result |
4108 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDest + i), xmm0); |
4109 | |
|
4110 | 0 | pSrc += 4 * 16; |
4111 | 0 | } |
4112 | 0 | } |
4113 | 0 | for (; i < nIters; i++) |
4114 | 0 | { |
4115 | 0 | pDest[i] = *pSrc; |
4116 | 0 | pSrc += 4; |
4117 | 0 | } |
4118 | 0 | } |
4119 | | #endif // HAVE_SSE2 |
4120 | | |
4121 | | /************************************************************************/ |
4122 | | /* GDALFastCopy() */ |
4123 | | /************************************************************************/ |
4124 | | |
4125 | | template <class T> |
4126 | | static inline void GDALFastCopy(T *CPL_RESTRICT pDest, int nDestStride, |
4127 | | const T *CPL_RESTRICT pSrc, int nSrcStride, |
4128 | | GPtrDiff_t nIters) |
4129 | 0 | { |
4130 | 0 | constexpr int sizeofT = static_cast<int>(sizeof(T)); |
4131 | 0 | if (nIters == 1) |
4132 | 0 | { |
4133 | 0 | *pDest = *pSrc; |
4134 | 0 | } |
4135 | 0 | else if (nDestStride == sizeofT) |
4136 | 0 | { |
4137 | 0 | if (nSrcStride == sizeofT) |
4138 | 0 | { |
4139 | 0 | memcpy(pDest, pSrc, nIters * sizeof(T)); |
4140 | 0 | } |
4141 | 0 | else if (nSrcStride == 2 * sizeofT) |
4142 | 0 | { |
4143 | 0 | GDALUnrolledCopy<T, 2, 1>(pDest, pSrc, nIters); |
4144 | 0 | } |
4145 | 0 | else if (nSrcStride == 3 * sizeofT) |
4146 | 0 | { |
4147 | 0 | GDALUnrolledCopy<T, 3, 1>(pDest, pSrc, nIters); |
4148 | 0 | } |
4149 | 0 | else if (nSrcStride == 4 * sizeofT) |
4150 | 0 | { |
4151 | 0 | GDALUnrolledCopy<T, 4, 1>(pDest, pSrc, nIters); |
4152 | 0 | } |
4153 | 0 | else |
4154 | 0 | { |
4155 | 0 | while (nIters-- > 0) |
4156 | 0 | { |
4157 | 0 | *pDest = *pSrc; |
4158 | 0 | pSrc += nSrcStride / sizeofT; |
4159 | 0 | pDest++; |
4160 | 0 | } |
4161 | 0 | } |
4162 | 0 | } |
4163 | 0 | else if (nSrcStride == sizeofT) |
4164 | 0 | { |
4165 | 0 | if (nDestStride == 2 * sizeofT) |
4166 | 0 | { |
4167 | 0 | GDALUnrolledCopy<T, 1, 2>(pDest, pSrc, nIters); |
4168 | 0 | } |
4169 | 0 | else if (nDestStride == 3 * sizeofT) |
4170 | 0 | { |
4171 | 0 | GDALUnrolledCopy<T, 1, 3>(pDest, pSrc, nIters); |
4172 | 0 | } |
4173 | 0 | else if (nDestStride == 4 * sizeofT) |
4174 | 0 | { |
4175 | 0 | GDALUnrolledCopy<T, 1, 4>(pDest, pSrc, nIters); |
4176 | 0 | } |
4177 | 0 | else |
4178 | 0 | { |
4179 | 0 | while (nIters-- > 0) |
4180 | 0 | { |
4181 | 0 | *pDest = *pSrc; |
4182 | 0 | pSrc++; |
4183 | 0 | pDest += nDestStride / sizeofT; |
4184 | 0 | } |
4185 | 0 | } |
4186 | 0 | } |
4187 | 0 | else |
4188 | 0 | { |
4189 | 0 | while (nIters-- > 0) |
4190 | 0 | { |
4191 | 0 | *pDest = *pSrc; |
4192 | 0 | pSrc += nSrcStride / sizeofT; |
4193 | 0 | pDest += nDestStride / sizeofT; |
4194 | 0 | } |
4195 | 0 | } |
4196 | 0 | } Unexecuted instantiation: rasterio.cpp:void GDALFastCopy<unsigned char>(unsigned char*, int, unsigned char const*, int, long long) Unexecuted instantiation: rasterio.cpp:void GDALFastCopy<short>(short*, int, short const*, int, long long) |
4197 | | |
4198 | | /************************************************************************/ |
4199 | | /* GDALFastCopyByte() */ |
4200 | | /************************************************************************/ |
4201 | | |
4202 | | static void GDALFastCopyByte(const GByte *CPL_RESTRICT pSrcData, |
4203 | | int nSrcPixelStride, GByte *CPL_RESTRICT pDstData, |
4204 | | int nDstPixelStride, GPtrDiff_t nWordCount) |
4205 | 0 | { |
4206 | 0 | GDALFastCopy(pDstData, nDstPixelStride, pSrcData, nSrcPixelStride, |
4207 | 0 | nWordCount); |
4208 | 0 | } |
4209 | | |
4210 | | /************************************************************************/ |
4211 | | /* GDALCopyWords() */ |
4212 | | /************************************************************************/ |
4213 | | |
4214 | | /** |
4215 | | * Copy pixel words from buffer to buffer. |
4216 | | * |
4217 | | * @see GDALCopyWords64() |
4218 | | */ |
4219 | | void CPL_STDCALL GDALCopyWords(const void *CPL_RESTRICT pSrcData, |
4220 | | GDALDataType eSrcType, int nSrcPixelStride, |
4221 | | void *CPL_RESTRICT pDstData, |
4222 | | GDALDataType eDstType, int nDstPixelStride, |
4223 | | int nWordCount) |
4224 | 0 | { |
4225 | 0 | GDALCopyWords64(pSrcData, eSrcType, nSrcPixelStride, pDstData, eDstType, |
4226 | 0 | nDstPixelStride, nWordCount); |
4227 | 0 | } |
4228 | | |
4229 | | /************************************************************************/ |
4230 | | /* GDALCopyWords64() */ |
4231 | | /************************************************************************/ |
4232 | | |
4233 | | /** |
4234 | | * Copy pixel words from buffer to buffer. |
4235 | | * |
4236 | | * This function is used to copy pixel word values from one memory buffer |
4237 | | * to another, with support for conversion between data types, and differing |
4238 | | * step factors. The data type conversion is done using the following |
4239 | | * rules: |
4240 | | * <ul> |
4241 | | * <li>Values assigned to a lower range integer type are clipped. For |
4242 | | * instance assigning GDT_Int16 values to a GDT_UInt8 buffer will cause values |
4243 | | * less the 0 to be set to 0, and values larger than 255 to be set to 255. |
4244 | | * </li> |
4245 | | * <li> |
4246 | | * Assignment from floating point to integer rounds to closest integer. |
4247 | | * +Infinity is mapped to the largest integer. -Infinity is mapped to the |
4248 | | * smallest integer. NaN is mapped to 0. |
4249 | | * </li> |
4250 | | * <li> |
4251 | | * Assignment from non-complex to complex will result in the imaginary part |
4252 | | * being set to zero on output. |
4253 | | * </li> |
4254 | | * <li> Assignment from complex to |
4255 | | * non-complex will result in the complex portion being lost and the real |
4256 | | * component being preserved (<i>not magnitude!</i>). |
4257 | | * </li> |
4258 | | * </ul> |
4259 | | * |
4260 | | * No assumptions are made about the source or destination words occurring |
4261 | | * on word boundaries. It is assumed that all values are in native machine |
4262 | | * byte order. |
4263 | | * |
4264 | | * @param pSrcData Pointer to source data to be converted. |
4265 | | * @param eSrcType the source data type (see GDALDataType enum) |
4266 | | * @param nSrcPixelStride Source pixel stride (i.e. distance between 2 words), |
4267 | | * in bytes |
4268 | | * @param pDstData Pointer to buffer where destination data should go |
4269 | | * @param eDstType the destination data type (see GDALDataType enum) |
4270 | | * @param nDstPixelStride Destination pixel stride (i.e. distance between 2 |
4271 | | * words), in bytes |
4272 | | * @param nWordCount number of words to be copied |
4273 | | * |
4274 | | * @note |
4275 | | * When adding a new data type to GDAL, you must do the following to |
4276 | | * support it properly within the GDALCopyWords function: |
4277 | | * 1. Add the data type to the switch on eSrcType in GDALCopyWords. |
4278 | | * This should invoke the appropriate GDALCopyWordsFromT wrapper. |
4279 | | * 2. Add the data type to the switch on eDstType in GDALCopyWordsFromT. |
4280 | | * This should call the appropriate GDALCopyWordsT template. |
4281 | | * 3. If appropriate, overload the appropriate CopyWord template in the |
4282 | | * above namespace. This will ensure that any conversion issues are |
4283 | | * handled (cases like the float -> int32 case, where the min/max) |
4284 | | * values are subject to roundoff error. |
4285 | | */ |
4286 | | |
4287 | | void CPL_STDCALL GDALCopyWords64(const void *CPL_RESTRICT pSrcData, |
4288 | | GDALDataType eSrcType, int nSrcPixelStride, |
4289 | | void *CPL_RESTRICT pDstData, |
4290 | | GDALDataType eDstType, int nDstPixelStride, |
4291 | | GPtrDiff_t nWordCount) |
4292 | | |
4293 | 0 | { |
4294 | | // On platforms where alignment matters, be careful |
4295 | 0 | const int nSrcDataTypeSize = GDALGetDataTypeSizeBytes(eSrcType); |
4296 | 0 | const int nDstDataTypeSize = GDALGetDataTypeSizeBytes(eDstType); |
4297 | 0 | if (CPL_UNLIKELY(nSrcDataTypeSize == 0 || nDstDataTypeSize == 0)) |
4298 | 0 | { |
4299 | 0 | CPLError(CE_Failure, CPLE_NotSupported, |
4300 | 0 | "GDALCopyWords64(): unsupported GDT_Unknown/GDT_TypeCount " |
4301 | 0 | "argument"); |
4302 | 0 | return; |
4303 | 0 | } |
4304 | 0 | if (!(eSrcType == eDstType && nSrcPixelStride == nDstPixelStride) && |
4305 | 0 | ((reinterpret_cast<uintptr_t>(pSrcData) % nSrcDataTypeSize) != 0 || |
4306 | 0 | (reinterpret_cast<uintptr_t>(pDstData) % nDstDataTypeSize) != 0 || |
4307 | 0 | (nSrcPixelStride % nSrcDataTypeSize) != 0 || |
4308 | 0 | (nDstPixelStride % nDstDataTypeSize) != 0)) |
4309 | 0 | { |
4310 | 0 | if (eSrcType == eDstType) |
4311 | 0 | { |
4312 | 0 | for (decltype(nWordCount) i = 0; i < nWordCount; i++) |
4313 | 0 | { |
4314 | 0 | memcpy(static_cast<GByte *>(pDstData) + nDstPixelStride * i, |
4315 | 0 | static_cast<const GByte *>(pSrcData) + |
4316 | 0 | nSrcPixelStride * i, |
4317 | 0 | nDstDataTypeSize); |
4318 | 0 | } |
4319 | 0 | } |
4320 | 0 | else |
4321 | 0 | { |
4322 | 0 | const auto getAlignedPtr = [](GByte *ptr, int align) |
4323 | 0 | { |
4324 | 0 | return ptr + |
4325 | 0 | ((align - (reinterpret_cast<uintptr_t>(ptr) % align)) % |
4326 | 0 | align); |
4327 | 0 | }; |
4328 | | |
4329 | | // The largest we need is for CFloat64 (16 bytes), so 32 bytes to |
4330 | | // be sure to get correctly aligned pointer. |
4331 | 0 | constexpr size_t SIZEOF_CFLOAT64 = 2 * sizeof(double); |
4332 | 0 | GByte abySrcBuffer[2 * SIZEOF_CFLOAT64]; |
4333 | 0 | GByte abyDstBuffer[2 * SIZEOF_CFLOAT64]; |
4334 | 0 | GByte *pabySrcBuffer = |
4335 | 0 | getAlignedPtr(abySrcBuffer, nSrcDataTypeSize); |
4336 | 0 | GByte *pabyDstBuffer = |
4337 | 0 | getAlignedPtr(abyDstBuffer, nDstDataTypeSize); |
4338 | 0 | for (decltype(nWordCount) i = 0; i < nWordCount; i++) |
4339 | 0 | { |
4340 | 0 | memcpy(pabySrcBuffer, |
4341 | 0 | static_cast<const GByte *>(pSrcData) + |
4342 | 0 | nSrcPixelStride * i, |
4343 | 0 | nSrcDataTypeSize); |
4344 | 0 | GDALCopyWords64(pabySrcBuffer, eSrcType, 0, pabyDstBuffer, |
4345 | 0 | eDstType, 0, 1); |
4346 | 0 | memcpy(static_cast<GByte *>(pDstData) + nDstPixelStride * i, |
4347 | 0 | pabyDstBuffer, nDstDataTypeSize); |
4348 | 0 | } |
4349 | 0 | } |
4350 | 0 | return; |
4351 | 0 | } |
4352 | | |
4353 | | // Deal with the case where we're replicating a single word into the |
4354 | | // provided buffer |
4355 | 0 | if (nSrcPixelStride == 0 && nWordCount > 1) |
4356 | 0 | { |
4357 | 0 | GDALReplicateWord(pSrcData, eSrcType, pDstData, eDstType, |
4358 | 0 | nDstPixelStride, nWordCount); |
4359 | 0 | return; |
4360 | 0 | } |
4361 | | |
4362 | 0 | if (eSrcType == eDstType) |
4363 | 0 | { |
4364 | 0 | if (eSrcType == GDT_UInt8 || eSrcType == GDT_Int8) |
4365 | 0 | { |
4366 | 0 | GDALFastCopy(static_cast<GByte *>(pDstData), nDstPixelStride, |
4367 | 0 | static_cast<const GByte *>(pSrcData), nSrcPixelStride, |
4368 | 0 | nWordCount); |
4369 | 0 | return; |
4370 | 0 | } |
4371 | | |
4372 | 0 | if (nSrcDataTypeSize == 2 && (nSrcPixelStride % 2) == 0 && |
4373 | 0 | (nDstPixelStride % 2) == 0) |
4374 | 0 | { |
4375 | 0 | GDALFastCopy(static_cast<short *>(pDstData), nDstPixelStride, |
4376 | 0 | static_cast<const short *>(pSrcData), nSrcPixelStride, |
4377 | 0 | nWordCount); |
4378 | 0 | return; |
4379 | 0 | } |
4380 | | |
4381 | 0 | if (nWordCount == 1) |
4382 | 0 | { |
4383 | | #if defined(CSA_BUILD) || defined(__COVERITY__) |
4384 | | // Avoid false positives... |
4385 | | memcpy(pDstData, pSrcData, nSrcDataTypeSize); |
4386 | | #else |
4387 | 0 | if (nSrcDataTypeSize == 2) |
4388 | 0 | memcpy(pDstData, pSrcData, 2); |
4389 | 0 | else if (nSrcDataTypeSize == 4) |
4390 | 0 | memcpy(pDstData, pSrcData, 4); |
4391 | 0 | else if (nSrcDataTypeSize == 8) |
4392 | 0 | memcpy(pDstData, pSrcData, 8); |
4393 | 0 | else /* if( eSrcType == GDT_CFloat64 ) */ |
4394 | 0 | memcpy(pDstData, pSrcData, 16); |
4395 | 0 | #endif |
4396 | 0 | return; |
4397 | 0 | } |
4398 | | |
4399 | | // Let memcpy() handle the case where we're copying a packed buffer |
4400 | | // of pixels. |
4401 | 0 | if (nSrcPixelStride == nDstPixelStride) |
4402 | 0 | { |
4403 | 0 | if (nSrcPixelStride == nSrcDataTypeSize) |
4404 | 0 | { |
4405 | 0 | memcpy(pDstData, pSrcData, nWordCount * nSrcDataTypeSize); |
4406 | 0 | return; |
4407 | 0 | } |
4408 | 0 | } |
4409 | 0 | } |
4410 | | |
4411 | | // Handle the more general case -- deals with conversion of data types |
4412 | | // directly. |
4413 | 0 | switch (eSrcType) |
4414 | 0 | { |
4415 | 0 | case GDT_UInt8: |
4416 | 0 | GDALCopyWordsFromT<unsigned char>( |
4417 | 0 | static_cast<const unsigned char *>(pSrcData), nSrcPixelStride, |
4418 | 0 | false, pDstData, eDstType, nDstPixelStride, nWordCount); |
4419 | 0 | break; |
4420 | 0 | case GDT_Int8: |
4421 | 0 | GDALCopyWordsFromT<signed char>( |
4422 | 0 | static_cast<const signed char *>(pSrcData), nSrcPixelStride, |
4423 | 0 | false, pDstData, eDstType, nDstPixelStride, nWordCount); |
4424 | 0 | break; |
4425 | 0 | case GDT_UInt16: |
4426 | 0 | GDALCopyWordsFromT<unsigned short>( |
4427 | 0 | static_cast<const unsigned short *>(pSrcData), nSrcPixelStride, |
4428 | 0 | false, pDstData, eDstType, nDstPixelStride, nWordCount); |
4429 | 0 | break; |
4430 | 0 | case GDT_Int16: |
4431 | 0 | GDALCopyWordsFromT<short>(static_cast<const short *>(pSrcData), |
4432 | 0 | nSrcPixelStride, false, pDstData, |
4433 | 0 | eDstType, nDstPixelStride, nWordCount); |
4434 | 0 | break; |
4435 | 0 | case GDT_UInt32: |
4436 | 0 | GDALCopyWordsFromT<unsigned int>( |
4437 | 0 | static_cast<const unsigned int *>(pSrcData), nSrcPixelStride, |
4438 | 0 | false, pDstData, eDstType, nDstPixelStride, nWordCount); |
4439 | 0 | break; |
4440 | 0 | case GDT_Int32: |
4441 | 0 | GDALCopyWordsFromT<int>(static_cast<const int *>(pSrcData), |
4442 | 0 | nSrcPixelStride, false, pDstData, eDstType, |
4443 | 0 | nDstPixelStride, nWordCount); |
4444 | 0 | break; |
4445 | 0 | case GDT_UInt64: |
4446 | 0 | GDALCopyWordsFromT<std::uint64_t>( |
4447 | 0 | static_cast<const std::uint64_t *>(pSrcData), nSrcPixelStride, |
4448 | 0 | false, pDstData, eDstType, nDstPixelStride, nWordCount); |
4449 | 0 | break; |
4450 | 0 | case GDT_Int64: |
4451 | 0 | GDALCopyWordsFromT<std::int64_t>( |
4452 | 0 | static_cast<const std::int64_t *>(pSrcData), nSrcPixelStride, |
4453 | 0 | false, pDstData, eDstType, nDstPixelStride, nWordCount); |
4454 | 0 | break; |
4455 | 0 | case GDT_Float16: |
4456 | 0 | GDALCopyWordsFromT<GFloat16>( |
4457 | 0 | static_cast<const GFloat16 *>(pSrcData), nSrcPixelStride, false, |
4458 | 0 | pDstData, eDstType, nDstPixelStride, nWordCount); |
4459 | 0 | break; |
4460 | 0 | case GDT_Float32: |
4461 | 0 | GDALCopyWordsFromT<float>(static_cast<const float *>(pSrcData), |
4462 | 0 | nSrcPixelStride, false, pDstData, |
4463 | 0 | eDstType, nDstPixelStride, nWordCount); |
4464 | 0 | break; |
4465 | 0 | case GDT_Float64: |
4466 | 0 | GDALCopyWordsFromT<double>(static_cast<const double *>(pSrcData), |
4467 | 0 | nSrcPixelStride, false, pDstData, |
4468 | 0 | eDstType, nDstPixelStride, nWordCount); |
4469 | 0 | break; |
4470 | 0 | case GDT_CInt16: |
4471 | 0 | GDALCopyWordsFromT<short>(static_cast<const short *>(pSrcData), |
4472 | 0 | nSrcPixelStride, true, pDstData, eDstType, |
4473 | 0 | nDstPixelStride, nWordCount); |
4474 | 0 | break; |
4475 | 0 | case GDT_CInt32: |
4476 | 0 | GDALCopyWordsFromT<int>(static_cast<const int *>(pSrcData), |
4477 | 0 | nSrcPixelStride, true, pDstData, eDstType, |
4478 | 0 | nDstPixelStride, nWordCount); |
4479 | 0 | break; |
4480 | 0 | case GDT_CFloat16: |
4481 | 0 | GDALCopyWordsFromT<GFloat16>( |
4482 | 0 | static_cast<const GFloat16 *>(pSrcData), nSrcPixelStride, true, |
4483 | 0 | pDstData, eDstType, nDstPixelStride, nWordCount); |
4484 | 0 | break; |
4485 | 0 | case GDT_CFloat32: |
4486 | 0 | GDALCopyWordsFromT<float>(static_cast<const float *>(pSrcData), |
4487 | 0 | nSrcPixelStride, true, pDstData, eDstType, |
4488 | 0 | nDstPixelStride, nWordCount); |
4489 | 0 | break; |
4490 | 0 | case GDT_CFloat64: |
4491 | 0 | GDALCopyWordsFromT<double>(static_cast<const double *>(pSrcData), |
4492 | 0 | nSrcPixelStride, true, pDstData, |
4493 | 0 | eDstType, nDstPixelStride, nWordCount); |
4494 | 0 | break; |
4495 | 0 | case GDT_Unknown: |
4496 | 0 | case GDT_TypeCount: |
4497 | 0 | CPLAssert(false); |
4498 | 0 | } |
4499 | 0 | } |
4500 | | |
4501 | | /************************************************************************/ |
4502 | | /* GDALCopyBits() */ |
4503 | | /************************************************************************/ |
4504 | | |
4505 | | /** |
4506 | | * Bitwise word copying. |
4507 | | * |
4508 | | * A function for moving sets of partial bytes around. Loosely |
4509 | | * speaking this is a bitwise analog to GDALCopyWords(). |
4510 | | * |
4511 | | * It copies nStepCount "words" where each word is nBitCount bits long. |
4512 | | * The nSrcStep and nDstStep are the number of bits from the start of one |
4513 | | * word to the next (same as nBitCount if they are packed). The nSrcOffset |
4514 | | * and nDstOffset are the offset into the source and destination buffers |
4515 | | * to start at, also measured in bits. |
4516 | | * |
4517 | | * All bit offsets are assumed to start from the high order bit in a byte |
4518 | | * (i.e. most significant bit first). Currently this function is not very |
4519 | | * optimized, but it may be improved for some common cases in the future |
4520 | | * as needed. |
4521 | | * |
4522 | | * @param pabySrcData the source data buffer. |
4523 | | * @param nSrcOffset the offset (in bits) in pabySrcData to the start of the |
4524 | | * first word to copy. |
4525 | | * @param nSrcStep the offset in bits from the start one source word to the |
4526 | | * start of the next. |
4527 | | * @param pabyDstData the destination data buffer. |
4528 | | * @param nDstOffset the offset (in bits) in pabyDstData to the start of the |
4529 | | * first word to copy over. |
4530 | | * @param nDstStep the offset in bits from the start one word to the |
4531 | | * start of the next. |
4532 | | * @param nBitCount the number of bits in a word to be copied. |
4533 | | * @param nStepCount the number of words to copy. |
4534 | | */ |
4535 | | |
4536 | | void GDALCopyBits(const GByte *pabySrcData, int nSrcOffset, int nSrcStep, |
4537 | | GByte *pabyDstData, int nDstOffset, int nDstStep, |
4538 | | int nBitCount, int nStepCount) |
4539 | | |
4540 | 0 | { |
4541 | 0 | VALIDATE_POINTER0(pabySrcData, "GDALCopyBits"); |
4542 | | |
4543 | 0 | for (int iStep = 0; iStep < nStepCount; iStep++) |
4544 | 0 | { |
4545 | 0 | for (int iBit = 0; iBit < nBitCount; iBit++) |
4546 | 0 | { |
4547 | 0 | if (pabySrcData[nSrcOffset >> 3] & (0x80 >> (nSrcOffset & 7))) |
4548 | 0 | pabyDstData[nDstOffset >> 3] |= (0x80 >> (nDstOffset & 7)); |
4549 | 0 | else |
4550 | 0 | pabyDstData[nDstOffset >> 3] &= ~(0x80 >> (nDstOffset & 7)); |
4551 | |
|
4552 | 0 | nSrcOffset++; |
4553 | 0 | nDstOffset++; |
4554 | 0 | } |
4555 | |
|
4556 | 0 | nSrcOffset += (nSrcStep - nBitCount); |
4557 | 0 | nDstOffset += (nDstStep - nBitCount); |
4558 | 0 | } |
4559 | 0 | } |
4560 | | |
4561 | | /************************************************************************/ |
4562 | | /* GDALBandGetBestOverviewLevel() */ |
4563 | | /************************************************************************/ |
4564 | | |
4565 | | int GDALBandGetBestOverviewLevel(GDALRasterBand *poBand, |
4566 | | double dfTargetDownsamplingRatio, |
4567 | | double dfOversamplingThreshold) |
4568 | 0 | { |
4569 | 0 | int iBestOvr = -1; |
4570 | 0 | double dfBestRatio = 0; |
4571 | 0 | const int nOvCount = poBand->GetOverviewCount(); |
4572 | 0 | constexpr double EPSILON = 1e-1; |
4573 | 0 | for (int iOvr = -1; iOvr < nOvCount; iOvr++) |
4574 | 0 | { |
4575 | 0 | double dfOvrRatio = 1.0; |
4576 | 0 | GDALRasterBand *poOvrBand = nullptr; |
4577 | 0 | if (iOvr >= 0) |
4578 | 0 | { |
4579 | 0 | poOvrBand = poBand->GetOverview(iOvr); |
4580 | 0 | if (poOvrBand == nullptr || |
4581 | 0 | poOvrBand->GetXSize() > poBand->GetXSize() || |
4582 | 0 | poOvrBand->GetYSize() > poBand->GetYSize()) |
4583 | 0 | { |
4584 | 0 | continue; |
4585 | 0 | } |
4586 | 0 | dfOvrRatio = std::min(static_cast<double>(poBand->GetXSize()) / |
4587 | 0 | poOvrBand->GetXSize(), |
4588 | 0 | static_cast<double>(poBand->GetYSize()) / |
4589 | 0 | poOvrBand->GetYSize()); |
4590 | 0 | } |
4591 | | |
4592 | | // Is it nearly the requested factor and better (lower) than |
4593 | | // the current best factor? |
4594 | | // Use an epsilon because of numerical instability. |
4595 | 0 | if (dfOvrRatio >= |
4596 | 0 | dfTargetDownsamplingRatio * dfOversamplingThreshold + EPSILON || |
4597 | 0 | dfOvrRatio <= dfBestRatio) |
4598 | 0 | { |
4599 | 0 | continue; |
4600 | 0 | } |
4601 | | |
4602 | 0 | if (poOvrBand) |
4603 | 0 | { |
4604 | | // Ignore AVERAGE_BIT2GRAYSCALE overviews. |
4605 | 0 | const char *pszResampling = |
4606 | 0 | poOvrBand->GetMetadataItem("RESAMPLING"); |
4607 | 0 | if (pszResampling != nullptr && |
4608 | 0 | STARTS_WITH_CI(pszResampling, "AVERAGE_BIT2")) |
4609 | 0 | { |
4610 | 0 | continue; |
4611 | 0 | } |
4612 | 0 | } |
4613 | | |
4614 | 0 | iBestOvr = iOvr; |
4615 | 0 | dfBestRatio = dfOvrRatio; |
4616 | 0 | if (std::abs(dfTargetDownsamplingRatio - dfOvrRatio) < EPSILON) |
4617 | 0 | { |
4618 | 0 | break; |
4619 | 0 | } |
4620 | 0 | } |
4621 | 0 | return iBestOvr; |
4622 | 0 | } |
4623 | | |
4624 | | /************************************************************************/ |
4625 | | /* GDALGetBestOverviewLevel() */ |
4626 | | /* */ |
4627 | | /* Returns the best overview level to satisfy the query or -1 if none */ |
4628 | | /* Also updates nXOff, nYOff, nXSize, nYSize and psExtraArg when */ |
4629 | | /* returning a valid overview level */ |
4630 | | /************************************************************************/ |
4631 | | |
4632 | | int GDALBandGetBestOverviewLevel(GDALRasterBand *poBand, int &nXOff, int &nYOff, |
4633 | | int &nXSize, int &nYSize, int nBufXSize, |
4634 | | int nBufYSize) |
4635 | 0 | { |
4636 | 0 | return GDALBandGetBestOverviewLevel2(poBand, nXOff, nYOff, nXSize, nYSize, |
4637 | 0 | nBufXSize, nBufYSize, nullptr); |
4638 | 0 | } |
4639 | | |
4640 | | int GDALBandGetBestOverviewLevel2(GDALRasterBand *poBand, int &nXOff, |
4641 | | int &nYOff, int &nXSize, int &nYSize, |
4642 | | int nBufXSize, int nBufYSize, |
4643 | | GDALRasterIOExtraArg *psExtraArg) |
4644 | 0 | { |
4645 | 0 | if (psExtraArg != nullptr && psExtraArg->nVersion > 1 && |
4646 | 0 | psExtraArg->bUseOnlyThisScale) |
4647 | 0 | return -1; |
4648 | | /* -------------------------------------------------------------------- */ |
4649 | | /* Compute the desired downsampling factor. It is */ |
4650 | | /* based on the least reduced axis, and represents the number */ |
4651 | | /* of source pixels to one destination pixel. */ |
4652 | | /* -------------------------------------------------------------------- */ |
4653 | 0 | const double dfDesiredDownsamplingFactor = |
4654 | 0 | ((nXSize / static_cast<double>(nBufXSize)) < |
4655 | 0 | (nYSize / static_cast<double>(nBufYSize)) || |
4656 | 0 | nBufYSize == 1) |
4657 | 0 | ? nXSize / static_cast<double>(nBufXSize) |
4658 | 0 | : nYSize / static_cast<double>(nBufYSize); |
4659 | | |
4660 | | /* -------------------------------------------------------------------- */ |
4661 | | /* Find the overview level that largest downsampling factor (most */ |
4662 | | /* downsampled) that is still less than (or only a little more) */ |
4663 | | /* downsampled than the request. */ |
4664 | | /* -------------------------------------------------------------------- */ |
4665 | |
|
4666 | 0 | const char *pszOversampligThreshold = |
4667 | 0 | CPLGetConfigOption("GDAL_OVERVIEW_OVERSAMPLING_THRESHOLD", nullptr); |
4668 | | |
4669 | | // Cf https://github.com/OSGeo/gdal/pull/9040#issuecomment-1898524693 |
4670 | 0 | const double dfOversamplingThreshold = |
4671 | 0 | pszOversampligThreshold ? CPLAtof(pszOversampligThreshold) |
4672 | 0 | : psExtraArg && psExtraArg->eResampleAlg != GRIORA_NearestNeighbour |
4673 | 0 | ? 1.0 |
4674 | 0 | : 1.2; |
4675 | 0 | const int iBestOvrLevel = GDALBandGetBestOverviewLevel( |
4676 | 0 | poBand, dfDesiredDownsamplingFactor, dfOversamplingThreshold); |
4677 | | |
4678 | | /* -------------------------------------------------------------------- */ |
4679 | | /* If we didn't find an overview that helps us, just return */ |
4680 | | /* indicating failure and the full resolution image will be used. */ |
4681 | | /* -------------------------------------------------------------------- */ |
4682 | 0 | if (iBestOvrLevel < 0) |
4683 | 0 | return -1; |
4684 | 0 | const GDALRasterBand *poBestOverview = poBand->GetOverview(iBestOvrLevel); |
4685 | | |
4686 | | /* -------------------------------------------------------------------- */ |
4687 | | /* Recompute the source window in terms of the selected */ |
4688 | | /* overview. */ |
4689 | | /* -------------------------------------------------------------------- */ |
4690 | 0 | const double dfXFactor = |
4691 | 0 | poBand->GetXSize() / static_cast<double>(poBestOverview->GetXSize()); |
4692 | 0 | const double dfYFactor = |
4693 | 0 | poBand->GetYSize() / static_cast<double>(poBestOverview->GetYSize()); |
4694 | 0 | CPLDebug("GDAL", "Selecting overview %d x %d", poBestOverview->GetXSize(), |
4695 | 0 | poBestOverview->GetYSize()); |
4696 | |
|
4697 | 0 | const int nOXOff = std::min(poBestOverview->GetXSize() - 1, |
4698 | 0 | static_cast<int>(nXOff / dfXFactor + 0.5)); |
4699 | 0 | const int nOYOff = std::min(poBestOverview->GetYSize() - 1, |
4700 | 0 | static_cast<int>(nYOff / dfYFactor + 0.5)); |
4701 | 0 | int nOXSize = std::max(1, static_cast<int>(nXSize / dfXFactor + 0.5)); |
4702 | 0 | int nOYSize = std::max(1, static_cast<int>(nYSize / dfYFactor + 0.5)); |
4703 | 0 | if (nOXOff + nOXSize > poBestOverview->GetXSize()) |
4704 | 0 | nOXSize = poBestOverview->GetXSize() - nOXOff; |
4705 | 0 | if (nOYOff + nOYSize > poBestOverview->GetYSize()) |
4706 | 0 | nOYSize = poBestOverview->GetYSize() - nOYOff; |
4707 | |
|
4708 | 0 | if (psExtraArg) |
4709 | 0 | { |
4710 | 0 | if (psExtraArg->bFloatingPointWindowValidity) |
4711 | 0 | { |
4712 | 0 | psExtraArg->dfXOff /= dfXFactor; |
4713 | 0 | psExtraArg->dfXSize /= dfXFactor; |
4714 | 0 | psExtraArg->dfYOff /= dfYFactor; |
4715 | 0 | psExtraArg->dfYSize /= dfYFactor; |
4716 | 0 | } |
4717 | 0 | else if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour) |
4718 | 0 | { |
4719 | 0 | psExtraArg->bFloatingPointWindowValidity = true; |
4720 | 0 | psExtraArg->dfXOff = nXOff / dfXFactor; |
4721 | 0 | psExtraArg->dfXSize = nXSize / dfXFactor; |
4722 | 0 | psExtraArg->dfYOff = nYOff / dfYFactor; |
4723 | 0 | psExtraArg->dfYSize = nYSize / dfYFactor; |
4724 | 0 | } |
4725 | 0 | } |
4726 | |
|
4727 | 0 | nXOff = nOXOff; |
4728 | 0 | nYOff = nOYOff; |
4729 | 0 | nXSize = nOXSize; |
4730 | 0 | nYSize = nOYSize; |
4731 | |
|
4732 | 0 | return iBestOvrLevel; |
4733 | 0 | } |
4734 | | |
4735 | | /************************************************************************/ |
4736 | | /* OverviewRasterIO() */ |
4737 | | /* */ |
4738 | | /* Special work function to utilize available overviews to */ |
4739 | | /* more efficiently satisfy downsampled requests. It will */ |
4740 | | /* return CE_Failure if there are no appropriate overviews */ |
4741 | | /* available but it doesn't emit any error messages. */ |
4742 | | /************************************************************************/ |
4743 | | |
4744 | | //! @cond Doxygen_Suppress |
4745 | | CPLErr GDALRasterBand::OverviewRasterIO( |
4746 | | GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, |
4747 | | void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, |
4748 | | GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg) |
4749 | | |
4750 | 0 | { |
4751 | 0 | GDALRasterIOExtraArg sExtraArg; |
4752 | 0 | GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg); |
4753 | |
|
4754 | 0 | const int nOverview = GDALBandGetBestOverviewLevel2( |
4755 | 0 | this, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, &sExtraArg); |
4756 | 0 | if (nOverview < 0) |
4757 | 0 | return CE_Failure; |
4758 | | |
4759 | | /* -------------------------------------------------------------------- */ |
4760 | | /* Recast the call in terms of the new raster layer. */ |
4761 | | /* -------------------------------------------------------------------- */ |
4762 | 0 | GDALRasterBand *poOverviewBand = GetOverview(nOverview); |
4763 | 0 | if (poOverviewBand == nullptr) |
4764 | 0 | return CE_Failure; |
4765 | | |
4766 | 0 | return poOverviewBand->RasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, |
4767 | 0 | pData, nBufXSize, nBufYSize, eBufType, |
4768 | 0 | nPixelSpace, nLineSpace, &sExtraArg); |
4769 | 0 | } |
4770 | | |
4771 | | /************************************************************************/ |
4772 | | /* TryOverviewRasterIO() */ |
4773 | | /************************************************************************/ |
4774 | | |
4775 | | CPLErr GDALRasterBand::TryOverviewRasterIO( |
4776 | | GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, |
4777 | | void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, |
4778 | | GSpacing nPixelSpace, GSpacing nLineSpace, GDALRasterIOExtraArg *psExtraArg, |
4779 | | int *pbTried) |
4780 | 0 | { |
4781 | 0 | int nXOffMod = nXOff; |
4782 | 0 | int nYOffMod = nYOff; |
4783 | 0 | int nXSizeMod = nXSize; |
4784 | 0 | int nYSizeMod = nYSize; |
4785 | 0 | GDALRasterIOExtraArg sExtraArg; |
4786 | |
|
4787 | 0 | GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg); |
4788 | |
|
4789 | 0 | int iOvrLevel = GDALBandGetBestOverviewLevel2( |
4790 | 0 | this, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, nBufXSize, nBufYSize, |
4791 | 0 | &sExtraArg); |
4792 | |
|
4793 | 0 | if (iOvrLevel >= 0) |
4794 | 0 | { |
4795 | 0 | GDALRasterBand *poOverviewBand = GetOverview(iOvrLevel); |
4796 | 0 | if (poOverviewBand) |
4797 | 0 | { |
4798 | 0 | *pbTried = TRUE; |
4799 | 0 | return poOverviewBand->RasterIO( |
4800 | 0 | eRWFlag, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, pData, |
4801 | 0 | nBufXSize, nBufYSize, eBufType, nPixelSpace, nLineSpace, |
4802 | 0 | &sExtraArg); |
4803 | 0 | } |
4804 | 0 | } |
4805 | | |
4806 | 0 | *pbTried = FALSE; |
4807 | 0 | return CE_None; |
4808 | 0 | } |
4809 | | |
4810 | | /************************************************************************/ |
4811 | | /* TryOverviewRasterIO() */ |
4812 | | /************************************************************************/ |
4813 | | |
4814 | | CPLErr GDALDataset::TryOverviewRasterIO( |
4815 | | GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, |
4816 | | void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, |
4817 | | int nBandCount, const int *panBandMap, GSpacing nPixelSpace, |
4818 | | GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg, |
4819 | | int *pbTried) |
4820 | 0 | { |
4821 | 0 | int nXOffMod = nXOff; |
4822 | 0 | int nYOffMod = nYOff; |
4823 | 0 | int nXSizeMod = nXSize; |
4824 | 0 | int nYSizeMod = nYSize; |
4825 | 0 | GDALRasterIOExtraArg sExtraArg; |
4826 | 0 | GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg); |
4827 | |
|
4828 | 0 | int iOvrLevel = GDALBandGetBestOverviewLevel2( |
4829 | 0 | papoBands[0], nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, nBufXSize, |
4830 | 0 | nBufYSize, &sExtraArg); |
4831 | |
|
4832 | 0 | if (iOvrLevel >= 0 && papoBands[0]->GetOverview(iOvrLevel) != nullptr && |
4833 | 0 | papoBands[0]->GetOverview(iOvrLevel)->GetDataset() != nullptr) |
4834 | 0 | { |
4835 | 0 | *pbTried = TRUE; |
4836 | 0 | return papoBands[0]->GetOverview(iOvrLevel)->GetDataset()->RasterIO( |
4837 | 0 | eRWFlag, nXOffMod, nYOffMod, nXSizeMod, nYSizeMod, pData, nBufXSize, |
4838 | 0 | nBufYSize, eBufType, nBandCount, panBandMap, nPixelSpace, |
4839 | 0 | nLineSpace, nBandSpace, &sExtraArg); |
4840 | 0 | } |
4841 | 0 | else |
4842 | 0 | { |
4843 | 0 | *pbTried = FALSE; |
4844 | 0 | return CE_None; |
4845 | 0 | } |
4846 | 0 | } |
4847 | | |
4848 | | /************************************************************************/ |
4849 | | /* GetBestOverviewLevel() */ |
4850 | | /* */ |
4851 | | /* Returns the best overview level to satisfy the query or -1 if none */ |
4852 | | /* Also updates nXOff, nYOff, nXSize, nYSize when returning a valid */ |
4853 | | /* overview level */ |
4854 | | /************************************************************************/ |
4855 | | |
4856 | | static int GDALDatasetGetBestOverviewLevel(GDALDataset *poDS, int &nXOff, |
4857 | | int &nYOff, int &nXSize, int &nYSize, |
4858 | | int nBufXSize, int nBufYSize, |
4859 | | int nBandCount, |
4860 | | const int *panBandMap, |
4861 | | GDALRasterIOExtraArg *psExtraArg) |
4862 | 0 | { |
4863 | 0 | int nOverviewCount = 0; |
4864 | 0 | GDALRasterBand *poFirstBand = nullptr; |
4865 | | |
4866 | | /* -------------------------------------------------------------------- */ |
4867 | | /* Check that all bands have the same number of overviews and */ |
4868 | | /* that they have all the same size and block dimensions */ |
4869 | | /* -------------------------------------------------------------------- */ |
4870 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
4871 | 0 | { |
4872 | 0 | GDALRasterBand *poBand = poDS->GetRasterBand(panBandMap[iBand]); |
4873 | 0 | if (poBand == nullptr) |
4874 | 0 | return -1; |
4875 | 0 | if (iBand == 0) |
4876 | 0 | { |
4877 | 0 | poFirstBand = poBand; |
4878 | 0 | nOverviewCount = poBand->GetOverviewCount(); |
4879 | 0 | } |
4880 | 0 | else if (nOverviewCount != poBand->GetOverviewCount()) |
4881 | 0 | { |
4882 | 0 | CPLDebug("GDAL", "GDALDataset::GetBestOverviewLevel() ... " |
4883 | 0 | "mismatched overview count, use std method."); |
4884 | 0 | return -1; |
4885 | 0 | } |
4886 | 0 | else |
4887 | 0 | { |
4888 | 0 | for (int iOverview = 0; iOverview < nOverviewCount; iOverview++) |
4889 | 0 | { |
4890 | 0 | GDALRasterBand *poOvrBand = poBand->GetOverview(iOverview); |
4891 | 0 | GDALRasterBand *poOvrFirstBand = |
4892 | 0 | poFirstBand->GetOverview(iOverview); |
4893 | 0 | if (poOvrBand == nullptr || poOvrFirstBand == nullptr) |
4894 | 0 | continue; |
4895 | | |
4896 | 0 | if (poOvrFirstBand->GetXSize() != poOvrBand->GetXSize() || |
4897 | 0 | poOvrFirstBand->GetYSize() != poOvrBand->GetYSize()) |
4898 | 0 | { |
4899 | 0 | CPLDebug("GDAL", |
4900 | 0 | "GDALDataset::GetBestOverviewLevel() ... " |
4901 | 0 | "mismatched overview sizes, use std method."); |
4902 | 0 | return -1; |
4903 | 0 | } |
4904 | 0 | int nBlockXSizeFirst = 0; |
4905 | 0 | int nBlockYSizeFirst = 0; |
4906 | 0 | poOvrFirstBand->GetBlockSize(&nBlockXSizeFirst, |
4907 | 0 | &nBlockYSizeFirst); |
4908 | |
|
4909 | 0 | int nBlockXSizeCurrent = 0; |
4910 | 0 | int nBlockYSizeCurrent = 0; |
4911 | 0 | poOvrBand->GetBlockSize(&nBlockXSizeCurrent, |
4912 | 0 | &nBlockYSizeCurrent); |
4913 | |
|
4914 | 0 | if (nBlockXSizeFirst != nBlockXSizeCurrent || |
4915 | 0 | nBlockYSizeFirst != nBlockYSizeCurrent) |
4916 | 0 | { |
4917 | 0 | CPLDebug("GDAL", "GDALDataset::GetBestOverviewLevel() ... " |
4918 | 0 | "mismatched block sizes, use std method."); |
4919 | 0 | return -1; |
4920 | 0 | } |
4921 | 0 | } |
4922 | 0 | } |
4923 | 0 | } |
4924 | 0 | if (poFirstBand == nullptr) |
4925 | 0 | return -1; |
4926 | | |
4927 | 0 | return GDALBandGetBestOverviewLevel2(poFirstBand, nXOff, nYOff, nXSize, |
4928 | 0 | nYSize, nBufXSize, nBufYSize, |
4929 | 0 | psExtraArg); |
4930 | 0 | } |
4931 | | |
4932 | | /************************************************************************/ |
4933 | | /* BlockBasedRasterIO() */ |
4934 | | /* */ |
4935 | | /* This convenience function implements a dataset level */ |
4936 | | /* RasterIO() interface based on calling down to fetch blocks, */ |
4937 | | /* much like the GDALRasterBand::IRasterIO(), but it handles */ |
4938 | | /* all bands at once, so that a format driver that handles a */ |
4939 | | /* request for different bands of the same block efficiently */ |
4940 | | /* (i.e. without re-reading interleaved data) will efficiently. */ |
4941 | | /* */ |
4942 | | /* This method is intended to be called by an overridden */ |
4943 | | /* IRasterIO() method in the driver specific GDALDataset */ |
4944 | | /* derived class. */ |
4945 | | /* */ |
4946 | | /* Default internal implementation of RasterIO() ... utilizes */ |
4947 | | /* the Block access methods to satisfy the request. This would */ |
4948 | | /* normally only be overridden by formats with overviews. */ |
4949 | | /* */ |
4950 | | /* To keep things relatively simple, this method does not */ |
4951 | | /* currently take advantage of some special cases addressed in */ |
4952 | | /* GDALRasterBand::IRasterIO(), so it is likely best to only */ |
4953 | | /* call it when you know it will help. That is in cases where */ |
4954 | | /* data is at 1:1 to the buffer, and you know the driver is */ |
4955 | | /* implementing interleaved IO efficiently on a block by block */ |
4956 | | /* basis. Overviews will be used when possible. */ |
4957 | | /************************************************************************/ |
4958 | | |
4959 | | CPLErr GDALDataset::BlockBasedRasterIO( |
4960 | | GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize, int nYSize, |
4961 | | void *pData, int nBufXSize, int nBufYSize, GDALDataType eBufType, |
4962 | | int nBandCount, const int *panBandMap, GSpacing nPixelSpace, |
4963 | | GSpacing nLineSpace, GSpacing nBandSpace, GDALRasterIOExtraArg *psExtraArg) |
4964 | | |
4965 | 0 | { |
4966 | 0 | CPLAssert(nullptr != pData); |
4967 | | |
4968 | 0 | GByte **papabySrcBlock = nullptr; |
4969 | 0 | GDALRasterBlock *poBlock = nullptr; |
4970 | 0 | GDALRasterBlock **papoBlocks = nullptr; |
4971 | 0 | int nLBlockX = -1; |
4972 | 0 | int nLBlockY = -1; |
4973 | 0 | int iBufYOff; |
4974 | 0 | int iBufXOff; |
4975 | 0 | int nBlockXSize = 1; |
4976 | 0 | int nBlockYSize = 1; |
4977 | 0 | CPLErr eErr = CE_None; |
4978 | 0 | GDALDataType eDataType = GDT_UInt8; |
4979 | |
|
4980 | 0 | const bool bUseIntegerRequestCoords = |
4981 | 0 | (!psExtraArg->bFloatingPointWindowValidity || |
4982 | 0 | (nXOff == psExtraArg->dfXOff && nYOff == psExtraArg->dfYOff && |
4983 | 0 | nXSize == psExtraArg->dfXSize && nYSize == psExtraArg->dfYSize)); |
4984 | | |
4985 | | /* -------------------------------------------------------------------- */ |
4986 | | /* Ensure that all bands share a common block size and data type. */ |
4987 | | /* -------------------------------------------------------------------- */ |
4988 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
4989 | 0 | { |
4990 | 0 | GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]); |
4991 | |
|
4992 | 0 | if (iBand == 0) |
4993 | 0 | { |
4994 | 0 | poBand->GetBlockSize(&nBlockXSize, &nBlockYSize); |
4995 | 0 | eDataType = poBand->GetRasterDataType(); |
4996 | 0 | } |
4997 | 0 | else |
4998 | 0 | { |
4999 | 0 | int nThisBlockXSize = 0; |
5000 | 0 | int nThisBlockYSize = 0; |
5001 | 0 | poBand->GetBlockSize(&nThisBlockXSize, &nThisBlockYSize); |
5002 | 0 | if (nThisBlockXSize != nBlockXSize || |
5003 | 0 | nThisBlockYSize != nBlockYSize) |
5004 | 0 | { |
5005 | 0 | CPLDebug("GDAL", "GDALDataset::BlockBasedRasterIO() ... " |
5006 | 0 | "mismatched block sizes, use std method."); |
5007 | 0 | return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, |
5008 | 0 | pData, nBufXSize, nBufYSize, eBufType, |
5009 | 0 | nBandCount, panBandMap, nPixelSpace, |
5010 | 0 | nLineSpace, nBandSpace, psExtraArg); |
5011 | 0 | } |
5012 | | |
5013 | 0 | if (eDataType != poBand->GetRasterDataType() && |
5014 | 0 | (nXSize != nBufXSize || nYSize != nBufYSize)) |
5015 | 0 | { |
5016 | 0 | CPLDebug("GDAL", "GDALDataset::BlockBasedRasterIO() ... " |
5017 | 0 | "mismatched band data types, use std method."); |
5018 | 0 | return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, |
5019 | 0 | pData, nBufXSize, nBufYSize, eBufType, |
5020 | 0 | nBandCount, panBandMap, nPixelSpace, |
5021 | 0 | nLineSpace, nBandSpace, psExtraArg); |
5022 | 0 | } |
5023 | 0 | } |
5024 | 0 | } |
5025 | | |
5026 | | /* ==================================================================== */ |
5027 | | /* In this special case at full resolution we step through in */ |
5028 | | /* blocks, turning the request over to the per-band */ |
5029 | | /* IRasterIO(), but ensuring that all bands of one block are */ |
5030 | | /* called before proceeding to the next. */ |
5031 | | /* ==================================================================== */ |
5032 | | |
5033 | 0 | if (nXSize == nBufXSize && nYSize == nBufYSize && bUseIntegerRequestCoords) |
5034 | 0 | { |
5035 | 0 | GDALRasterIOExtraArg sDummyExtraArg; |
5036 | 0 | INIT_RASTERIO_EXTRA_ARG(sDummyExtraArg); |
5037 | |
|
5038 | 0 | int nChunkYSize = 0; |
5039 | 0 | int nChunkXSize = 0; |
5040 | |
|
5041 | 0 | for (iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff += nChunkYSize) |
5042 | 0 | { |
5043 | 0 | const int nChunkYOff = iBufYOff + nYOff; |
5044 | 0 | nChunkYSize = std::min(nBlockYSize - (nChunkYOff % nBlockYSize), |
5045 | 0 | (nYOff + nYSize) - nChunkYOff); |
5046 | |
|
5047 | 0 | for (iBufXOff = 0; iBufXOff < nBufXSize; iBufXOff += nChunkXSize) |
5048 | 0 | { |
5049 | 0 | const int nChunkXOff = iBufXOff + nXOff; |
5050 | 0 | nChunkXSize = std::min(nBlockXSize - (nChunkXOff % nBlockXSize), |
5051 | 0 | (nXOff + nXSize) - nChunkXOff); |
5052 | |
|
5053 | 0 | GByte *pabyChunkData = |
5054 | 0 | static_cast<GByte *>(pData) + iBufXOff * nPixelSpace + |
5055 | 0 | static_cast<GPtrDiff_t>(iBufYOff) * nLineSpace; |
5056 | |
|
5057 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
5058 | 0 | { |
5059 | 0 | GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]); |
5060 | |
|
5061 | 0 | eErr = poBand->IRasterIO( |
5062 | 0 | eRWFlag, nChunkXOff, nChunkYOff, nChunkXSize, |
5063 | 0 | nChunkYSize, |
5064 | 0 | pabyChunkData + |
5065 | 0 | static_cast<GPtrDiff_t>(iBand) * nBandSpace, |
5066 | 0 | nChunkXSize, nChunkYSize, eBufType, nPixelSpace, |
5067 | 0 | nLineSpace, &sDummyExtraArg); |
5068 | 0 | if (eErr != CE_None) |
5069 | 0 | return eErr; |
5070 | 0 | } |
5071 | 0 | } |
5072 | | |
5073 | 0 | if (psExtraArg->pfnProgress != nullptr && |
5074 | 0 | !psExtraArg->pfnProgress( |
5075 | 0 | 1.0 * std::min(nBufYSize, iBufYOff + nChunkYSize) / |
5076 | 0 | nBufYSize, |
5077 | 0 | "", psExtraArg->pProgressData)) |
5078 | 0 | { |
5079 | 0 | return CE_Failure; |
5080 | 0 | } |
5081 | 0 | } |
5082 | | |
5083 | 0 | return CE_None; |
5084 | 0 | } |
5085 | | |
5086 | | /* Below code is not compatible with that case. It would need a complete */ |
5087 | | /* separate code like done in GDALRasterBand::IRasterIO. */ |
5088 | 0 | if (eRWFlag == GF_Write && (nBufXSize < nXSize || nBufYSize < nYSize)) |
5089 | 0 | { |
5090 | 0 | return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, |
5091 | 0 | nBufXSize, nBufYSize, eBufType, nBandCount, |
5092 | 0 | panBandMap, nPixelSpace, nLineSpace, |
5093 | 0 | nBandSpace, psExtraArg); |
5094 | 0 | } |
5095 | | |
5096 | | /* We could have a smarter implementation, but that will do for now */ |
5097 | 0 | if (psExtraArg->eResampleAlg != GRIORA_NearestNeighbour && |
5098 | 0 | (nBufXSize != nXSize || nBufYSize != nYSize)) |
5099 | 0 | { |
5100 | 0 | return BandBasedRasterIO(eRWFlag, nXOff, nYOff, nXSize, nYSize, pData, |
5101 | 0 | nBufXSize, nBufYSize, eBufType, nBandCount, |
5102 | 0 | panBandMap, nPixelSpace, nLineSpace, |
5103 | 0 | nBandSpace, psExtraArg); |
5104 | 0 | } |
5105 | | |
5106 | | /* ==================================================================== */ |
5107 | | /* Loop reading required source blocks to satisfy output */ |
5108 | | /* request. This is the most general implementation. */ |
5109 | | /* ==================================================================== */ |
5110 | | |
5111 | 0 | const int nBandDataSize = GDALGetDataTypeSizeBytes(eDataType); |
5112 | |
|
5113 | 0 | papabySrcBlock = |
5114 | 0 | static_cast<GByte **>(CPLCalloc(sizeof(GByte *), nBandCount)); |
5115 | 0 | papoBlocks = |
5116 | 0 | static_cast<GDALRasterBlock **>(CPLCalloc(sizeof(void *), nBandCount)); |
5117 | | |
5118 | | /* -------------------------------------------------------------------- */ |
5119 | | /* Select an overview level if appropriate. */ |
5120 | | /* -------------------------------------------------------------------- */ |
5121 | |
|
5122 | 0 | GDALRasterIOExtraArg sExtraArg; |
5123 | 0 | GDALCopyRasterIOExtraArg(&sExtraArg, psExtraArg); |
5124 | 0 | const int nOverviewLevel = GDALDatasetGetBestOverviewLevel( |
5125 | 0 | this, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, nBandCount, |
5126 | 0 | panBandMap, &sExtraArg); |
5127 | 0 | if (nOverviewLevel >= 0) |
5128 | 0 | { |
5129 | 0 | GetRasterBand(panBandMap[0]) |
5130 | 0 | ->GetOverview(nOverviewLevel) |
5131 | 0 | ->GetBlockSize(&nBlockXSize, &nBlockYSize); |
5132 | 0 | } |
5133 | |
|
5134 | 0 | double dfXOff = nXOff; |
5135 | 0 | double dfYOff = nYOff; |
5136 | 0 | double dfXSize = nXSize; |
5137 | 0 | double dfYSize = nYSize; |
5138 | 0 | if (sExtraArg.bFloatingPointWindowValidity) |
5139 | 0 | { |
5140 | 0 | dfXOff = sExtraArg.dfXOff; |
5141 | 0 | dfYOff = sExtraArg.dfYOff; |
5142 | 0 | dfXSize = sExtraArg.dfXSize; |
5143 | 0 | dfYSize = sExtraArg.dfYSize; |
5144 | 0 | } |
5145 | | |
5146 | | /* -------------------------------------------------------------------- */ |
5147 | | /* Compute stepping increment. */ |
5148 | | /* -------------------------------------------------------------------- */ |
5149 | 0 | const double dfSrcXInc = dfXSize / static_cast<double>(nBufXSize); |
5150 | 0 | const double dfSrcYInc = dfYSize / static_cast<double>(nBufYSize); |
5151 | |
|
5152 | 0 | constexpr double EPS = 1e-10; |
5153 | | /* -------------------------------------------------------------------- */ |
5154 | | /* Loop over buffer computing source locations. */ |
5155 | | /* -------------------------------------------------------------------- */ |
5156 | 0 | for (iBufYOff = 0; iBufYOff < nBufYSize; iBufYOff++) |
5157 | 0 | { |
5158 | 0 | GPtrDiff_t iSrcOffset; |
5159 | | |
5160 | | // Add small epsilon to avoid some numeric precision issues. |
5161 | 0 | const double dfSrcY = (iBufYOff + 0.5) * dfSrcYInc + dfYOff + EPS; |
5162 | 0 | const int iSrcY = static_cast<int>(std::min( |
5163 | 0 | std::max(0.0, dfSrcY), static_cast<double>(nRasterYSize - 1))); |
5164 | |
|
5165 | 0 | GPtrDiff_t iBufOffset = static_cast<GPtrDiff_t>(iBufYOff) * |
5166 | 0 | static_cast<GPtrDiff_t>(nLineSpace); |
5167 | |
|
5168 | 0 | for (iBufXOff = 0; iBufXOff < nBufXSize; iBufXOff++) |
5169 | 0 | { |
5170 | 0 | const double dfSrcX = (iBufXOff + 0.5) * dfSrcXInc + dfXOff + EPS; |
5171 | 0 | const int iSrcX = static_cast<int>(std::min( |
5172 | 0 | std::max(0.0, dfSrcX), static_cast<double>(nRasterXSize - 1))); |
5173 | | |
5174 | | // FIXME: this code likely doesn't work if the dirty block gets |
5175 | | // flushed to disk before being completely written. In the meantime, |
5176 | | // bJustInitialize should probably be set to FALSE even if it is not |
5177 | | // ideal performance wise, and for lossy compression |
5178 | | |
5179 | | /* -------------------------------------------------------------------- |
5180 | | */ |
5181 | | /* Ensure we have the appropriate block loaded. */ |
5182 | | /* -------------------------------------------------------------------- |
5183 | | */ |
5184 | 0 | if (iSrcX < nLBlockX * nBlockXSize || |
5185 | 0 | iSrcX - nBlockXSize >= nLBlockX * nBlockXSize || |
5186 | 0 | iSrcY < nLBlockY * nBlockYSize || |
5187 | 0 | iSrcY - nBlockYSize >= nLBlockY * nBlockYSize) |
5188 | 0 | { |
5189 | 0 | nLBlockX = iSrcX / nBlockXSize; |
5190 | 0 | nLBlockY = iSrcY / nBlockYSize; |
5191 | |
|
5192 | 0 | const bool bJustInitialize = |
5193 | 0 | eRWFlag == GF_Write && nYOff <= nLBlockY * nBlockYSize && |
5194 | 0 | nYOff + nYSize - nBlockYSize >= nLBlockY * nBlockYSize && |
5195 | 0 | nXOff <= nLBlockX * nBlockXSize && |
5196 | 0 | nXOff + nXSize - nBlockXSize >= nLBlockX * nBlockXSize; |
5197 | | /*bool bMemZeroBuffer = FALSE; |
5198 | | if( eRWFlag == GF_Write && !bJustInitialize && |
5199 | | nXOff <= nLBlockX * nBlockXSize && |
5200 | | nYOff <= nLBlockY * nBlockYSize && |
5201 | | (nXOff + nXSize >= (nLBlockX+1) * nBlockXSize || |
5202 | | (nXOff + nXSize == GetRasterXSize() && |
5203 | | (nLBlockX+1) * nBlockXSize > GetRasterXSize())) && |
5204 | | (nYOff + nYSize >= (nLBlockY+1) * nBlockYSize || |
5205 | | (nYOff + nYSize == GetRasterYSize() && |
5206 | | (nLBlockY+1) * nBlockYSize > GetRasterYSize())) ) |
5207 | | { |
5208 | | bJustInitialize = TRUE; |
5209 | | bMemZeroBuffer = TRUE; |
5210 | | }*/ |
5211 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
5212 | 0 | { |
5213 | 0 | GDALRasterBand *poBand = GetRasterBand(panBandMap[iBand]); |
5214 | 0 | if (nOverviewLevel >= 0) |
5215 | 0 | poBand = poBand->GetOverview(nOverviewLevel); |
5216 | 0 | poBlock = poBand->GetLockedBlockRef(nLBlockX, nLBlockY, |
5217 | 0 | bJustInitialize); |
5218 | 0 | if (poBlock == nullptr) |
5219 | 0 | { |
5220 | 0 | eErr = CE_Failure; |
5221 | 0 | goto CleanupAndReturn; |
5222 | 0 | } |
5223 | | |
5224 | 0 | if (eRWFlag == GF_Write) |
5225 | 0 | poBlock->MarkDirty(); |
5226 | |
|
5227 | 0 | if (papoBlocks[iBand] != nullptr) |
5228 | 0 | papoBlocks[iBand]->DropLock(); |
5229 | |
|
5230 | 0 | papoBlocks[iBand] = poBlock; |
5231 | |
|
5232 | 0 | papabySrcBlock[iBand] = |
5233 | 0 | static_cast<GByte *>(poBlock->GetDataRef()); |
5234 | | /*if( bMemZeroBuffer ) |
5235 | | { |
5236 | | memset(papabySrcBlock[iBand], 0, |
5237 | | static_cast<GPtrDiff_t>(nBandDataSize) * nBlockXSize |
5238 | | * nBlockYSize); |
5239 | | }*/ |
5240 | 0 | } |
5241 | 0 | } |
5242 | | |
5243 | | /* -------------------------------------------------------------------- |
5244 | | */ |
5245 | | /* Copy over this pixel of data. */ |
5246 | | /* -------------------------------------------------------------------- |
5247 | | */ |
5248 | 0 | iSrcOffset = (static_cast<GPtrDiff_t>(iSrcX) - |
5249 | 0 | static_cast<GPtrDiff_t>(nLBlockX) * nBlockXSize + |
5250 | 0 | (static_cast<GPtrDiff_t>(iSrcY) - |
5251 | 0 | static_cast<GPtrDiff_t>(nLBlockY) * nBlockYSize) * |
5252 | 0 | nBlockXSize) * |
5253 | 0 | nBandDataSize; |
5254 | |
|
5255 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
5256 | 0 | { |
5257 | 0 | GByte *pabySrcBlock = papabySrcBlock[iBand]; |
5258 | 0 | GPtrDiff_t iBandBufOffset = |
5259 | 0 | iBufOffset + static_cast<GPtrDiff_t>(iBand) * |
5260 | 0 | static_cast<GPtrDiff_t>(nBandSpace); |
5261 | |
|
5262 | 0 | if (eDataType == eBufType) |
5263 | 0 | { |
5264 | 0 | if (eRWFlag == GF_Read) |
5265 | 0 | memcpy(static_cast<GByte *>(pData) + iBandBufOffset, |
5266 | 0 | pabySrcBlock + iSrcOffset, nBandDataSize); |
5267 | 0 | else |
5268 | 0 | memcpy(pabySrcBlock + iSrcOffset, |
5269 | 0 | static_cast<const GByte *>(pData) + |
5270 | 0 | iBandBufOffset, |
5271 | 0 | nBandDataSize); |
5272 | 0 | } |
5273 | 0 | else |
5274 | 0 | { |
5275 | | /* type to type conversion ... ouch, this is expensive way |
5276 | | of handling single words */ |
5277 | |
|
5278 | 0 | if (eRWFlag == GF_Read) |
5279 | 0 | GDALCopyWords64(pabySrcBlock + iSrcOffset, eDataType, 0, |
5280 | 0 | static_cast<GByte *>(pData) + |
5281 | 0 | iBandBufOffset, |
5282 | 0 | eBufType, 0, 1); |
5283 | 0 | else |
5284 | 0 | GDALCopyWords64(static_cast<const GByte *>(pData) + |
5285 | 0 | iBandBufOffset, |
5286 | 0 | eBufType, 0, pabySrcBlock + iSrcOffset, |
5287 | 0 | eDataType, 0, 1); |
5288 | 0 | } |
5289 | 0 | } |
5290 | |
|
5291 | 0 | iBufOffset += static_cast<int>(nPixelSpace); |
5292 | 0 | } |
5293 | 0 | } |
5294 | | |
5295 | | /* -------------------------------------------------------------------- */ |
5296 | | /* CleanupAndReturn. */ |
5297 | | /* -------------------------------------------------------------------- */ |
5298 | 0 | CleanupAndReturn: |
5299 | 0 | CPLFree(papabySrcBlock); |
5300 | 0 | if (papoBlocks != nullptr) |
5301 | 0 | { |
5302 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
5303 | 0 | { |
5304 | 0 | if (papoBlocks[iBand] != nullptr) |
5305 | 0 | papoBlocks[iBand]->DropLock(); |
5306 | 0 | } |
5307 | 0 | CPLFree(papoBlocks); |
5308 | 0 | } |
5309 | |
|
5310 | 0 | return eErr; |
5311 | 0 | } |
5312 | | |
5313 | | //! @endcond |
5314 | | |
5315 | | /************************************************************************/ |
5316 | | /* GDALCopyWholeRasterGetSwathSize() */ |
5317 | | /************************************************************************/ |
5318 | | |
5319 | | static void GDALCopyWholeRasterGetSwathSize(GDALRasterBand *poSrcPrototypeBand, |
5320 | | GDALRasterBand *poDstPrototypeBand, |
5321 | | int nBandCount, |
5322 | | int bDstIsCompressed, |
5323 | | int bInterleave, int *pnSwathCols, |
5324 | | int *pnSwathLines) |
5325 | 0 | { |
5326 | 0 | GDALDataType eDT = poDstPrototypeBand->GetRasterDataType(); |
5327 | 0 | int nSrcBlockXSize = 0; |
5328 | 0 | int nSrcBlockYSize = 0; |
5329 | 0 | int nBlockXSize = 0; |
5330 | 0 | int nBlockYSize = 0; |
5331 | |
|
5332 | 0 | int nXSize = poSrcPrototypeBand->GetXSize(); |
5333 | 0 | int nYSize = poSrcPrototypeBand->GetYSize(); |
5334 | |
|
5335 | 0 | poSrcPrototypeBand->GetBlockSize(&nSrcBlockXSize, &nSrcBlockYSize); |
5336 | 0 | poDstPrototypeBand->GetBlockSize(&nBlockXSize, &nBlockYSize); |
5337 | |
|
5338 | 0 | const int nMaxBlockXSize = std::max(nBlockXSize, nSrcBlockXSize); |
5339 | 0 | const int nMaxBlockYSize = std::max(nBlockYSize, nSrcBlockYSize); |
5340 | |
|
5341 | 0 | int nPixelSize = GDALGetDataTypeSizeBytes(eDT); |
5342 | 0 | if (bInterleave) |
5343 | 0 | nPixelSize *= nBandCount; |
5344 | | |
5345 | | // aim for one row of blocks. Do not settle for less. |
5346 | 0 | int nSwathCols = nXSize; |
5347 | 0 | int nSwathLines = nMaxBlockYSize; |
5348 | |
|
5349 | 0 | const char *pszSrcCompression = poSrcPrototypeBand->GetMetadataItem( |
5350 | 0 | GDALMD_COMPRESSION, GDAL_MDD_IMAGE_STRUCTURE); |
5351 | 0 | if (pszSrcCompression == nullptr) |
5352 | 0 | { |
5353 | 0 | auto poSrcDS = poSrcPrototypeBand->GetDataset(); |
5354 | 0 | if (poSrcDS) |
5355 | 0 | pszSrcCompression = poSrcDS->GetMetadataItem( |
5356 | 0 | GDALMD_COMPRESSION, GDAL_MDD_IMAGE_STRUCTURE); |
5357 | 0 | } |
5358 | | |
5359 | | /* -------------------------------------------------------------------- */ |
5360 | | /* What will our swath size be? */ |
5361 | | /* -------------------------------------------------------------------- */ |
5362 | | // When writing interleaved data in a compressed format, we want to be sure |
5363 | | // that each block will only be written once, so the swath size must not be |
5364 | | // greater than the block cache. |
5365 | 0 | const char *pszSwathSize = CPLGetConfigOption("GDAL_SWATH_SIZE", nullptr); |
5366 | 0 | int nTargetSwathSize; |
5367 | 0 | if (pszSwathSize != nullptr) |
5368 | 0 | nTargetSwathSize = static_cast<int>( |
5369 | 0 | std::min(GIntBig(INT_MAX), CPLAtoGIntBig(pszSwathSize))); |
5370 | 0 | else |
5371 | 0 | { |
5372 | | // As a default, take one 1/4 of the cache size. |
5373 | 0 | nTargetSwathSize = static_cast<int>( |
5374 | 0 | std::min(GIntBig(INT_MAX), GDALGetCacheMax64() / 4)); |
5375 | | |
5376 | | // but if the minimum idal swath buf size is less, then go for it to |
5377 | | // avoid unnecessarily abusing RAM usage. |
5378 | | // but try to use 10 MB at least. |
5379 | 0 | GIntBig nIdealSwathBufSize = |
5380 | 0 | static_cast<GIntBig>(nSwathCols) * nSwathLines * nPixelSize; |
5381 | 0 | int nMinTargetSwathSize = 10 * 1000 * 1000; |
5382 | |
|
5383 | 0 | if ((poSrcPrototypeBand->GetSuggestedBlockAccessPattern() & |
5384 | 0 | GSBAP_LARGEST_CHUNK_POSSIBLE) != 0) |
5385 | 0 | { |
5386 | 0 | nMinTargetSwathSize = nTargetSwathSize; |
5387 | 0 | } |
5388 | |
|
5389 | 0 | if (nIdealSwathBufSize < nTargetSwathSize && |
5390 | 0 | nIdealSwathBufSize < nMinTargetSwathSize) |
5391 | 0 | { |
5392 | 0 | nIdealSwathBufSize = nMinTargetSwathSize; |
5393 | 0 | } |
5394 | |
|
5395 | 0 | if (pszSrcCompression != nullptr && |
5396 | 0 | EQUAL(pszSrcCompression, "JPEG2000") && |
5397 | 0 | (!bDstIsCompressed || ((nSrcBlockXSize % nBlockXSize) == 0 && |
5398 | 0 | (nSrcBlockYSize % nBlockYSize) == 0))) |
5399 | 0 | { |
5400 | 0 | nIdealSwathBufSize = |
5401 | 0 | std::max(nIdealSwathBufSize, static_cast<GIntBig>(nSwathCols) * |
5402 | 0 | nSrcBlockYSize * nPixelSize); |
5403 | 0 | } |
5404 | 0 | if (nTargetSwathSize > nIdealSwathBufSize) |
5405 | 0 | nTargetSwathSize = static_cast<int>( |
5406 | 0 | std::min(GIntBig(INT_MAX), nIdealSwathBufSize)); |
5407 | 0 | } |
5408 | |
|
5409 | 0 | if (nTargetSwathSize < 1000000) |
5410 | 0 | nTargetSwathSize = 1000000; |
5411 | | |
5412 | | /* But let's check that */ |
5413 | 0 | if (bDstIsCompressed && bInterleave && |
5414 | 0 | nTargetSwathSize > GDALGetCacheMax64()) |
5415 | 0 | { |
5416 | 0 | CPLError(CE_Warning, CPLE_AppDefined, |
5417 | 0 | "When translating into a compressed interleave format, " |
5418 | 0 | "the block cache size (" CPL_FRMT_GIB ") " |
5419 | 0 | "should be at least the size of the swath (%d) " |
5420 | 0 | "(GDAL_SWATH_SIZE config. option)", |
5421 | 0 | GDALGetCacheMax64(), nTargetSwathSize); |
5422 | 0 | } |
5423 | |
|
5424 | 0 | #define IS_DIVIDER_OF(x, y) ((y) % (x) == 0) |
5425 | 0 | #define ROUND_TO(x, y) (((x) / (y)) * (y)) |
5426 | | |
5427 | | // if both input and output datasets are tiled, that the tile dimensions |
5428 | | // are "compatible", try to stick to a swath dimension that is a multiple |
5429 | | // of input and output block dimensions. |
5430 | 0 | if (nBlockXSize != nXSize && nSrcBlockXSize != nXSize && |
5431 | 0 | IS_DIVIDER_OF(nBlockXSize, nMaxBlockXSize) && |
5432 | 0 | IS_DIVIDER_OF(nSrcBlockXSize, nMaxBlockXSize) && |
5433 | 0 | IS_DIVIDER_OF(nBlockYSize, nMaxBlockYSize) && |
5434 | 0 | IS_DIVIDER_OF(nSrcBlockYSize, nMaxBlockYSize)) |
5435 | 0 | { |
5436 | 0 | if (static_cast<GIntBig>(nMaxBlockXSize) * nMaxBlockYSize * |
5437 | 0 | nPixelSize <= |
5438 | 0 | static_cast<GIntBig>(nTargetSwathSize)) |
5439 | 0 | { |
5440 | 0 | nSwathCols = nTargetSwathSize / (nMaxBlockYSize * nPixelSize); |
5441 | 0 | nSwathCols = ROUND_TO(nSwathCols, nMaxBlockXSize); |
5442 | 0 | if (nSwathCols == 0) |
5443 | 0 | nSwathCols = nMaxBlockXSize; |
5444 | 0 | if (nSwathCols > nXSize) |
5445 | 0 | nSwathCols = nXSize; |
5446 | 0 | nSwathLines = nMaxBlockYSize; |
5447 | |
|
5448 | 0 | if (static_cast<GIntBig>(nSwathCols) * nSwathLines * nPixelSize > |
5449 | 0 | static_cast<GIntBig>(nTargetSwathSize)) |
5450 | 0 | { |
5451 | 0 | nSwathCols = nXSize; |
5452 | 0 | nSwathLines = nBlockYSize; |
5453 | 0 | } |
5454 | 0 | } |
5455 | 0 | } |
5456 | |
|
5457 | 0 | const GIntBig nMemoryPerCol = static_cast<GIntBig>(nSwathCols) * nPixelSize; |
5458 | 0 | const GIntBig nSwathBufSize = nMemoryPerCol * nSwathLines; |
5459 | 0 | if (nSwathBufSize > static_cast<GIntBig>(nTargetSwathSize)) |
5460 | 0 | { |
5461 | 0 | nSwathLines = static_cast<int>(nTargetSwathSize / nMemoryPerCol); |
5462 | 0 | if (nSwathLines == 0) |
5463 | 0 | nSwathLines = 1; |
5464 | |
|
5465 | 0 | CPLDebug( |
5466 | 0 | "GDAL", |
5467 | 0 | "GDALCopyWholeRasterGetSwathSize(): adjusting to %d line swath " |
5468 | 0 | "since requirement (" CPL_FRMT_GIB " bytes) exceed target swath " |
5469 | 0 | "size (%d bytes) (GDAL_SWATH_SIZE config. option)", |
5470 | 0 | nSwathLines, nBlockYSize * nMemoryPerCol, nTargetSwathSize); |
5471 | 0 | } |
5472 | | // If we are processing single scans, try to handle several at once. |
5473 | | // If we are handling swaths already, only grow the swath if a row |
5474 | | // of blocks is substantially less than our target buffer size. |
5475 | 0 | else if (nSwathLines == 1 || |
5476 | 0 | nMemoryPerCol * nSwathLines < |
5477 | 0 | static_cast<GIntBig>(nTargetSwathSize) / 10) |
5478 | 0 | { |
5479 | 0 | nSwathLines = std::min( |
5480 | 0 | nYSize, |
5481 | 0 | std::max(1, static_cast<int>(nTargetSwathSize / nMemoryPerCol))); |
5482 | | |
5483 | | /* If possible try to align to source and target block height */ |
5484 | 0 | if ((nSwathLines % nMaxBlockYSize) != 0 && |
5485 | 0 | nSwathLines > nMaxBlockYSize && |
5486 | 0 | IS_DIVIDER_OF(nBlockYSize, nMaxBlockYSize) && |
5487 | 0 | IS_DIVIDER_OF(nSrcBlockYSize, nMaxBlockYSize)) |
5488 | 0 | nSwathLines = ROUND_TO(nSwathLines, nMaxBlockYSize); |
5489 | 0 | } |
5490 | |
|
5491 | 0 | if (pszSrcCompression != nullptr && EQUAL(pszSrcCompression, "JPEG2000") && |
5492 | 0 | (!bDstIsCompressed || (IS_DIVIDER_OF(nBlockXSize, nSrcBlockXSize) && |
5493 | 0 | IS_DIVIDER_OF(nBlockYSize, nSrcBlockYSize)))) |
5494 | 0 | { |
5495 | | // Typical use case: converting from Pleaiades that is 2048x2048 tiled. |
5496 | 0 | if (nSwathLines < nSrcBlockYSize) |
5497 | 0 | { |
5498 | 0 | nSwathLines = nSrcBlockYSize; |
5499 | | |
5500 | | // Number of pixels that can be read/write simultaneously. |
5501 | 0 | nSwathCols = nTargetSwathSize / (nSrcBlockXSize * nPixelSize); |
5502 | 0 | nSwathCols = ROUND_TO(nSwathCols, nSrcBlockXSize); |
5503 | 0 | if (nSwathCols == 0) |
5504 | 0 | nSwathCols = nSrcBlockXSize; |
5505 | 0 | if (nSwathCols > nXSize) |
5506 | 0 | nSwathCols = nXSize; |
5507 | |
|
5508 | 0 | CPLDebug( |
5509 | 0 | "GDAL", |
5510 | 0 | "GDALCopyWholeRasterGetSwathSize(): because of compression and " |
5511 | 0 | "too high block, " |
5512 | 0 | "use partial width at one time"); |
5513 | 0 | } |
5514 | 0 | else if ((nSwathLines % nSrcBlockYSize) != 0) |
5515 | 0 | { |
5516 | | /* Round on a multiple of nSrcBlockYSize */ |
5517 | 0 | nSwathLines = ROUND_TO(nSwathLines, nSrcBlockYSize); |
5518 | 0 | CPLDebug( |
5519 | 0 | "GDAL", |
5520 | 0 | "GDALCopyWholeRasterGetSwathSize(): because of compression, " |
5521 | 0 | "round nSwathLines to block height : %d", |
5522 | 0 | nSwathLines); |
5523 | 0 | } |
5524 | 0 | } |
5525 | 0 | else if (bDstIsCompressed) |
5526 | 0 | { |
5527 | 0 | if (nSwathLines < nBlockYSize) |
5528 | 0 | { |
5529 | 0 | nSwathLines = nBlockYSize; |
5530 | | |
5531 | | // Number of pixels that can be read/write simultaneously. |
5532 | 0 | nSwathCols = nTargetSwathSize / (nSwathLines * nPixelSize); |
5533 | 0 | nSwathCols = ROUND_TO(nSwathCols, nBlockXSize); |
5534 | 0 | if (nSwathCols == 0) |
5535 | 0 | nSwathCols = nBlockXSize; |
5536 | 0 | if (nSwathCols > nXSize) |
5537 | 0 | nSwathCols = nXSize; |
5538 | |
|
5539 | 0 | CPLDebug( |
5540 | 0 | "GDAL", |
5541 | 0 | "GDALCopyWholeRasterGetSwathSize(): because of compression and " |
5542 | 0 | "too high block, " |
5543 | 0 | "use partial width at one time"); |
5544 | 0 | } |
5545 | 0 | else if ((nSwathLines % nBlockYSize) != 0) |
5546 | 0 | { |
5547 | | // Round on a multiple of nBlockYSize. |
5548 | 0 | nSwathLines = ROUND_TO(nSwathLines, nBlockYSize); |
5549 | 0 | CPLDebug( |
5550 | 0 | "GDAL", |
5551 | 0 | "GDALCopyWholeRasterGetSwathSize(): because of compression, " |
5552 | 0 | "round nSwathLines to block height : %d", |
5553 | 0 | nSwathLines); |
5554 | 0 | } |
5555 | 0 | } |
5556 | |
|
5557 | 0 | *pnSwathCols = nSwathCols; |
5558 | 0 | *pnSwathLines = nSwathLines; |
5559 | 0 | } |
5560 | | |
5561 | | /************************************************************************/ |
5562 | | /* GDALDatasetCopyWholeRaster() */ |
5563 | | /************************************************************************/ |
5564 | | |
5565 | | /** |
5566 | | * \brief Copy all dataset raster data. |
5567 | | * |
5568 | | * This function copies the complete raster contents of one dataset to |
5569 | | * another similarly configured dataset. The source and destination |
5570 | | * dataset must have the same number of bands, and the same width |
5571 | | * and height. The bands do not have to have the same data type. |
5572 | | * |
5573 | | * This function is primarily intended to support implementation of |
5574 | | * driver specific CreateCopy() functions. It implements efficient copying, |
5575 | | * in particular "chunking" the copy in substantial blocks and, if appropriate, |
5576 | | * performing the transfer in a pixel interleaved fashion. |
5577 | | * |
5578 | | * Currently the only papszOptions value supported are : |
5579 | | * <ul> |
5580 | | * <li>"INTERLEAVE=PIXEL/BAND" to force pixel (resp. band) interleaved read and |
5581 | | * write access pattern (this does not modify the layout of the destination |
5582 | | * data)</li> |
5583 | | * <li>"COMPRESSED=YES" to force alignment on target dataset block |
5584 | | * sizes to achieve best compression.</li> |
5585 | | * <li>"SKIP_HOLES=YES" to skip chunks |
5586 | | * for which GDALGetDataCoverageStatus() returns GDAL_DATA_COVERAGE_STATUS_EMPTY |
5587 | | * (GDAL >= 2.2)</li> |
5588 | | * </ul> |
5589 | | * More options may be supported in the future. |
5590 | | * |
5591 | | * @param hSrcDS the source dataset |
5592 | | * @param hDstDS the destination dataset |
5593 | | * @param papszOptions transfer hints in "StringList" Name=Value format. |
5594 | | * @param pfnProgress progress reporting function. |
5595 | | * @param pProgressData callback data for progress function. |
5596 | | * |
5597 | | * @return CE_None on success, or CE_Failure on failure. |
5598 | | */ |
5599 | | |
5600 | | CPLErr CPL_STDCALL GDALDatasetCopyWholeRaster(GDALDatasetH hSrcDS, |
5601 | | GDALDatasetH hDstDS, |
5602 | | CSLConstList papszOptions, |
5603 | | GDALProgressFunc pfnProgress, |
5604 | | void *pProgressData) |
5605 | | |
5606 | 0 | { |
5607 | 0 | VALIDATE_POINTER1(hSrcDS, "GDALDatasetCopyWholeRaster", CE_Failure); |
5608 | 0 | VALIDATE_POINTER1(hDstDS, "GDALDatasetCopyWholeRaster", CE_Failure); |
5609 | | |
5610 | 0 | GDALDataset *poSrcDS = GDALDataset::FromHandle(hSrcDS); |
5611 | 0 | GDALDataset *poDstDS = GDALDataset::FromHandle(hDstDS); |
5612 | |
|
5613 | 0 | if (pfnProgress == nullptr) |
5614 | 0 | pfnProgress = GDALDummyProgress; |
5615 | | |
5616 | | /* -------------------------------------------------------------------- */ |
5617 | | /* Confirm the datasets match in size and band counts. */ |
5618 | | /* -------------------------------------------------------------------- */ |
5619 | 0 | const int nXSize = poDstDS->GetRasterXSize(); |
5620 | 0 | const int nYSize = poDstDS->GetRasterYSize(); |
5621 | 0 | const int nBandCount = poDstDS->GetRasterCount(); |
5622 | |
|
5623 | 0 | if (poSrcDS->GetRasterXSize() != nXSize || |
5624 | 0 | poSrcDS->GetRasterYSize() != nYSize || |
5625 | 0 | poSrcDS->GetRasterCount() != nBandCount) |
5626 | 0 | { |
5627 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
5628 | 0 | "Input and output dataset sizes or band counts do not\n" |
5629 | 0 | "match in GDALDatasetCopyWholeRaster()"); |
5630 | 0 | return CE_Failure; |
5631 | 0 | } |
5632 | | |
5633 | | /* -------------------------------------------------------------------- */ |
5634 | | /* Report preliminary (0) progress. */ |
5635 | | /* -------------------------------------------------------------------- */ |
5636 | 0 | if (!pfnProgress(0.0, nullptr, pProgressData)) |
5637 | 0 | { |
5638 | 0 | CPLError(CE_Failure, CPLE_UserInterrupt, |
5639 | 0 | "User terminated CreateCopy()"); |
5640 | 0 | return CE_Failure; |
5641 | 0 | } |
5642 | | |
5643 | | /* -------------------------------------------------------------------- */ |
5644 | | /* Get our prototype band, and assume the others are similarly */ |
5645 | | /* configured. */ |
5646 | | /* -------------------------------------------------------------------- */ |
5647 | 0 | if (nBandCount == 0) |
5648 | 0 | return CE_None; |
5649 | | |
5650 | 0 | GDALRasterBand *poSrcPrototypeBand = poSrcDS->GetRasterBand(1); |
5651 | 0 | GDALRasterBand *poDstPrototypeBand = poDstDS->GetRasterBand(1); |
5652 | 0 | GDALDataType eDT = poDstPrototypeBand->GetRasterDataType(); |
5653 | | |
5654 | | /* -------------------------------------------------------------------- */ |
5655 | | /* Do we want to try and do the operation in a pixel */ |
5656 | | /* interleaved fashion? */ |
5657 | | /* -------------------------------------------------------------------- */ |
5658 | 0 | bool bInterleave = false; |
5659 | 0 | const char *pszInterleave = |
5660 | 0 | poSrcDS->GetMetadataItem(GDALMD_INTERLEAVE, GDAL_MDD_IMAGE_STRUCTURE); |
5661 | 0 | if (pszInterleave != nullptr && |
5662 | 0 | (EQUAL(pszInterleave, "PIXEL") || EQUAL(pszInterleave, "LINE"))) |
5663 | 0 | bInterleave = true; |
5664 | |
|
5665 | 0 | pszInterleave = |
5666 | 0 | poDstDS->GetMetadataItem(GDALMD_INTERLEAVE, GDAL_MDD_IMAGE_STRUCTURE); |
5667 | 0 | if (pszInterleave != nullptr && |
5668 | 0 | (EQUAL(pszInterleave, "PIXEL") || EQUAL(pszInterleave, "LINE"))) |
5669 | 0 | bInterleave = true; |
5670 | |
|
5671 | 0 | pszInterleave = CSLFetchNameValue(papszOptions, GDALMD_INTERLEAVE); |
5672 | 0 | if (pszInterleave != nullptr && EQUAL(pszInterleave, "PIXEL")) |
5673 | 0 | bInterleave = true; |
5674 | 0 | else if (pszInterleave != nullptr && EQUAL(pszInterleave, "BAND")) |
5675 | 0 | bInterleave = false; |
5676 | | // attributes is specific to the TileDB driver |
5677 | 0 | else if (pszInterleave != nullptr && EQUAL(pszInterleave, "ATTRIBUTES")) |
5678 | 0 | bInterleave = true; |
5679 | 0 | else if (pszInterleave != nullptr) |
5680 | 0 | { |
5681 | 0 | CPLError(CE_Warning, CPLE_NotSupported, |
5682 | 0 | "Unsupported value for option INTERLEAVE"); |
5683 | 0 | } |
5684 | | |
5685 | | // If the destination is compressed, we must try to write blocks just once, |
5686 | | // to save disk space (GTiff case for example), and to avoid data loss |
5687 | | // (JPEG compression for example). |
5688 | 0 | bool bDstIsCompressed = false; |
5689 | 0 | const char *pszDstCompressed = |
5690 | 0 | CSLFetchNameValue(papszOptions, "COMPRESSED"); |
5691 | 0 | if (pszDstCompressed != nullptr && CPLTestBool(pszDstCompressed)) |
5692 | 0 | bDstIsCompressed = true; |
5693 | | |
5694 | | /* -------------------------------------------------------------------- */ |
5695 | | /* What will our swath size be? */ |
5696 | | /* -------------------------------------------------------------------- */ |
5697 | |
|
5698 | 0 | int nSwathCols = 0; |
5699 | 0 | int nSwathLines = 0; |
5700 | 0 | GDALCopyWholeRasterGetSwathSize(poSrcPrototypeBand, poDstPrototypeBand, |
5701 | 0 | nBandCount, bDstIsCompressed, bInterleave, |
5702 | 0 | &nSwathCols, &nSwathLines); |
5703 | |
|
5704 | 0 | int nPixelSize = GDALGetDataTypeSizeBytes(eDT); |
5705 | 0 | if (bInterleave) |
5706 | 0 | nPixelSize *= nBandCount; |
5707 | |
|
5708 | 0 | void *pSwathBuf = VSI_MALLOC3_VERBOSE(nSwathCols, nSwathLines, nPixelSize); |
5709 | 0 | if (pSwathBuf == nullptr) |
5710 | 0 | { |
5711 | 0 | return CE_Failure; |
5712 | 0 | } |
5713 | | |
5714 | 0 | CPLDebug("GDAL", |
5715 | 0 | "GDALDatasetCopyWholeRaster(): %d*%d swaths, bInterleave=%d", |
5716 | 0 | nSwathCols, nSwathLines, static_cast<int>(bInterleave)); |
5717 | | |
5718 | | // Advise the source raster that we are going to read it completely |
5719 | | // Note: this might already have been done by GDALCreateCopy() in the |
5720 | | // likely case this function is indirectly called by it |
5721 | 0 | poSrcDS->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nBandCount, |
5722 | 0 | nullptr, nullptr); |
5723 | | |
5724 | | /* ==================================================================== */ |
5725 | | /* Band oriented (uninterleaved) case. */ |
5726 | | /* ==================================================================== */ |
5727 | 0 | CPLErr eErr = CE_None; |
5728 | 0 | const bool bCheckHoles = |
5729 | 0 | CPLTestBool(CSLFetchNameValueDef(papszOptions, "SKIP_HOLES", "NO")); |
5730 | |
|
5731 | 0 | if (!bInterleave) |
5732 | 0 | { |
5733 | 0 | GDALRasterIOExtraArg sExtraArg; |
5734 | 0 | INIT_RASTERIO_EXTRA_ARG(sExtraArg); |
5735 | 0 | CPL_IGNORE_RET_VAL(sExtraArg.pfnProgress); // to make cppcheck happy |
5736 | |
|
5737 | 0 | const GIntBig nTotalBlocks = static_cast<GIntBig>(nBandCount) * |
5738 | 0 | DIV_ROUND_UP(nYSize, nSwathLines) * |
5739 | 0 | DIV_ROUND_UP(nXSize, nSwathCols); |
5740 | 0 | GIntBig nBlocksDone = 0; |
5741 | |
|
5742 | 0 | for (int iBand = 0; iBand < nBandCount && eErr == CE_None; iBand++) |
5743 | 0 | { |
5744 | 0 | int nBand = iBand + 1; |
5745 | |
|
5746 | 0 | for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines) |
5747 | 0 | { |
5748 | 0 | int nThisLines = nSwathLines; |
5749 | |
|
5750 | 0 | if (iY + nThisLines > nYSize) |
5751 | 0 | nThisLines = nYSize - iY; |
5752 | |
|
5753 | 0 | for (int iX = 0; iX < nXSize && eErr == CE_None; |
5754 | 0 | iX += nSwathCols) |
5755 | 0 | { |
5756 | 0 | int nThisCols = nSwathCols; |
5757 | |
|
5758 | 0 | if (iX + nThisCols > nXSize) |
5759 | 0 | nThisCols = nXSize - iX; |
5760 | |
|
5761 | 0 | int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA; |
5762 | 0 | if (bCheckHoles) |
5763 | 0 | { |
5764 | 0 | nStatus = poSrcDS->GetRasterBand(nBand) |
5765 | 0 | ->GetDataCoverageStatus( |
5766 | 0 | iX, iY, nThisCols, nThisLines, |
5767 | 0 | GDAL_DATA_COVERAGE_STATUS_DATA); |
5768 | 0 | } |
5769 | 0 | if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA) |
5770 | 0 | { |
5771 | 0 | sExtraArg.pfnProgress = GDALScaledProgress; |
5772 | 0 | sExtraArg.pProgressData = GDALCreateScaledProgress( |
5773 | 0 | nBlocksDone / static_cast<double>(nTotalBlocks), |
5774 | 0 | (nBlocksDone + 0.5) / |
5775 | 0 | static_cast<double>(nTotalBlocks), |
5776 | 0 | pfnProgress, pProgressData); |
5777 | 0 | if (sExtraArg.pProgressData == nullptr) |
5778 | 0 | sExtraArg.pfnProgress = nullptr; |
5779 | |
|
5780 | 0 | eErr = poSrcDS->RasterIO(GF_Read, iX, iY, nThisCols, |
5781 | 0 | nThisLines, pSwathBuf, |
5782 | 0 | nThisCols, nThisLines, eDT, 1, |
5783 | 0 | &nBand, 0, 0, 0, &sExtraArg); |
5784 | |
|
5785 | 0 | GDALDestroyScaledProgress(sExtraArg.pProgressData); |
5786 | |
|
5787 | 0 | if (eErr == CE_None) |
5788 | 0 | eErr = poDstDS->RasterIO( |
5789 | 0 | GF_Write, iX, iY, nThisCols, nThisLines, |
5790 | 0 | pSwathBuf, nThisCols, nThisLines, eDT, 1, |
5791 | 0 | &nBand, 0, 0, 0, nullptr); |
5792 | 0 | } |
5793 | |
|
5794 | 0 | nBlocksDone++; |
5795 | 0 | if (eErr == CE_None && |
5796 | 0 | !pfnProgress(nBlocksDone / |
5797 | 0 | static_cast<double>(nTotalBlocks), |
5798 | 0 | nullptr, pProgressData)) |
5799 | 0 | { |
5800 | 0 | eErr = CE_Failure; |
5801 | 0 | CPLError(CE_Failure, CPLE_UserInterrupt, |
5802 | 0 | "User terminated CreateCopy()"); |
5803 | 0 | } |
5804 | 0 | } |
5805 | 0 | } |
5806 | 0 | } |
5807 | 0 | } |
5808 | | |
5809 | | /* ==================================================================== */ |
5810 | | /* Pixel interleaved case. */ |
5811 | | /* ==================================================================== */ |
5812 | 0 | else /* if( bInterleave ) */ |
5813 | 0 | { |
5814 | 0 | GDALRasterIOExtraArg sExtraArg; |
5815 | 0 | INIT_RASTERIO_EXTRA_ARG(sExtraArg); |
5816 | 0 | CPL_IGNORE_RET_VAL(sExtraArg.pfnProgress); // to make cppcheck happy |
5817 | |
|
5818 | 0 | const GIntBig nTotalBlocks = |
5819 | 0 | static_cast<GIntBig>(DIV_ROUND_UP(nYSize, nSwathLines)) * |
5820 | 0 | DIV_ROUND_UP(nXSize, nSwathCols); |
5821 | 0 | GIntBig nBlocksDone = 0; |
5822 | |
|
5823 | 0 | for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines) |
5824 | 0 | { |
5825 | 0 | int nThisLines = nSwathLines; |
5826 | |
|
5827 | 0 | if (iY + nThisLines > nYSize) |
5828 | 0 | nThisLines = nYSize - iY; |
5829 | |
|
5830 | 0 | for (int iX = 0; iX < nXSize && eErr == CE_None; iX += nSwathCols) |
5831 | 0 | { |
5832 | 0 | int nThisCols = nSwathCols; |
5833 | |
|
5834 | 0 | if (iX + nThisCols > nXSize) |
5835 | 0 | nThisCols = nXSize - iX; |
5836 | |
|
5837 | 0 | int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA; |
5838 | 0 | if (bCheckHoles) |
5839 | 0 | { |
5840 | 0 | nStatus = 0; |
5841 | 0 | for (int iBand = 0; iBand < nBandCount; iBand++) |
5842 | 0 | { |
5843 | 0 | nStatus |= poSrcDS->GetRasterBand(iBand + 1) |
5844 | 0 | ->GetDataCoverageStatus( |
5845 | 0 | iX, iY, nThisCols, nThisLines, |
5846 | 0 | GDAL_DATA_COVERAGE_STATUS_DATA); |
5847 | 0 | if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA) |
5848 | 0 | break; |
5849 | 0 | } |
5850 | 0 | } |
5851 | 0 | if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA) |
5852 | 0 | { |
5853 | 0 | sExtraArg.pfnProgress = GDALScaledProgress; |
5854 | 0 | sExtraArg.pProgressData = GDALCreateScaledProgress( |
5855 | 0 | nBlocksDone / static_cast<double>(nTotalBlocks), |
5856 | 0 | (nBlocksDone + 0.5) / static_cast<double>(nTotalBlocks), |
5857 | 0 | pfnProgress, pProgressData); |
5858 | 0 | if (sExtraArg.pProgressData == nullptr) |
5859 | 0 | sExtraArg.pfnProgress = nullptr; |
5860 | |
|
5861 | 0 | eErr = poSrcDS->RasterIO(GF_Read, iX, iY, nThisCols, |
5862 | 0 | nThisLines, pSwathBuf, nThisCols, |
5863 | 0 | nThisLines, eDT, nBandCount, |
5864 | 0 | nullptr, 0, 0, 0, &sExtraArg); |
5865 | |
|
5866 | 0 | GDALDestroyScaledProgress(sExtraArg.pProgressData); |
5867 | |
|
5868 | 0 | if (eErr == CE_None) |
5869 | 0 | eErr = poDstDS->RasterIO( |
5870 | 0 | GF_Write, iX, iY, nThisCols, nThisLines, pSwathBuf, |
5871 | 0 | nThisCols, nThisLines, eDT, nBandCount, nullptr, 0, |
5872 | 0 | 0, 0, nullptr); |
5873 | 0 | } |
5874 | |
|
5875 | 0 | nBlocksDone++; |
5876 | 0 | if (eErr == CE_None && |
5877 | 0 | !pfnProgress(nBlocksDone / |
5878 | 0 | static_cast<double>(nTotalBlocks), |
5879 | 0 | nullptr, pProgressData)) |
5880 | 0 | { |
5881 | 0 | eErr = CE_Failure; |
5882 | 0 | CPLError(CE_Failure, CPLE_UserInterrupt, |
5883 | 0 | "User terminated CreateCopy()"); |
5884 | 0 | } |
5885 | 0 | } |
5886 | 0 | } |
5887 | 0 | } |
5888 | | |
5889 | | /* -------------------------------------------------------------------- */ |
5890 | | /* Cleanup */ |
5891 | | /* -------------------------------------------------------------------- */ |
5892 | 0 | CPLFree(pSwathBuf); |
5893 | |
|
5894 | 0 | return eErr; |
5895 | 0 | } |
5896 | | |
5897 | | /************************************************************************/ |
5898 | | /* GDALRasterBandCopyWholeRaster() */ |
5899 | | /************************************************************************/ |
5900 | | |
5901 | | /** |
5902 | | * \brief Copy a whole raster band |
5903 | | * |
5904 | | * This function copies the complete raster contents of one band to |
5905 | | * another similarly configured band. The source and destination |
5906 | | * bands must have the same width and height. The bands do not have |
5907 | | * to have the same data type. |
5908 | | * |
5909 | | * It implements efficient copying, in particular "chunking" the copy in |
5910 | | * substantial blocks. |
5911 | | * |
5912 | | * Currently the only papszOptions value supported are : |
5913 | | * <ul> |
5914 | | * <li>"COMPRESSED=YES" to force alignment on target dataset block sizes to |
5915 | | * achieve best compression.</li> |
5916 | | * <li>"SKIP_HOLES=YES" to skip chunks for which GDALGetDataCoverageStatus() |
5917 | | * returns GDAL_DATA_COVERAGE_STATUS_EMPTY (GDAL >= 2.2)</li> |
5918 | | * </ul> |
5919 | | * |
5920 | | * @param hSrcBand the source band |
5921 | | * @param hDstBand the destination band |
5922 | | * @param papszOptions transfer hints in "StringList" Name=Value format. |
5923 | | * @param pfnProgress progress reporting function. |
5924 | | * @param pProgressData callback data for progress function. |
5925 | | * |
5926 | | * @return CE_None on success, or CE_Failure on failure. |
5927 | | */ |
5928 | | |
5929 | | CPLErr CPL_STDCALL GDALRasterBandCopyWholeRaster( |
5930 | | GDALRasterBandH hSrcBand, GDALRasterBandH hDstBand, |
5931 | | const char *const *const papszOptions, GDALProgressFunc pfnProgress, |
5932 | | void *pProgressData) |
5933 | | |
5934 | 0 | { |
5935 | 0 | VALIDATE_POINTER1(hSrcBand, "GDALRasterBandCopyWholeRaster", CE_Failure); |
5936 | 0 | VALIDATE_POINTER1(hDstBand, "GDALRasterBandCopyWholeRaster", CE_Failure); |
5937 | | |
5938 | 0 | GDALRasterBand *poSrcBand = GDALRasterBand::FromHandle(hSrcBand); |
5939 | 0 | GDALRasterBand *poDstBand = GDALRasterBand::FromHandle(hDstBand); |
5940 | 0 | CPLErr eErr = CE_None; |
5941 | |
|
5942 | 0 | if (pfnProgress == nullptr) |
5943 | 0 | pfnProgress = GDALDummyProgress; |
5944 | | |
5945 | | /* -------------------------------------------------------------------- */ |
5946 | | /* Confirm the datasets match in size and band counts. */ |
5947 | | /* -------------------------------------------------------------------- */ |
5948 | 0 | int nXSize = poSrcBand->GetXSize(); |
5949 | 0 | int nYSize = poSrcBand->GetYSize(); |
5950 | |
|
5951 | 0 | if (poDstBand->GetXSize() != nXSize || poDstBand->GetYSize() != nYSize) |
5952 | 0 | { |
5953 | 0 | CPLError(CE_Failure, CPLE_AppDefined, |
5954 | 0 | "Input and output band sizes do not\n" |
5955 | 0 | "match in GDALRasterBandCopyWholeRaster()"); |
5956 | 0 | return CE_Failure; |
5957 | 0 | } |
5958 | | |
5959 | | /* -------------------------------------------------------------------- */ |
5960 | | /* Report preliminary (0) progress. */ |
5961 | | /* -------------------------------------------------------------------- */ |
5962 | 0 | if (!pfnProgress(0.0, nullptr, pProgressData)) |
5963 | 0 | { |
5964 | 0 | CPLError(CE_Failure, CPLE_UserInterrupt, |
5965 | 0 | "User terminated CreateCopy()"); |
5966 | 0 | return CE_Failure; |
5967 | 0 | } |
5968 | | |
5969 | 0 | GDALDataType eDT = poDstBand->GetRasterDataType(); |
5970 | | |
5971 | | // If the destination is compressed, we must try to write blocks just once, |
5972 | | // to save disk space (GTiff case for example), and to avoid data loss |
5973 | | // (JPEG compression for example). |
5974 | 0 | bool bDstIsCompressed = false; |
5975 | 0 | const char *pszDstCompressed = |
5976 | 0 | CSLFetchNameValue(const_cast<char **>(papszOptions), "COMPRESSED"); |
5977 | 0 | if (pszDstCompressed != nullptr && CPLTestBool(pszDstCompressed)) |
5978 | 0 | bDstIsCompressed = true; |
5979 | | |
5980 | | /* -------------------------------------------------------------------- */ |
5981 | | /* What will our swath size be? */ |
5982 | | /* -------------------------------------------------------------------- */ |
5983 | |
|
5984 | 0 | int nSwathCols = 0; |
5985 | 0 | int nSwathLines = 0; |
5986 | 0 | GDALCopyWholeRasterGetSwathSize(poSrcBand, poDstBand, 1, bDstIsCompressed, |
5987 | 0 | FALSE, &nSwathCols, &nSwathLines); |
5988 | |
|
5989 | 0 | const int nPixelSize = GDALGetDataTypeSizeBytes(eDT); |
5990 | |
|
5991 | 0 | void *pSwathBuf = VSI_MALLOC3_VERBOSE(nSwathCols, nSwathLines, nPixelSize); |
5992 | 0 | if (pSwathBuf == nullptr) |
5993 | 0 | { |
5994 | 0 | return CE_Failure; |
5995 | 0 | } |
5996 | | |
5997 | 0 | CPLDebug("GDAL", "GDALRasterBandCopyWholeRaster(): %d*%d swaths", |
5998 | 0 | nSwathCols, nSwathLines); |
5999 | |
|
6000 | 0 | const bool bCheckHoles = |
6001 | 0 | CPLTestBool(CSLFetchNameValueDef(papszOptions, "SKIP_HOLES", "NO")); |
6002 | | |
6003 | | // Advise the source raster that we are going to read it completely |
6004 | 0 | poSrcBand->AdviseRead(0, 0, nXSize, nYSize, nXSize, nYSize, eDT, nullptr); |
6005 | | |
6006 | | /* ==================================================================== */ |
6007 | | /* Band oriented (uninterleaved) case. */ |
6008 | | /* ==================================================================== */ |
6009 | |
|
6010 | 0 | for (int iY = 0; iY < nYSize && eErr == CE_None; iY += nSwathLines) |
6011 | 0 | { |
6012 | 0 | int nThisLines = nSwathLines; |
6013 | |
|
6014 | 0 | if (iY + nThisLines > nYSize) |
6015 | 0 | nThisLines = nYSize - iY; |
6016 | |
|
6017 | 0 | for (int iX = 0; iX < nXSize && eErr == CE_None; iX += nSwathCols) |
6018 | 0 | { |
6019 | 0 | int nThisCols = nSwathCols; |
6020 | |
|
6021 | 0 | if (iX + nThisCols > nXSize) |
6022 | 0 | nThisCols = nXSize - iX; |
6023 | |
|
6024 | 0 | int nStatus = GDAL_DATA_COVERAGE_STATUS_DATA; |
6025 | 0 | if (bCheckHoles) |
6026 | 0 | { |
6027 | 0 | nStatus = poSrcBand->GetDataCoverageStatus( |
6028 | 0 | iX, iY, nThisCols, nThisLines, |
6029 | 0 | GDAL_DATA_COVERAGE_STATUS_DATA); |
6030 | 0 | } |
6031 | 0 | if (nStatus & GDAL_DATA_COVERAGE_STATUS_DATA) |
6032 | 0 | { |
6033 | 0 | eErr = poSrcBand->RasterIO(GF_Read, iX, iY, nThisCols, |
6034 | 0 | nThisLines, pSwathBuf, nThisCols, |
6035 | 0 | nThisLines, eDT, 0, 0, nullptr); |
6036 | |
|
6037 | 0 | if (eErr == CE_None) |
6038 | 0 | eErr = poDstBand->RasterIO(GF_Write, iX, iY, nThisCols, |
6039 | 0 | nThisLines, pSwathBuf, nThisCols, |
6040 | 0 | nThisLines, eDT, 0, 0, nullptr); |
6041 | 0 | } |
6042 | |
|
6043 | 0 | if (eErr == CE_None && !pfnProgress(double(iY + nThisLines) / |
6044 | 0 | static_cast<double>(nYSize), |
6045 | 0 | nullptr, pProgressData)) |
6046 | 0 | { |
6047 | 0 | eErr = CE_Failure; |
6048 | 0 | CPLError(CE_Failure, CPLE_UserInterrupt, |
6049 | 0 | "User terminated CreateCopy()"); |
6050 | 0 | } |
6051 | 0 | } |
6052 | 0 | } |
6053 | | |
6054 | | /* -------------------------------------------------------------------- */ |
6055 | | /* Cleanup */ |
6056 | | /* -------------------------------------------------------------------- */ |
6057 | 0 | CPLFree(pSwathBuf); |
6058 | |
|
6059 | 0 | return eErr; |
6060 | 0 | } |
6061 | | |
6062 | | /************************************************************************/ |
6063 | | /* GDALCopyRasterIOExtraArg () */ |
6064 | | /************************************************************************/ |
6065 | | |
6066 | | void GDALCopyRasterIOExtraArg(GDALRasterIOExtraArg *psDestArg, |
6067 | | const GDALRasterIOExtraArg *psSrcArg) |
6068 | 0 | { |
6069 | 0 | INIT_RASTERIO_EXTRA_ARG(*psDestArg); |
6070 | 0 | if (psSrcArg) |
6071 | 0 | { |
6072 | 0 | psDestArg->eResampleAlg = psSrcArg->eResampleAlg; |
6073 | 0 | psDestArg->pfnProgress = psSrcArg->pfnProgress; |
6074 | 0 | psDestArg->pProgressData = psSrcArg->pProgressData; |
6075 | 0 | psDestArg->bFloatingPointWindowValidity = |
6076 | 0 | psSrcArg->bFloatingPointWindowValidity; |
6077 | 0 | if (psSrcArg->bFloatingPointWindowValidity) |
6078 | 0 | { |
6079 | 0 | psDestArg->dfXOff = psSrcArg->dfXOff; |
6080 | 0 | psDestArg->dfYOff = psSrcArg->dfYOff; |
6081 | 0 | psDestArg->dfXSize = psSrcArg->dfXSize; |
6082 | 0 | psDestArg->dfYSize = psSrcArg->dfYSize; |
6083 | 0 | } |
6084 | 0 | if (psSrcArg->nVersion >= 2) |
6085 | 0 | { |
6086 | 0 | psDestArg->bUseOnlyThisScale = psSrcArg->bUseOnlyThisScale; |
6087 | 0 | } |
6088 | 0 | if (psSrcArg->nVersion >= 3) |
6089 | 0 | { |
6090 | 0 | psDestArg->bOperateInBufType = psSrcArg->bOperateInBufType; |
6091 | 0 | } |
6092 | 0 | } |
6093 | 0 | } |
6094 | | |
6095 | | /************************************************************************/ |
6096 | | /* HasOnlyNoData() */ |
6097 | | /************************************************************************/ |
6098 | | |
6099 | | template <class T> static inline bool IsEqualToNoData(T value, T noDataValue) |
6100 | 0 | { |
6101 | 0 | return value == noDataValue; |
6102 | 0 | } Unexecuted instantiation: rasterio.cpp:bool IsEqualToNoData<unsigned char>(unsigned char, unsigned char) Unexecuted instantiation: rasterio.cpp:bool IsEqualToNoData<unsigned short>(unsigned short, unsigned short) Unexecuted instantiation: rasterio.cpp:bool IsEqualToNoData<unsigned int>(unsigned int, unsigned int) Unexecuted instantiation: rasterio.cpp:bool IsEqualToNoData<unsigned long>(unsigned long, unsigned long) |
6103 | | |
6104 | | template <> bool IsEqualToNoData<GFloat16>(GFloat16 value, GFloat16 noDataValue) |
6105 | 0 | { |
6106 | 0 | using std::isnan; |
6107 | 0 | return isnan(noDataValue) ? isnan(value) : value == noDataValue; |
6108 | 0 | } |
6109 | | |
6110 | | template <> bool IsEqualToNoData<float>(float value, float noDataValue) |
6111 | 0 | { |
6112 | 0 | return std::isnan(noDataValue) ? std::isnan(value) : value == noDataValue; |
6113 | 0 | } |
6114 | | |
6115 | | template <> bool IsEqualToNoData<double>(double value, double noDataValue) |
6116 | 0 | { |
6117 | 0 | return std::isnan(noDataValue) ? std::isnan(value) : value == noDataValue; |
6118 | 0 | } |
6119 | | |
6120 | | template <class T> |
6121 | | static bool HasOnlyNoDataT(const T *pBuffer, T noDataValue, size_t nWidth, |
6122 | | size_t nHeight, size_t nLineStride, |
6123 | | size_t nComponents) |
6124 | 0 | { |
6125 | | // Fast test: check the 4 corners and the middle pixel. |
6126 | 0 | for (size_t iBand = 0; iBand < nComponents; iBand++) |
6127 | 0 | { |
6128 | 0 | if (!(IsEqualToNoData(pBuffer[iBand], noDataValue) && |
6129 | 0 | IsEqualToNoData(pBuffer[(nWidth - 1) * nComponents + iBand], |
6130 | 0 | noDataValue) && |
6131 | 0 | IsEqualToNoData( |
6132 | 0 | pBuffer[((nHeight - 1) / 2 * nLineStride + (nWidth - 1) / 2) * |
6133 | 0 | nComponents + |
6134 | 0 | iBand], |
6135 | 0 | noDataValue) && |
6136 | 0 | IsEqualToNoData( |
6137 | 0 | pBuffer[(nHeight - 1) * nLineStride * nComponents + iBand], |
6138 | 0 | noDataValue) && |
6139 | 0 | IsEqualToNoData( |
6140 | 0 | pBuffer[((nHeight - 1) * nLineStride + nWidth - 1) * |
6141 | 0 | nComponents + |
6142 | 0 | iBand], |
6143 | 0 | noDataValue))) |
6144 | 0 | { |
6145 | 0 | return false; |
6146 | 0 | } |
6147 | 0 | } |
6148 | | |
6149 | | // Test all pixels. |
6150 | 0 | for (size_t iY = 0; iY < nHeight; iY++) |
6151 | 0 | { |
6152 | 0 | const T *pBufferLine = pBuffer + iY * nLineStride * nComponents; |
6153 | 0 | for (size_t iX = 0; iX < nWidth * nComponents; iX++) |
6154 | 0 | { |
6155 | 0 | if (!IsEqualToNoData(pBufferLine[iX], noDataValue)) |
6156 | 0 | { |
6157 | 0 | return false; |
6158 | 0 | } |
6159 | 0 | } |
6160 | 0 | } |
6161 | 0 | return true; |
6162 | 0 | } Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<unsigned char>(unsigned char const*, unsigned char, unsigned long, unsigned long, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<unsigned short>(unsigned short const*, unsigned short, unsigned long, unsigned long, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<unsigned int>(unsigned int const*, unsigned int, unsigned long, unsigned long, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<unsigned long>(unsigned long const*, unsigned long, unsigned long, unsigned long, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<cpl::Float16>(cpl::Float16 const*, cpl::Float16, unsigned long, unsigned long, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<float>(float const*, float, unsigned long, unsigned long, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:bool HasOnlyNoDataT<double>(double const*, double, unsigned long, unsigned long, unsigned long, unsigned long) |
6163 | | |
6164 | | /************************************************************************/ |
6165 | | /* GDALBufferHasOnlyNoData() */ |
6166 | | /************************************************************************/ |
6167 | | |
6168 | | bool GDALBufferHasOnlyNoData(const void *pBuffer, double dfNoDataValue, |
6169 | | size_t nWidth, size_t nHeight, size_t nLineStride, |
6170 | | size_t nComponents, int nBitsPerSample, |
6171 | | GDALBufferSampleFormat nSampleFormat) |
6172 | 0 | { |
6173 | | // In the case where the nodata is 0, we can compare several bytes at |
6174 | | // once. Select the largest natural integer type for the architecture. |
6175 | 0 | if (dfNoDataValue == 0.0 && nWidth == nLineStride && |
6176 | | // Do not use this optimized code path for floating point numbers, |
6177 | | // as it can't detect negative zero. |
6178 | 0 | nSampleFormat != GSF_FLOATING_POINT) |
6179 | 0 | { |
6180 | 0 | const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer); |
6181 | 0 | const size_t nSize = |
6182 | 0 | static_cast<size_t>((static_cast<uint64_t>(nWidth) * nHeight * |
6183 | 0 | nComponents * nBitsPerSample + |
6184 | 0 | 7) / |
6185 | 0 | 8); |
6186 | 0 | #ifdef HAVE_SSE2 |
6187 | 0 | size_t n = nSize; |
6188 | | // Align to 16 bytes |
6189 | 0 | while ((reinterpret_cast<uintptr_t>(pabyBuffer) & 15) != 0 && n > 0) |
6190 | 0 | { |
6191 | 0 | --n; |
6192 | 0 | if (*pabyBuffer) |
6193 | 0 | return false; |
6194 | 0 | pabyBuffer++; |
6195 | 0 | } |
6196 | | |
6197 | 0 | const auto zero = _mm_setzero_si128(); |
6198 | 0 | constexpr int UNROLLING = 4; |
6199 | 0 | while (n >= UNROLLING * sizeof(zero)) |
6200 | 0 | { |
6201 | 0 | const auto v0 = _mm_load_si128(reinterpret_cast<const __m128i *>( |
6202 | 0 | pabyBuffer + 0 * sizeof(zero))); |
6203 | 0 | const auto v1 = _mm_load_si128(reinterpret_cast<const __m128i *>( |
6204 | 0 | pabyBuffer + 1 * sizeof(zero))); |
6205 | 0 | const auto v2 = _mm_load_si128(reinterpret_cast<const __m128i *>( |
6206 | 0 | pabyBuffer + 2 * sizeof(zero))); |
6207 | 0 | const auto v3 = _mm_load_si128(reinterpret_cast<const __m128i *>( |
6208 | 0 | pabyBuffer + 3 * sizeof(zero))); |
6209 | 0 | const auto v = |
6210 | 0 | _mm_or_si128(_mm_or_si128(v0, v1), _mm_or_si128(v2, v3)); |
6211 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
6212 | | if (!_mm_test_all_zeros(v, v)) |
6213 | | #else |
6214 | 0 | if (_mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) != 0xFFFF) |
6215 | 0 | #endif |
6216 | 0 | { |
6217 | 0 | return false; |
6218 | 0 | } |
6219 | 0 | pabyBuffer += UNROLLING * sizeof(zero); |
6220 | 0 | n -= UNROLLING * sizeof(zero); |
6221 | 0 | } |
6222 | | |
6223 | 0 | while (n > 0) |
6224 | 0 | { |
6225 | 0 | --n; |
6226 | 0 | if (*pabyBuffer) |
6227 | 0 | return false; |
6228 | 0 | pabyBuffer++; |
6229 | 0 | } |
6230 | | #else |
6231 | | #if SIZEOF_VOIDP >= 8 || defined(__x86_64__) |
6232 | | // We test __x86_64__ for x32 arch where SIZEOF_VOIDP == 4 |
6233 | | typedef std::uint64_t WordType; |
6234 | | #else |
6235 | | typedef std::uint32_t WordType; |
6236 | | #endif |
6237 | | |
6238 | | const size_t nInitialIters = |
6239 | | std::min(sizeof(WordType) - |
6240 | | static_cast<size_t>( |
6241 | | reinterpret_cast<std::uintptr_t>(pabyBuffer) % |
6242 | | sizeof(WordType)), |
6243 | | nSize); |
6244 | | size_t i = 0; |
6245 | | for (; i < nInitialIters; i++) |
6246 | | { |
6247 | | if (pabyBuffer[i]) |
6248 | | return false; |
6249 | | } |
6250 | | for (; i + sizeof(WordType) - 1 < nSize; i += sizeof(WordType)) |
6251 | | { |
6252 | | if (*(reinterpret_cast<const WordType *>(pabyBuffer + i))) |
6253 | | return false; |
6254 | | } |
6255 | | for (; i < nSize; i++) |
6256 | | { |
6257 | | if (pabyBuffer[i]) |
6258 | | return false; |
6259 | | } |
6260 | | #endif |
6261 | 0 | return true; |
6262 | 0 | } |
6263 | | |
6264 | 0 | #ifdef HAVE_SSE2 |
6265 | 0 | else if (dfNoDataValue == 0.0 && nWidth == nLineStride && |
6266 | 0 | nBitsPerSample == 32 && nSampleFormat == GSF_FLOATING_POINT) |
6267 | 0 | { |
6268 | 0 | const auto signMask = _mm_set1_epi32(0x7FFFFFFF); |
6269 | 0 | const auto zero = _mm_setzero_si128(); |
6270 | 0 | const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer); |
6271 | 0 | const size_t n = nWidth * nHeight * nComponents; |
6272 | |
|
6273 | 0 | size_t i = 0; |
6274 | 0 | constexpr int UNROLLING = 4; |
6275 | 0 | constexpr size_t VALUES_PER_ITER = |
6276 | 0 | UNROLLING * sizeof(zero) / sizeof(float); |
6277 | 0 | for (; i + VALUES_PER_ITER <= n; i += VALUES_PER_ITER) |
6278 | 0 | { |
6279 | 0 | const auto v0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6280 | 0 | pabyBuffer + 0 * sizeof(zero))); |
6281 | 0 | const auto v1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6282 | 0 | pabyBuffer + 1 * sizeof(zero))); |
6283 | 0 | const auto v2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6284 | 0 | pabyBuffer + 2 * sizeof(zero))); |
6285 | 0 | const auto v3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6286 | 0 | pabyBuffer + 3 * sizeof(zero))); |
6287 | 0 | auto v = _mm_or_si128(_mm_or_si128(v0, v1), _mm_or_si128(v2, v3)); |
6288 | | // Clear the sign bit (makes -0.0 become +0.0) |
6289 | 0 | v = _mm_and_si128(v, signMask); |
6290 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
6291 | | if (!_mm_test_all_zeros(v, v)) |
6292 | | #else |
6293 | 0 | if (_mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) != 0xFFFF) |
6294 | 0 | #endif |
6295 | 0 | { |
6296 | 0 | return false; |
6297 | 0 | } |
6298 | 0 | pabyBuffer += UNROLLING * sizeof(zero); |
6299 | 0 | } |
6300 | | |
6301 | 0 | for (; i < n; i++) |
6302 | 0 | { |
6303 | 0 | uint32_t bits; |
6304 | 0 | memcpy(&bits, pabyBuffer, sizeof(bits)); |
6305 | 0 | pabyBuffer += sizeof(bits); |
6306 | 0 | if ((bits & 0x7FFFFFFF) != 0) |
6307 | 0 | return false; |
6308 | 0 | } |
6309 | | |
6310 | 0 | return true; |
6311 | 0 | } |
6312 | | |
6313 | 0 | else if (dfNoDataValue == 0.0 && nWidth == nLineStride && |
6314 | 0 | nBitsPerSample == 64 && nSampleFormat == GSF_FLOATING_POINT) |
6315 | 0 | { |
6316 | 0 | const auto signMask = _mm_set1_epi64x(0x7FFFFFFFFFFFFFFFLL); |
6317 | 0 | const auto zero = _mm_setzero_si128(); |
6318 | 0 | const GByte *pabyBuffer = static_cast<const GByte *>(pBuffer); |
6319 | 0 | const size_t n = nWidth * nHeight * nComponents; |
6320 | |
|
6321 | 0 | size_t i = 0; |
6322 | 0 | constexpr int UNROLLING = 4; |
6323 | 0 | constexpr size_t VALUES_PER_ITER = |
6324 | 0 | UNROLLING * sizeof(zero) / sizeof(double); |
6325 | 0 | for (; i + VALUES_PER_ITER <= n; i += VALUES_PER_ITER) |
6326 | 0 | { |
6327 | 0 | const auto v0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6328 | 0 | pabyBuffer + 0 * sizeof(zero))); |
6329 | 0 | const auto v1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6330 | 0 | pabyBuffer + 1 * sizeof(zero))); |
6331 | 0 | const auto v2 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6332 | 0 | pabyBuffer + 2 * sizeof(zero))); |
6333 | 0 | const auto v3 = _mm_loadu_si128(reinterpret_cast<const __m128i *>( |
6334 | 0 | pabyBuffer + 3 * sizeof(zero))); |
6335 | 0 | auto v = _mm_or_si128(_mm_or_si128(v0, v1), _mm_or_si128(v2, v3)); |
6336 | | // Clear the sign bit (makes -0.0 become +0.0) |
6337 | 0 | v = _mm_and_si128(v, signMask); |
6338 | | #if defined(__SSE4_1__) || defined(__AVX__) || defined(USE_NEON_OPTIMIZATIONS) |
6339 | | if (!_mm_test_all_zeros(v, v)) |
6340 | | #else |
6341 | 0 | if (_mm_movemask_epi8(_mm_cmpeq_epi8(v, zero)) != 0xFFFF) |
6342 | 0 | #endif |
6343 | 0 | { |
6344 | 0 | return false; |
6345 | 0 | } |
6346 | 0 | pabyBuffer += UNROLLING * sizeof(zero); |
6347 | 0 | } |
6348 | | |
6349 | 0 | for (; i < n; i++) |
6350 | 0 | { |
6351 | 0 | uint64_t bits; |
6352 | 0 | memcpy(&bits, pabyBuffer, sizeof(bits)); |
6353 | 0 | pabyBuffer += sizeof(bits); |
6354 | 0 | if ((bits & 0x7FFFFFFFFFFFFFFFULL) != 0) |
6355 | 0 | return false; |
6356 | 0 | } |
6357 | | |
6358 | 0 | return true; |
6359 | 0 | } |
6360 | 0 | #endif |
6361 | | |
6362 | 0 | if (nBitsPerSample == 8 && nSampleFormat == GSF_UNSIGNED_INT) |
6363 | 0 | { |
6364 | 0 | return GDALIsValueInRange<uint8_t>(dfNoDataValue) && |
6365 | 0 | HasOnlyNoDataT(static_cast<const uint8_t *>(pBuffer), |
6366 | 0 | static_cast<uint8_t>(dfNoDataValue), nWidth, |
6367 | 0 | nHeight, nLineStride, nComponents); |
6368 | 0 | } |
6369 | 0 | if (nBitsPerSample == 8 && nSampleFormat == GSF_SIGNED_INT) |
6370 | 0 | { |
6371 | | // Use unsigned implementation by converting the nodatavalue to |
6372 | | // unsigned |
6373 | 0 | return GDALIsValueInRange<int8_t>(dfNoDataValue) && |
6374 | 0 | HasOnlyNoDataT( |
6375 | 0 | static_cast<const uint8_t *>(pBuffer), |
6376 | 0 | static_cast<uint8_t>(static_cast<int8_t>(dfNoDataValue)), |
6377 | 0 | nWidth, nHeight, nLineStride, nComponents); |
6378 | 0 | } |
6379 | 0 | if (nBitsPerSample == 16 && nSampleFormat == GSF_UNSIGNED_INT) |
6380 | 0 | { |
6381 | 0 | return GDALIsValueInRange<uint16_t>(dfNoDataValue) && |
6382 | 0 | HasOnlyNoDataT(static_cast<const uint16_t *>(pBuffer), |
6383 | 0 | static_cast<uint16_t>(dfNoDataValue), nWidth, |
6384 | 0 | nHeight, nLineStride, nComponents); |
6385 | 0 | } |
6386 | 0 | if (nBitsPerSample == 16 && nSampleFormat == GSF_SIGNED_INT) |
6387 | 0 | { |
6388 | | // Use unsigned implementation by converting the nodatavalue to |
6389 | | // unsigned |
6390 | 0 | return GDALIsValueInRange<int16_t>(dfNoDataValue) && |
6391 | 0 | HasOnlyNoDataT( |
6392 | 0 | static_cast<const uint16_t *>(pBuffer), |
6393 | 0 | static_cast<uint16_t>(static_cast<int16_t>(dfNoDataValue)), |
6394 | 0 | nWidth, nHeight, nLineStride, nComponents); |
6395 | 0 | } |
6396 | 0 | if (nBitsPerSample == 32 && nSampleFormat == GSF_UNSIGNED_INT) |
6397 | 0 | { |
6398 | 0 | return GDALIsValueInRange<uint32_t>(dfNoDataValue) && |
6399 | 0 | HasOnlyNoDataT(static_cast<const uint32_t *>(pBuffer), |
6400 | 0 | static_cast<uint32_t>(dfNoDataValue), nWidth, |
6401 | 0 | nHeight, nLineStride, nComponents); |
6402 | 0 | } |
6403 | 0 | if (nBitsPerSample == 32 && nSampleFormat == GSF_SIGNED_INT) |
6404 | 0 | { |
6405 | | // Use unsigned implementation by converting the nodatavalue to |
6406 | | // unsigned |
6407 | 0 | return GDALIsValueInRange<int32_t>(dfNoDataValue) && |
6408 | 0 | HasOnlyNoDataT( |
6409 | 0 | static_cast<const uint32_t *>(pBuffer), |
6410 | 0 | static_cast<uint32_t>(static_cast<int32_t>(dfNoDataValue)), |
6411 | 0 | nWidth, nHeight, nLineStride, nComponents); |
6412 | 0 | } |
6413 | 0 | if (nBitsPerSample == 64 && nSampleFormat == GSF_UNSIGNED_INT) |
6414 | 0 | { |
6415 | 0 | return GDALIsValueInRange<uint64_t>(dfNoDataValue) && |
6416 | 0 | HasOnlyNoDataT(static_cast<const uint64_t *>(pBuffer), |
6417 | 0 | static_cast<uint64_t>(dfNoDataValue), nWidth, |
6418 | 0 | nHeight, nLineStride, nComponents); |
6419 | 0 | } |
6420 | 0 | if (nBitsPerSample == 64 && nSampleFormat == GSF_SIGNED_INT) |
6421 | 0 | { |
6422 | | // Use unsigned implementation by converting the nodatavalue to |
6423 | | // unsigned |
6424 | 0 | return GDALIsValueInRange<int64_t>(dfNoDataValue) && |
6425 | 0 | HasOnlyNoDataT( |
6426 | 0 | static_cast<const uint64_t *>(pBuffer), |
6427 | 0 | static_cast<uint64_t>(static_cast<int64_t>(dfNoDataValue)), |
6428 | 0 | nWidth, nHeight, nLineStride, nComponents); |
6429 | 0 | } |
6430 | 0 | if (nBitsPerSample == 16 && nSampleFormat == GSF_FLOATING_POINT) |
6431 | 0 | { |
6432 | 0 | return (std::isnan(dfNoDataValue) || |
6433 | 0 | GDALIsValueInRange<GFloat16>(dfNoDataValue)) && |
6434 | 0 | HasOnlyNoDataT(static_cast<const GFloat16 *>(pBuffer), |
6435 | 0 | static_cast<GFloat16>(dfNoDataValue), nWidth, |
6436 | 0 | nHeight, nLineStride, nComponents); |
6437 | 0 | } |
6438 | 0 | if (nBitsPerSample == 32 && nSampleFormat == GSF_FLOATING_POINT) |
6439 | 0 | { |
6440 | 0 | return (std::isnan(dfNoDataValue) || |
6441 | 0 | GDALIsValueInRange<float>(dfNoDataValue)) && |
6442 | 0 | HasOnlyNoDataT(static_cast<const float *>(pBuffer), |
6443 | 0 | static_cast<float>(dfNoDataValue), nWidth, |
6444 | 0 | nHeight, nLineStride, nComponents); |
6445 | 0 | } |
6446 | 0 | if (nBitsPerSample == 64 && nSampleFormat == GSF_FLOATING_POINT) |
6447 | 0 | { |
6448 | 0 | return HasOnlyNoDataT(static_cast<const double *>(pBuffer), |
6449 | 0 | dfNoDataValue, nWidth, nHeight, nLineStride, |
6450 | 0 | nComponents); |
6451 | 0 | } |
6452 | 0 | return false; |
6453 | 0 | } |
6454 | | |
6455 | | #ifdef HAVE_SSE2 |
6456 | | |
6457 | | /************************************************************************/ |
6458 | | /* GDALDeinterleave3Byte() */ |
6459 | | /************************************************************************/ |
6460 | | |
6461 | | #if defined(__GNUC__) && !defined(__clang__) |
6462 | | __attribute__((optimize("no-tree-vectorize"))) |
6463 | | #endif |
6464 | | static void GDALDeinterleave3Byte(const GByte *CPL_RESTRICT pabySrc, |
6465 | | GByte *CPL_RESTRICT pabyDest0, |
6466 | | GByte *CPL_RESTRICT pabyDest1, |
6467 | | GByte *CPL_RESTRICT pabyDest2, size_t nIters) |
6468 | | #ifdef USE_NEON_OPTIMIZATIONS |
6469 | | { |
6470 | | return GDALDeinterleave3Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, pabyDest2, |
6471 | | nIters); |
6472 | | } |
6473 | | #else |
6474 | 0 | { |
6475 | 0 | #ifdef HAVE_SSSE3_AT_COMPILE_TIME |
6476 | 0 | if (CPLHaveRuntimeSSSE3()) |
6477 | 0 | { |
6478 | 0 | return GDALDeinterleave3Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, |
6479 | 0 | pabyDest2, nIters); |
6480 | 0 | } |
6481 | 0 | #endif |
6482 | | |
6483 | 0 | size_t i = 0; |
6484 | 0 | if (((reinterpret_cast<uintptr_t>(pabySrc) | |
6485 | 0 | reinterpret_cast<uintptr_t>(pabyDest0) | |
6486 | 0 | reinterpret_cast<uintptr_t>(pabyDest1) | |
6487 | 0 | reinterpret_cast<uintptr_t>(pabyDest2)) % |
6488 | 0 | sizeof(unsigned int)) == 0) |
6489 | 0 | { |
6490 | | // Slightly better than GCC autovectorizer |
6491 | 0 | for (size_t j = 0; i + 3 < nIters; i += 4, ++j) |
6492 | 0 | { |
6493 | 0 | unsigned int word0 = |
6494 | 0 | *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i); |
6495 | 0 | unsigned int word1 = |
6496 | 0 | *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i + 4); |
6497 | 0 | unsigned int word2 = |
6498 | 0 | *reinterpret_cast<const unsigned int *>(pabySrc + 3 * i + 8); |
6499 | 0 | reinterpret_cast<unsigned int *>(pabyDest0)[j] = |
6500 | 0 | (word0 & 0xff) | ((word0 >> 24) << 8) | (word1 & 0x00ff0000) | |
6501 | 0 | ((word2 >> 8) << 24); |
6502 | 0 | reinterpret_cast<unsigned int *>(pabyDest1)[j] = |
6503 | 0 | ((word0 >> 8) & 0xff) | ((word1 & 0xff) << 8) | |
6504 | 0 | (((word1 >> 24)) << 16) | ((word2 >> 16) << 24); |
6505 | 0 | pabyDest2[j * 4] = static_cast<GByte>(word0 >> 16); |
6506 | 0 | pabyDest2[j * 4 + 1] = static_cast<GByte>(word1 >> 8); |
6507 | 0 | pabyDest2[j * 4 + 2] = static_cast<GByte>(word2); |
6508 | 0 | pabyDest2[j * 4 + 3] = static_cast<GByte>(word2 >> 24); |
6509 | 0 | } |
6510 | 0 | } |
6511 | 0 | #if defined(__clang__) |
6512 | 0 | #pragma clang loop vectorize(disable) |
6513 | 0 | #endif |
6514 | 0 | for (; i < nIters; ++i) |
6515 | 0 | { |
6516 | 0 | pabyDest0[i] = pabySrc[3 * i + 0]; |
6517 | 0 | pabyDest1[i] = pabySrc[3 * i + 1]; |
6518 | 0 | pabyDest2[i] = pabySrc[3 * i + 2]; |
6519 | 0 | } |
6520 | 0 | } |
6521 | | #endif |
6522 | | |
6523 | | /************************************************************************/ |
6524 | | /* GDALDeinterleave4Byte() */ |
6525 | | /************************************************************************/ |
6526 | | |
6527 | | #if !defined(__GNUC__) || defined(__clang__) |
6528 | | |
6529 | | /************************************************************************/ |
6530 | | /* deinterleave() */ |
6531 | | /************************************************************************/ |
6532 | | |
6533 | | template <bool SHIFT, bool MASK> |
6534 | | inline __m128i deinterleave(__m128i &xmm0_ori, __m128i &xmm1_ori, |
6535 | | __m128i &xmm2_ori, __m128i &xmm3_ori) |
6536 | 0 | { |
6537 | | // Set higher 24bit of each int32 packed word to 0 |
6538 | 0 | if (SHIFT) |
6539 | 0 | { |
6540 | 0 | xmm0_ori = _mm_srli_epi32(xmm0_ori, 8); |
6541 | 0 | xmm1_ori = _mm_srli_epi32(xmm1_ori, 8); |
6542 | 0 | xmm2_ori = _mm_srli_epi32(xmm2_ori, 8); |
6543 | 0 | xmm3_ori = _mm_srli_epi32(xmm3_ori, 8); |
6544 | 0 | } |
6545 | 0 | __m128i xmm0; |
6546 | 0 | __m128i xmm1; |
6547 | 0 | __m128i xmm2; |
6548 | 0 | __m128i xmm3; |
6549 | 0 | if (MASK) |
6550 | 0 | { |
6551 | 0 | const __m128i xmm_mask = _mm_set1_epi32(0xff); |
6552 | 0 | xmm0 = _mm_and_si128(xmm0_ori, xmm_mask); |
6553 | 0 | xmm1 = _mm_and_si128(xmm1_ori, xmm_mask); |
6554 | 0 | xmm2 = _mm_and_si128(xmm2_ori, xmm_mask); |
6555 | 0 | xmm3 = _mm_and_si128(xmm3_ori, xmm_mask); |
6556 | 0 | } |
6557 | 0 | else |
6558 | 0 | { |
6559 | 0 | xmm0 = xmm0_ori; |
6560 | 0 | xmm1 = xmm1_ori; |
6561 | 0 | xmm2 = xmm2_ori; |
6562 | 0 | xmm3 = xmm3_ori; |
6563 | 0 | } |
6564 | | // Pack int32 to int16 |
6565 | 0 | xmm0 = _mm_packs_epi32(xmm0, xmm1); |
6566 | 0 | xmm2 = _mm_packs_epi32(xmm2, xmm3); |
6567 | | // Pack int16 to uint8 |
6568 | 0 | xmm0 = _mm_packus_epi16(xmm0, xmm2); |
6569 | 0 | return xmm0; |
6570 | 0 | } Unexecuted instantiation: long long __vector(2) deinterleave<false, true>(long long __vector(2)&, long long __vector(2)&, long long __vector(2)&, long long __vector(2)&) Unexecuted instantiation: long long __vector(2) deinterleave<true, true>(long long __vector(2)&, long long __vector(2)&, long long __vector(2)&, long long __vector(2)&) Unexecuted instantiation: long long __vector(2) deinterleave<true, false>(long long __vector(2)&, long long __vector(2)&, long long __vector(2)&, long long __vector(2)&) |
6571 | | |
6572 | | static void GDALDeinterleave4Byte(const GByte *CPL_RESTRICT pabySrc, |
6573 | | GByte *CPL_RESTRICT pabyDest0, |
6574 | | GByte *CPL_RESTRICT pabyDest1, |
6575 | | GByte *CPL_RESTRICT pabyDest2, |
6576 | | GByte *CPL_RESTRICT pabyDest3, size_t nIters) |
6577 | | #ifdef USE_NEON_OPTIMIZATIONS |
6578 | | { |
6579 | | return GDALDeinterleave4Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, pabyDest2, |
6580 | | pabyDest3, nIters); |
6581 | | } |
6582 | | #else |
6583 | 0 | { |
6584 | 0 | #ifdef HAVE_SSSE3_AT_COMPILE_TIME |
6585 | 0 | if (CPLHaveRuntimeSSSE3()) |
6586 | 0 | { |
6587 | 0 | return GDALDeinterleave4Byte_SSSE3(pabySrc, pabyDest0, pabyDest1, |
6588 | 0 | pabyDest2, pabyDest3, nIters); |
6589 | 0 | } |
6590 | 0 | #endif |
6591 | | |
6592 | | // Not the optimal SSE2-only code, as gcc auto-vectorizer manages to |
6593 | | // do something slightly better. |
6594 | 0 | size_t i = 0; |
6595 | 0 | for (; i + 15 < nIters; i += 16) |
6596 | 0 | { |
6597 | 0 | __m128i xmm0_ori = _mm_loadu_si128( |
6598 | 0 | reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 0)); |
6599 | 0 | __m128i xmm1_ori = _mm_loadu_si128( |
6600 | 0 | reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 16)); |
6601 | 0 | __m128i xmm2_ori = _mm_loadu_si128( |
6602 | 0 | reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 32)); |
6603 | 0 | __m128i xmm3_ori = _mm_loadu_si128( |
6604 | 0 | reinterpret_cast<__m128i const *>(pabySrc + 4 * i + 48)); |
6605 | |
|
6606 | 0 | _mm_storeu_si128( |
6607 | 0 | reinterpret_cast<__m128i *>(pabyDest0 + i), |
6608 | 0 | deinterleave<false, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori)); |
6609 | 0 | _mm_storeu_si128( |
6610 | 0 | reinterpret_cast<__m128i *>(pabyDest1 + i), |
6611 | 0 | deinterleave<true, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori)); |
6612 | 0 | _mm_storeu_si128( |
6613 | 0 | reinterpret_cast<__m128i *>(pabyDest2 + i), |
6614 | 0 | deinterleave<true, true>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori)); |
6615 | 0 | _mm_storeu_si128( |
6616 | 0 | reinterpret_cast<__m128i *>(pabyDest3 + i), |
6617 | 0 | deinterleave<true, false>(xmm0_ori, xmm1_ori, xmm2_ori, xmm3_ori)); |
6618 | 0 | } |
6619 | |
|
6620 | 0 | #if defined(__clang__) |
6621 | 0 | #pragma clang loop vectorize(disable) |
6622 | 0 | #endif |
6623 | 0 | for (; i < nIters; ++i) |
6624 | 0 | { |
6625 | 0 | pabyDest0[i] = pabySrc[4 * i + 0]; |
6626 | 0 | pabyDest1[i] = pabySrc[4 * i + 1]; |
6627 | 0 | pabyDest2[i] = pabySrc[4 * i + 2]; |
6628 | 0 | pabyDest3[i] = pabySrc[4 * i + 3]; |
6629 | 0 | } |
6630 | 0 | } |
6631 | | #endif |
6632 | | #else |
6633 | | // GCC autovectorizer does an excellent job |
6634 | | __attribute__((optimize("tree-vectorize"))) static void GDALDeinterleave4Byte( |
6635 | | const GByte *CPL_RESTRICT pabySrc, GByte *CPL_RESTRICT pabyDest0, |
6636 | | GByte *CPL_RESTRICT pabyDest1, GByte *CPL_RESTRICT pabyDest2, |
6637 | | GByte *CPL_RESTRICT pabyDest3, size_t nIters) |
6638 | | { |
6639 | | for (size_t i = 0; i < nIters; ++i) |
6640 | | { |
6641 | | pabyDest0[i] = pabySrc[4 * i + 0]; |
6642 | | pabyDest1[i] = pabySrc[4 * i + 1]; |
6643 | | pabyDest2[i] = pabySrc[4 * i + 2]; |
6644 | | pabyDest3[i] = pabySrc[4 * i + 3]; |
6645 | | } |
6646 | | } |
6647 | | #endif |
6648 | | |
6649 | | #else |
6650 | | |
6651 | | /************************************************************************/ |
6652 | | /* GDALDeinterleave3Byte() */ |
6653 | | /************************************************************************/ |
6654 | | |
6655 | | // TODO: Enabling below could help on non-Intel architectures where GCC knows |
6656 | | // how to auto-vectorize |
6657 | | // #if defined(__GNUC__) |
6658 | | //__attribute__((optimize("tree-vectorize"))) |
6659 | | // #endif |
6660 | | static void GDALDeinterleave3Byte(const GByte *CPL_RESTRICT pabySrc, |
6661 | | GByte *CPL_RESTRICT pabyDest0, |
6662 | | GByte *CPL_RESTRICT pabyDest1, |
6663 | | GByte *CPL_RESTRICT pabyDest2, size_t nIters) |
6664 | | { |
6665 | | for (size_t i = 0; i < nIters; ++i) |
6666 | | { |
6667 | | pabyDest0[i] = pabySrc[3 * i + 0]; |
6668 | | pabyDest1[i] = pabySrc[3 * i + 1]; |
6669 | | pabyDest2[i] = pabySrc[3 * i + 2]; |
6670 | | } |
6671 | | } |
6672 | | |
6673 | | /************************************************************************/ |
6674 | | /* GDALDeinterleave4Byte() */ |
6675 | | /************************************************************************/ |
6676 | | |
6677 | | // TODO: Enabling below could help on non-Intel architectures where gcc knows |
6678 | | // how to auto-vectorize |
6679 | | // #if defined(__GNUC__) |
6680 | | //__attribute__((optimize("tree-vectorize"))) |
6681 | | // #endif |
6682 | | static void GDALDeinterleave4Byte(const GByte *CPL_RESTRICT pabySrc, |
6683 | | GByte *CPL_RESTRICT pabyDest0, |
6684 | | GByte *CPL_RESTRICT pabyDest1, |
6685 | | GByte *CPL_RESTRICT pabyDest2, |
6686 | | GByte *CPL_RESTRICT pabyDest3, size_t nIters) |
6687 | | { |
6688 | | for (size_t i = 0; i < nIters; ++i) |
6689 | | { |
6690 | | pabyDest0[i] = pabySrc[4 * i + 0]; |
6691 | | pabyDest1[i] = pabySrc[4 * i + 1]; |
6692 | | pabyDest2[i] = pabySrc[4 * i + 2]; |
6693 | | pabyDest3[i] = pabySrc[4 * i + 3]; |
6694 | | } |
6695 | | } |
6696 | | |
6697 | | #endif |
6698 | | |
6699 | | /************************************************************************/ |
6700 | | /* GDALDeinterleave() */ |
6701 | | /************************************************************************/ |
6702 | | |
6703 | | /*! Copy values from a pixel-interleave buffer to multiple per-component |
6704 | | buffers. |
6705 | | |
6706 | | In pseudo-code |
6707 | | \verbatim |
6708 | | for(size_t i = 0; i < nIters; ++i) |
6709 | | for(int iComp = 0; iComp < nComponents; iComp++ ) |
6710 | | ppDestBuffer[iComp][i] = pSourceBuffer[nComponents * i + iComp] |
6711 | | \endverbatim |
6712 | | |
6713 | | The implementation is optimized for a few cases, like de-interleaving |
6714 | | of 3 or 4-components Byte buffers. |
6715 | | |
6716 | | \since GDAL 3.6 |
6717 | | */ |
6718 | | void GDALDeinterleave(const void *pSourceBuffer, GDALDataType eSourceDT, |
6719 | | int nComponents, void **ppDestBuffer, |
6720 | | GDALDataType eDestDT, size_t nIters) |
6721 | 0 | { |
6722 | 0 | if (eSourceDT == eDestDT) |
6723 | 0 | { |
6724 | 0 | if (eSourceDT == GDT_UInt8 || eSourceDT == GDT_Int8) |
6725 | 0 | { |
6726 | 0 | if (nComponents == 3) |
6727 | 0 | { |
6728 | 0 | const GByte *CPL_RESTRICT pabySrc = |
6729 | 0 | static_cast<const GByte *>(pSourceBuffer); |
6730 | 0 | GByte *CPL_RESTRICT pabyDest0 = |
6731 | 0 | static_cast<GByte *>(ppDestBuffer[0]); |
6732 | 0 | GByte *CPL_RESTRICT pabyDest1 = |
6733 | 0 | static_cast<GByte *>(ppDestBuffer[1]); |
6734 | 0 | GByte *CPL_RESTRICT pabyDest2 = |
6735 | 0 | static_cast<GByte *>(ppDestBuffer[2]); |
6736 | 0 | GDALDeinterleave3Byte(pabySrc, pabyDest0, pabyDest1, pabyDest2, |
6737 | 0 | nIters); |
6738 | 0 | return; |
6739 | 0 | } |
6740 | 0 | else if (nComponents == 4) |
6741 | 0 | { |
6742 | 0 | const GByte *CPL_RESTRICT pabySrc = |
6743 | 0 | static_cast<const GByte *>(pSourceBuffer); |
6744 | 0 | GByte *CPL_RESTRICT pabyDest0 = |
6745 | 0 | static_cast<GByte *>(ppDestBuffer[0]); |
6746 | 0 | GByte *CPL_RESTRICT pabyDest1 = |
6747 | 0 | static_cast<GByte *>(ppDestBuffer[1]); |
6748 | 0 | GByte *CPL_RESTRICT pabyDest2 = |
6749 | 0 | static_cast<GByte *>(ppDestBuffer[2]); |
6750 | 0 | GByte *CPL_RESTRICT pabyDest3 = |
6751 | 0 | static_cast<GByte *>(ppDestBuffer[3]); |
6752 | 0 | GDALDeinterleave4Byte(pabySrc, pabyDest0, pabyDest1, pabyDest2, |
6753 | 0 | pabyDest3, nIters); |
6754 | 0 | return; |
6755 | 0 | } |
6756 | 0 | } |
6757 | | #if ((defined(__GNUC__) && !defined(__clang__)) || \ |
6758 | | defined(__INTEL_CLANG_COMPILER)) && \ |
6759 | | defined(HAVE_SSE2) && defined(HAVE_SSSE3_AT_COMPILE_TIME) |
6760 | | else if ((eSourceDT == GDT_Int16 || eSourceDT == GDT_UInt16) && |
6761 | | CPLHaveRuntimeSSSE3()) |
6762 | | { |
6763 | | if (nComponents == 3) |
6764 | | { |
6765 | | const GUInt16 *CPL_RESTRICT panSrc = |
6766 | | static_cast<const GUInt16 *>(pSourceBuffer); |
6767 | | GUInt16 *CPL_RESTRICT panDest0 = |
6768 | | static_cast<GUInt16 *>(ppDestBuffer[0]); |
6769 | | GUInt16 *CPL_RESTRICT panDest1 = |
6770 | | static_cast<GUInt16 *>(ppDestBuffer[1]); |
6771 | | GUInt16 *CPL_RESTRICT panDest2 = |
6772 | | static_cast<GUInt16 *>(ppDestBuffer[2]); |
6773 | | GDALDeinterleave3UInt16_SSSE3(panSrc, panDest0, panDest1, |
6774 | | panDest2, nIters); |
6775 | | return; |
6776 | | } |
6777 | | #if !defined(__INTEL_CLANG_COMPILER) |
6778 | | // ICC autovectorizer doesn't do a good job, at least with icx |
6779 | | // 2022.1.0.20220316 |
6780 | | else if (nComponents == 4) |
6781 | | { |
6782 | | const GUInt16 *CPL_RESTRICT panSrc = |
6783 | | static_cast<const GUInt16 *>(pSourceBuffer); |
6784 | | GUInt16 *CPL_RESTRICT panDest0 = |
6785 | | static_cast<GUInt16 *>(ppDestBuffer[0]); |
6786 | | GUInt16 *CPL_RESTRICT panDest1 = |
6787 | | static_cast<GUInt16 *>(ppDestBuffer[1]); |
6788 | | GUInt16 *CPL_RESTRICT panDest2 = |
6789 | | static_cast<GUInt16 *>(ppDestBuffer[2]); |
6790 | | GUInt16 *CPL_RESTRICT panDest3 = |
6791 | | static_cast<GUInt16 *>(ppDestBuffer[3]); |
6792 | | GDALDeinterleave4UInt16_SSSE3(panSrc, panDest0, panDest1, |
6793 | | panDest2, panDest3, nIters); |
6794 | | return; |
6795 | | } |
6796 | | #endif |
6797 | | } |
6798 | | #endif |
6799 | 0 | } |
6800 | | |
6801 | 0 | const int nSourceDTSize = GDALGetDataTypeSizeBytes(eSourceDT); |
6802 | 0 | const int nDestDTSize = GDALGetDataTypeSizeBytes(eDestDT); |
6803 | 0 | for (int iComp = 0; iComp < nComponents; iComp++) |
6804 | 0 | { |
6805 | 0 | GDALCopyWords64(static_cast<const GByte *>(pSourceBuffer) + |
6806 | 0 | iComp * nSourceDTSize, |
6807 | 0 | eSourceDT, nComponents * nSourceDTSize, |
6808 | 0 | ppDestBuffer[iComp], eDestDT, nDestDTSize, nIters); |
6809 | 0 | } |
6810 | 0 | } |
6811 | | |
6812 | | /************************************************************************/ |
6813 | | /* GDALTranspose2DSingleToSingle() */ |
6814 | | /************************************************************************/ |
6815 | | /** |
6816 | | * Transpose a 2D array of non-complex values, in a efficient (cache-oblivious) way. |
6817 | | * |
6818 | | * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth. |
6819 | | * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight. |
6820 | | * @param nSrcWidth Width of pSrc array. |
6821 | | * @param nSrcHeight Height of pSrc array. |
6822 | | */ |
6823 | | |
6824 | | template <class DST, class SRC> |
6825 | | void GDALTranspose2DSingleToSingle(const SRC *CPL_RESTRICT pSrc, |
6826 | | DST *CPL_RESTRICT pDst, size_t nSrcWidth, |
6827 | | size_t nSrcHeight) |
6828 | 0 | { |
6829 | 0 | constexpr size_t blocksize = 32; |
6830 | 0 | for (size_t i = 0; i < nSrcHeight; i += blocksize) |
6831 | 0 | { |
6832 | 0 | const size_t max_k = std::min(i + blocksize, nSrcHeight); |
6833 | 0 | for (size_t j = 0; j < nSrcWidth; j += blocksize) |
6834 | 0 | { |
6835 | | // transpose the block beginning at [i,j] |
6836 | 0 | const size_t max_l = std::min(j + blocksize, nSrcWidth); |
6837 | 0 | for (size_t k = i; k < max_k; ++k) |
6838 | 0 | { |
6839 | 0 | for (size_t l = j; l < max_l; ++l) |
6840 | 0 | { |
6841 | 0 | GDALCopyWord(pSrc[l + k * nSrcWidth], |
6842 | 0 | pDst[k + l * nSrcHeight]); |
6843 | 0 | } |
6844 | 0 | } |
6845 | 0 | } |
6846 | 0 | } |
6847 | 0 | } Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, unsigned char>(unsigned char const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, signed char>(signed char const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, unsigned short>(unsigned short const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, short>(short const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, unsigned int>(unsigned int const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, int>(int const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, unsigned long>(unsigned long const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, long>(long const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, cpl::Float16>(cpl::Float16 const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, float>(float const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned char, double>(double const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, unsigned char>(unsigned char const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, signed char>(signed char const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, unsigned short>(unsigned short const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, short>(short const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, unsigned int>(unsigned int const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, int>(int const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, unsigned long>(unsigned long const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, long>(long const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, cpl::Float16>(cpl::Float16 const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, float>(float const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<signed char, double>(double const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, unsigned char>(unsigned char const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, signed char>(signed char const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, unsigned short>(unsigned short const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, short>(short const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, unsigned int>(unsigned int const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, int>(int const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, unsigned long>(unsigned long const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, long>(long const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, cpl::Float16>(cpl::Float16 const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, float>(float const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned short, double>(double const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, unsigned char>(unsigned char const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, signed char>(signed char const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, unsigned short>(unsigned short const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, short>(short const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, unsigned int>(unsigned int const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, int>(int const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, unsigned long>(unsigned long const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, long>(long const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, cpl::Float16>(cpl::Float16 const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, float>(float const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<short, double>(double const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, unsigned char>(unsigned char const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, signed char>(signed char const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, unsigned short>(unsigned short const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, short>(short const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, unsigned int>(unsigned int const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, int>(int const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, unsigned long>(unsigned long const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, long>(long const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, cpl::Float16>(cpl::Float16 const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, float>(float const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned int, double>(double const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, unsigned char>(unsigned char const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, signed char>(signed char const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, unsigned short>(unsigned short const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, short>(short const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, unsigned int>(unsigned int const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, int>(int const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, unsigned long>(unsigned long const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, long>(long const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, cpl::Float16>(cpl::Float16 const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, float>(float const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<int, double>(double const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, unsigned char>(unsigned char const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, signed char>(signed char const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, unsigned short>(unsigned short const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, short>(short const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, unsigned int>(unsigned int const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, int>(int const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, unsigned long>(unsigned long const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, long>(long const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, cpl::Float16>(cpl::Float16 const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, float>(float const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<unsigned long, double>(double const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, unsigned char>(unsigned char const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, signed char>(signed char const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, unsigned short>(unsigned short const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, short>(short const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, unsigned int>(unsigned int const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, int>(int const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, unsigned long>(unsigned long const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, long>(long const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, cpl::Float16>(cpl::Float16 const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, float>(float const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<long, double>(double const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, unsigned char>(unsigned char const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, signed char>(signed char const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, unsigned short>(unsigned short const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, short>(short const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, unsigned int>(unsigned int const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, int>(int const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, unsigned long>(unsigned long const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, long>(long const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, cpl::Float16>(cpl::Float16 const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, float>(float const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<cpl::Float16, double>(double const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, unsigned char>(unsigned char const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, signed char>(signed char const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, unsigned short>(unsigned short const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, short>(short const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, unsigned int>(unsigned int const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, int>(int const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, unsigned long>(unsigned long const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, long>(long const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, cpl::Float16>(cpl::Float16 const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, float>(float const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<float, double>(double const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, unsigned char>(unsigned char const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, signed char>(signed char const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, unsigned short>(unsigned short const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, short>(short const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, unsigned int>(unsigned int const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, int>(int const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, unsigned long>(unsigned long const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, long>(long const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, cpl::Float16>(cpl::Float16 const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, float>(float const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToSingle<double, double>(double const*, double*, unsigned long, unsigned long) |
6848 | | |
6849 | | /************************************************************************/ |
6850 | | /* GDALTranspose2DComplexToComplex() */ |
6851 | | /************************************************************************/ |
6852 | | /** |
6853 | | * Transpose a 2D array of complex values into an array of complex values, |
6854 | | * in a efficient (cache-oblivious) way. |
6855 | | * |
6856 | | * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth. |
6857 | | * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight. |
6858 | | * @param nSrcWidth Width of pSrc array. |
6859 | | * @param nSrcHeight Height of pSrc array. |
6860 | | */ |
6861 | | template <class DST, class SRC> |
6862 | | void GDALTranspose2DComplexToComplex(const SRC *CPL_RESTRICT pSrc, |
6863 | | DST *CPL_RESTRICT pDst, size_t nSrcWidth, |
6864 | | size_t nSrcHeight) |
6865 | 0 | { |
6866 | 0 | constexpr size_t blocksize = 32; |
6867 | 0 | for (size_t i = 0; i < nSrcHeight; i += blocksize) |
6868 | 0 | { |
6869 | 0 | const size_t max_k = std::min(i + blocksize, nSrcHeight); |
6870 | 0 | for (size_t j = 0; j < nSrcWidth; j += blocksize) |
6871 | 0 | { |
6872 | | // transpose the block beginning at [i,j] |
6873 | 0 | const size_t max_l = std::min(j + blocksize, nSrcWidth); |
6874 | 0 | for (size_t k = i; k < max_k; ++k) |
6875 | 0 | { |
6876 | 0 | for (size_t l = j; l < max_l; ++l) |
6877 | 0 | { |
6878 | 0 | GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 0], |
6879 | 0 | pDst[2 * (k + l * nSrcHeight) + 0]); |
6880 | 0 | GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 1], |
6881 | 0 | pDst[2 * (k + l * nSrcHeight) + 1]); |
6882 | 0 | } |
6883 | 0 | } |
6884 | 0 | } |
6885 | 0 | } |
6886 | 0 | } Unexecuted instantiation: void GDALTranspose2DComplexToComplex<short, short>(short const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<short, int>(int const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<short, cpl::Float16>(cpl::Float16 const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<short, float>(float const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<short, double>(double const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<int, short>(short const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<int, int>(int const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<int, cpl::Float16>(cpl::Float16 const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<int, float>(float const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<int, double>(double const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<cpl::Float16, short>(short const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<cpl::Float16, int>(int const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<cpl::Float16, cpl::Float16>(cpl::Float16 const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<cpl::Float16, float>(float const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<cpl::Float16, double>(double const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<float, short>(short const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<float, int>(int const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<float, cpl::Float16>(cpl::Float16 const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<float, float>(float const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<float, double>(double const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<double, short>(short const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<double, int>(int const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<double, cpl::Float16>(cpl::Float16 const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<double, float>(float const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToComplex<double, double>(double const*, double*, unsigned long, unsigned long) |
6887 | | |
6888 | | /************************************************************************/ |
6889 | | /* GDALTranspose2DComplexToSingle() */ |
6890 | | /************************************************************************/ |
6891 | | /** |
6892 | | * Transpose a 2D array of complex values into an array of non-complex values, |
6893 | | * in a efficient (cache-oblivious) way. |
6894 | | * |
6895 | | * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth. |
6896 | | * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight. |
6897 | | * @param nSrcWidth Width of pSrc array. |
6898 | | * @param nSrcHeight Height of pSrc array. |
6899 | | */ |
6900 | | template <class DST, class SRC> |
6901 | | void GDALTranspose2DComplexToSingle(const SRC *CPL_RESTRICT pSrc, |
6902 | | DST *CPL_RESTRICT pDst, size_t nSrcWidth, |
6903 | | size_t nSrcHeight) |
6904 | 0 | { |
6905 | 0 | constexpr size_t blocksize = 32; |
6906 | 0 | for (size_t i = 0; i < nSrcHeight; i += blocksize) |
6907 | 0 | { |
6908 | 0 | const size_t max_k = std::min(i + blocksize, nSrcHeight); |
6909 | 0 | for (size_t j = 0; j < nSrcWidth; j += blocksize) |
6910 | 0 | { |
6911 | | // transpose the block beginning at [i,j] |
6912 | 0 | const size_t max_l = std::min(j + blocksize, nSrcWidth); |
6913 | 0 | for (size_t k = i; k < max_k; ++k) |
6914 | 0 | { |
6915 | 0 | for (size_t l = j; l < max_l; ++l) |
6916 | 0 | { |
6917 | 0 | GDALCopyWord(pSrc[2 * (l + k * nSrcWidth) + 0], |
6918 | 0 | pDst[k + l * nSrcHeight]); |
6919 | 0 | } |
6920 | 0 | } |
6921 | 0 | } |
6922 | 0 | } |
6923 | 0 | } Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned char, short>(short const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned char, int>(int const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned char, cpl::Float16>(cpl::Float16 const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned char, float>(float const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned char, double>(double const*, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<signed char, short>(short const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<signed char, int>(int const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<signed char, cpl::Float16>(cpl::Float16 const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<signed char, float>(float const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<signed char, double>(double const*, signed char*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned short, short>(short const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned short, int>(int const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned short, cpl::Float16>(cpl::Float16 const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned short, float>(float const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned short, double>(double const*, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<short, short>(short const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<short, int>(int const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<short, cpl::Float16>(cpl::Float16 const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<short, float>(float const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<short, double>(double const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned int, short>(short const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned int, int>(int const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned int, cpl::Float16>(cpl::Float16 const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned int, float>(float const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned int, double>(double const*, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<int, short>(short const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<int, int>(int const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<int, cpl::Float16>(cpl::Float16 const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<int, float>(float const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<int, double>(double const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned long, short>(short const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned long, int>(int const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned long, cpl::Float16>(cpl::Float16 const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned long, float>(float const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<unsigned long, double>(double const*, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<long, short>(short const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<long, int>(int const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<long, cpl::Float16>(cpl::Float16 const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<long, float>(float const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<long, double>(double const*, long*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<cpl::Float16, short>(short const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<cpl::Float16, int>(int const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<cpl::Float16, cpl::Float16>(cpl::Float16 const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<cpl::Float16, float>(float const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<cpl::Float16, double>(double const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<float, short>(short const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<float, int>(int const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<float, cpl::Float16>(cpl::Float16 const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<float, float>(float const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<float, double>(double const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<double, short>(short const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<double, int>(int const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<double, cpl::Float16>(cpl::Float16 const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<double, float>(float const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DComplexToSingle<double, double>(double const*, double*, unsigned long, unsigned long) |
6924 | | |
6925 | | /************************************************************************/ |
6926 | | /* GDALTranspose2DSingleToComplex() */ |
6927 | | /************************************************************************/ |
6928 | | /** |
6929 | | * Transpose a 2D array of non-complex values into an array of complex values, |
6930 | | * in a efficient (cache-oblivious) way. |
6931 | | * |
6932 | | * @param pSrc Source array of height = nSrcHeight and width = nSrcWidth. |
6933 | | * @param pDst Destination transposed array of height = nSrcWidth and width = nSrcHeight. |
6934 | | * @param nSrcWidth Width of pSrc array. |
6935 | | * @param nSrcHeight Height of pSrc array. |
6936 | | */ |
6937 | | template <class DST, class SRC> |
6938 | | void GDALTranspose2DSingleToComplex(const SRC *CPL_RESTRICT pSrc, |
6939 | | DST *CPL_RESTRICT pDst, size_t nSrcWidth, |
6940 | | size_t nSrcHeight) |
6941 | 0 | { |
6942 | 0 | constexpr size_t blocksize = 32; |
6943 | 0 | for (size_t i = 0; i < nSrcHeight; i += blocksize) |
6944 | 0 | { |
6945 | 0 | const size_t max_k = std::min(i + blocksize, nSrcHeight); |
6946 | 0 | for (size_t j = 0; j < nSrcWidth; j += blocksize) |
6947 | 0 | { |
6948 | | // transpose the block beginning at [i,j] |
6949 | 0 | const size_t max_l = std::min(j + blocksize, nSrcWidth); |
6950 | 0 | for (size_t k = i; k < max_k; ++k) |
6951 | 0 | { |
6952 | 0 | for (size_t l = j; l < max_l; ++l) |
6953 | 0 | { |
6954 | 0 | GDALCopyWord(pSrc[l + k * nSrcWidth], |
6955 | 0 | pDst[2 * (k + l * nSrcHeight) + 0]); |
6956 | 0 | pDst[2 * (k + l * nSrcHeight) + 1] = 0; |
6957 | 0 | } |
6958 | 0 | } |
6959 | 0 | } |
6960 | 0 | } |
6961 | 0 | } Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, unsigned char>(unsigned char const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, signed char>(signed char const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, unsigned short>(unsigned short const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, short>(short const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, unsigned int>(unsigned int const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, int>(int const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, unsigned long>(unsigned long const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, long>(long const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, cpl::Float16>(cpl::Float16 const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, float>(float const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<short, double>(double const*, short*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, unsigned char>(unsigned char const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, signed char>(signed char const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, unsigned short>(unsigned short const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, short>(short const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, unsigned int>(unsigned int const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, int>(int const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, unsigned long>(unsigned long const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, long>(long const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, cpl::Float16>(cpl::Float16 const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, float>(float const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<int, double>(double const*, int*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, unsigned char>(unsigned char const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, signed char>(signed char const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, unsigned short>(unsigned short const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, short>(short const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, unsigned int>(unsigned int const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, int>(int const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, unsigned long>(unsigned long const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, long>(long const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, cpl::Float16>(cpl::Float16 const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, float>(float const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<cpl::Float16, double>(double const*, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, unsigned char>(unsigned char const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, signed char>(signed char const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, unsigned short>(unsigned short const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, short>(short const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, unsigned int>(unsigned int const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, int>(int const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, unsigned long>(unsigned long const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, long>(long const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, cpl::Float16>(cpl::Float16 const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, float>(float const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<float, double>(double const*, float*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, unsigned char>(unsigned char const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, signed char>(signed char const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, unsigned short>(unsigned short const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, short>(short const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, unsigned int>(unsigned int const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, int>(int const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, unsigned long>(unsigned long const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, long>(long const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, cpl::Float16>(cpl::Float16 const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, float>(float const*, double*, unsigned long, unsigned long) Unexecuted instantiation: void GDALTranspose2DSingleToComplex<double, double>(double const*, double*, unsigned long, unsigned long) |
6962 | | |
6963 | | /************************************************************************/ |
6964 | | /* GDALTranspose2D() */ |
6965 | | /************************************************************************/ |
6966 | | |
6967 | | template <class DST, bool DST_IS_COMPLEX> |
6968 | | static void GDALTranspose2D(const void *pSrc, GDALDataType eSrcType, DST *pDst, |
6969 | | size_t nSrcWidth, size_t nSrcHeight) |
6970 | 0 | { |
6971 | 0 | #define CALL_GDALTranspose2D_internal(SRC_TYPE) \ |
6972 | 0 | do \ |
6973 | 0 | { \ |
6974 | 0 | if constexpr (DST_IS_COMPLEX) \ |
6975 | 0 | { \ |
6976 | 0 | GDALTranspose2DSingleToComplex( \ |
6977 | 0 | static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \ |
6978 | 0 | nSrcHeight); \ |
6979 | 0 | } \ |
6980 | 0 | else \ |
6981 | 0 | { \ |
6982 | 0 | GDALTranspose2DSingleToSingle(static_cast<const SRC_TYPE *>(pSrc), \ |
6983 | 0 | pDst, nSrcWidth, nSrcHeight); \ |
6984 | 0 | } \ |
6985 | 0 | } while (0) |
6986 | |
|
6987 | 0 | #define CALL_GDALTranspose2DComplex_internal(SRC_TYPE) \ |
6988 | 0 | do \ |
6989 | 0 | { \ |
6990 | 0 | if constexpr (DST_IS_COMPLEX) \ |
6991 | 0 | { \ |
6992 | 0 | GDALTranspose2DComplexToComplex( \ |
6993 | 0 | static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \ |
6994 | 0 | nSrcHeight); \ |
6995 | 0 | } \ |
6996 | 0 | else \ |
6997 | 0 | { \ |
6998 | 0 | GDALTranspose2DComplexToSingle( \ |
6999 | 0 | static_cast<const SRC_TYPE *>(pSrc), pDst, nSrcWidth, \ |
7000 | 0 | nSrcHeight); \ |
7001 | 0 | } \ |
7002 | 0 | } while (0) |
7003 | | |
7004 | | // clang-format off |
7005 | 0 | switch (eSrcType) |
7006 | 0 | { |
7007 | 0 | case GDT_UInt8: CALL_GDALTranspose2D_internal(uint8_t); break; |
7008 | 0 | case GDT_Int8: CALL_GDALTranspose2D_internal(int8_t); break; |
7009 | 0 | case GDT_UInt16: CALL_GDALTranspose2D_internal(uint16_t); break; |
7010 | 0 | case GDT_Int16: CALL_GDALTranspose2D_internal(int16_t); break; |
7011 | 0 | case GDT_UInt32: CALL_GDALTranspose2D_internal(uint32_t); break; |
7012 | 0 | case GDT_Int32: CALL_GDALTranspose2D_internal(int32_t); break; |
7013 | 0 | case GDT_UInt64: CALL_GDALTranspose2D_internal(uint64_t); break; |
7014 | 0 | case GDT_Int64: CALL_GDALTranspose2D_internal(int64_t); break; |
7015 | 0 | case GDT_Float16: CALL_GDALTranspose2D_internal(GFloat16); break; |
7016 | 0 | case GDT_Float32: CALL_GDALTranspose2D_internal(float); break; |
7017 | 0 | case GDT_Float64: CALL_GDALTranspose2D_internal(double); break; |
7018 | 0 | case GDT_CInt16: CALL_GDALTranspose2DComplex_internal(int16_t); break; |
7019 | 0 | case GDT_CInt32: CALL_GDALTranspose2DComplex_internal(int32_t); break; |
7020 | 0 | case GDT_CFloat16: CALL_GDALTranspose2DComplex_internal(GFloat16); break; |
7021 | 0 | case GDT_CFloat32: CALL_GDALTranspose2DComplex_internal(float); break; |
7022 | 0 | case GDT_CFloat64: CALL_GDALTranspose2DComplex_internal(double); break; |
7023 | 0 | case GDT_Unknown: |
7024 | 0 | case GDT_TypeCount: |
7025 | 0 | break; |
7026 | 0 | } |
7027 | | // clang-format on |
7028 | |
|
7029 | 0 | #undef CALL_GDALTranspose2D_internal |
7030 | 0 | #undef CALL_GDALTranspose2DComplex_internal |
7031 | 0 | } Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<unsigned char, false>(void const*, GDALDataType, unsigned char*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<signed char, false>(void const*, GDALDataType, signed char*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<unsigned short, false>(void const*, GDALDataType, unsigned short*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<short, false>(void const*, GDALDataType, short*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<unsigned int, false>(void const*, GDALDataType, unsigned int*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<int, false>(void const*, GDALDataType, int*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<unsigned long, false>(void const*, GDALDataType, unsigned long*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<long, false>(void const*, GDALDataType, long*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<cpl::Float16, false>(void const*, GDALDataType, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<float, false>(void const*, GDALDataType, float*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<double, false>(void const*, GDALDataType, double*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<short, true>(void const*, GDALDataType, short*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<int, true>(void const*, GDALDataType, int*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<cpl::Float16, true>(void const*, GDALDataType, cpl::Float16*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<float, true>(void const*, GDALDataType, float*, unsigned long, unsigned long) Unexecuted instantiation: rasterio.cpp:void GDALTranspose2D<double, true>(void const*, GDALDataType, double*, unsigned long, unsigned long) |
7032 | | |
7033 | | /************************************************************************/ |
7034 | | /* GDALInterleave2Byte() */ |
7035 | | /************************************************************************/ |
7036 | | |
7037 | | #if defined(HAVE_SSE2) && \ |
7038 | | (!defined(__GNUC__) || defined(__INTEL_CLANG_COMPILER)) |
7039 | | |
7040 | | // ICC autovectorizer doesn't do a good job at generating good SSE code, |
7041 | | // at least with icx 2024.0.2.20231213, but it nicely unrolls the below loop. |
7042 | | #if defined(__GNUC__) |
7043 | | __attribute__((noinline)) |
7044 | | #endif |
7045 | | static void GDALInterleave2Byte(const uint8_t *CPL_RESTRICT pSrc, |
7046 | | uint8_t *CPL_RESTRICT pDst, size_t nIters) |
7047 | | { |
7048 | | size_t i = 0; |
7049 | | constexpr size_t VALS_PER_ITER = 16; |
7050 | | for (i = 0; i + VALS_PER_ITER <= nIters; i += VALS_PER_ITER) |
7051 | | { |
7052 | | __m128i xmm0 = |
7053 | | _mm_loadu_si128(reinterpret_cast<__m128i const *>(pSrc + i)); |
7054 | | __m128i xmm1 = _mm_loadu_si128( |
7055 | | reinterpret_cast<__m128i const *>(pSrc + i + nIters)); |
7056 | | _mm_storeu_si128(reinterpret_cast<__m128i *>(pDst + 2 * i), |
7057 | | _mm_unpacklo_epi8(xmm0, xmm1)); |
7058 | | _mm_storeu_si128( |
7059 | | reinterpret_cast<__m128i *>(pDst + 2 * i + VALS_PER_ITER), |
7060 | | _mm_unpackhi_epi8(xmm0, xmm1)); |
7061 | | } |
7062 | | #if defined(__clang__) |
7063 | | #pragma clang loop vectorize(disable) |
7064 | | #endif |
7065 | | for (; i < nIters; ++i) |
7066 | | { |
7067 | | pDst[2 * i + 0] = pSrc[i + 0 * nIters]; |
7068 | | pDst[2 * i + 1] = pSrc[i + 1 * nIters]; |
7069 | | } |
7070 | | } |
7071 | | |
7072 | | #else |
7073 | | |
7074 | | #if defined(__GNUC__) && !defined(__clang__) |
7075 | | __attribute__((optimize("tree-vectorize"))) |
7076 | | #endif |
7077 | | #if defined(__GNUC__) |
7078 | | __attribute__((noinline)) |
7079 | | #endif |
7080 | | #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER) |
7081 | | // clang++ -O2 -fsanitize=undefined fails to vectorize, ignore that warning |
7082 | | #pragma clang diagnostic push |
7083 | | #pragma clang diagnostic ignored "-Wpass-failed" |
7084 | | #endif |
7085 | | static void GDALInterleave2Byte(const uint8_t *CPL_RESTRICT pSrc, |
7086 | | uint8_t *CPL_RESTRICT pDst, size_t nIters) |
7087 | 0 | { |
7088 | 0 | #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER) |
7089 | 0 | #pragma clang loop vectorize(enable) |
7090 | 0 | #endif |
7091 | 0 | for (size_t i = 0; i < nIters; ++i) |
7092 | 0 | { |
7093 | 0 | pDst[2 * i + 0] = pSrc[i + 0 * nIters]; |
7094 | 0 | pDst[2 * i + 1] = pSrc[i + 1 * nIters]; |
7095 | 0 | } |
7096 | 0 | } |
7097 | | #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER) |
7098 | | #pragma clang diagnostic pop |
7099 | | #endif |
7100 | | |
7101 | | #endif |
7102 | | |
7103 | | /************************************************************************/ |
7104 | | /* GDALInterleave4Byte() */ |
7105 | | /************************************************************************/ |
7106 | | |
7107 | | #if defined(HAVE_SSE2) && \ |
7108 | | (!defined(__GNUC__) || defined(__INTEL_CLANG_COMPILER)) |
7109 | | |
7110 | | // ICC autovectorizer doesn't do a good job at generating good SSE code, |
7111 | | // at least with icx 2024.0.2.20231213, but it nicely unrolls the below loop. |
7112 | | #if defined(__GNUC__) |
7113 | | __attribute__((noinline)) |
7114 | | #endif |
7115 | | static void GDALInterleave4Byte(const uint8_t *CPL_RESTRICT pSrc, |
7116 | | uint8_t *CPL_RESTRICT pDst, size_t nIters) |
7117 | | { |
7118 | | size_t i = 0; |
7119 | | constexpr size_t VALS_PER_ITER = 16; |
7120 | | for (i = 0; i + VALS_PER_ITER <= nIters; i += VALS_PER_ITER) |
7121 | | { |
7122 | | __m128i xmm0 = _mm_loadu_si128( |
7123 | | reinterpret_cast<__m128i const *>(pSrc + i + 0 * nIters)); |
7124 | | __m128i xmm1 = _mm_loadu_si128( |
7125 | | reinterpret_cast<__m128i const *>(pSrc + i + 1 * nIters)); |
7126 | | __m128i xmm2 = _mm_loadu_si128( |
7127 | | reinterpret_cast<__m128i const *>(pSrc + i + 2 * nIters)); |
7128 | | __m128i xmm3 = _mm_loadu_si128( |
7129 | | reinterpret_cast<__m128i const *>(pSrc + i + 3 * nIters)); |
7130 | | auto tmp0 = _mm_unpacklo_epi8( |
7131 | | xmm0, |
7132 | | xmm1); // (xmm0_0, xmm1_0, xmm0_1, xmm1_1, xmm0_2, xmm1_2, ...) |
7133 | | auto tmp1 = _mm_unpackhi_epi8( |
7134 | | xmm0, |
7135 | | xmm1); // (xmm0_8, xmm1_8, xmm0_9, xmm1_9, xmm0_10, xmm1_10, ...) |
7136 | | auto tmp2 = _mm_unpacklo_epi8( |
7137 | | xmm2, |
7138 | | xmm3); // (xmm2_0, xmm3_0, xmm2_1, xmm3_1, xmm2_2, xmm3_2, ...) |
7139 | | auto tmp3 = _mm_unpackhi_epi8( |
7140 | | xmm2, |
7141 | | xmm3); // (xmm2_8, xmm3_8, xmm2_9, xmm3_9, xmm2_10, xmm3_10, ...) |
7142 | | auto tmp2_0 = _mm_unpacklo_epi16( |
7143 | | tmp0, |
7144 | | tmp2); // (xmm0_0, xmm1_0, xmm2_0, xmm3_0, xmm0_1, xmm1_1, xmm2_1, xmm3_1, ...) |
7145 | | auto tmp2_1 = _mm_unpackhi_epi16(tmp0, tmp2); |
7146 | | auto tmp2_2 = _mm_unpacklo_epi16(tmp1, tmp3); |
7147 | | auto tmp2_3 = _mm_unpackhi_epi16(tmp1, tmp3); |
7148 | | _mm_storeu_si128( |
7149 | | reinterpret_cast<__m128i *>(pDst + 4 * i + 0 * VALS_PER_ITER), |
7150 | | tmp2_0); |
7151 | | _mm_storeu_si128( |
7152 | | reinterpret_cast<__m128i *>(pDst + 4 * i + 1 * VALS_PER_ITER), |
7153 | | tmp2_1); |
7154 | | _mm_storeu_si128( |
7155 | | reinterpret_cast<__m128i *>(pDst + 4 * i + 2 * VALS_PER_ITER), |
7156 | | tmp2_2); |
7157 | | _mm_storeu_si128( |
7158 | | reinterpret_cast<__m128i *>(pDst + 4 * i + 3 * VALS_PER_ITER), |
7159 | | tmp2_3); |
7160 | | } |
7161 | | #if defined(__clang__) |
7162 | | #pragma clang loop vectorize(disable) |
7163 | | #endif |
7164 | | for (; i < nIters; ++i) |
7165 | | { |
7166 | | pDst[4 * i + 0] = pSrc[i + 0 * nIters]; |
7167 | | pDst[4 * i + 1] = pSrc[i + 1 * nIters]; |
7168 | | pDst[4 * i + 2] = pSrc[i + 2 * nIters]; |
7169 | | pDst[4 * i + 3] = pSrc[i + 3 * nIters]; |
7170 | | } |
7171 | | } |
7172 | | |
7173 | | #else |
7174 | | |
7175 | | #if defined(__GNUC__) && !defined(__clang__) |
7176 | | __attribute__((optimize("tree-vectorize"))) |
7177 | | #endif |
7178 | | #if defined(__GNUC__) |
7179 | | __attribute__((noinline)) |
7180 | | #endif |
7181 | | #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER) |
7182 | | // clang++ -O2 -fsanitize=undefined fails to vectorize, ignore that warning |
7183 | | #pragma clang diagnostic push |
7184 | | #pragma clang diagnostic ignored "-Wpass-failed" |
7185 | | #endif |
7186 | | static void GDALInterleave4Byte(const uint8_t *CPL_RESTRICT pSrc, |
7187 | | uint8_t *CPL_RESTRICT pDst, size_t nIters) |
7188 | 0 | { |
7189 | 0 | #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER) |
7190 | 0 | #pragma clang loop vectorize(enable) |
7191 | 0 | #endif |
7192 | 0 | for (size_t i = 0; i < nIters; ++i) |
7193 | 0 | { |
7194 | 0 | pDst[4 * i + 0] = pSrc[i + 0 * nIters]; |
7195 | 0 | pDst[4 * i + 1] = pSrc[i + 1 * nIters]; |
7196 | 0 | pDst[4 * i + 2] = pSrc[i + 2 * nIters]; |
7197 | 0 | pDst[4 * i + 3] = pSrc[i + 3 * nIters]; |
7198 | 0 | } |
7199 | 0 | } |
7200 | | #if defined(__clang__) && !defined(__INTEL_CLANG_COMPILER) |
7201 | | #pragma clang diagnostic pop |
7202 | | #endif |
7203 | | |
7204 | | #endif |
7205 | | |
7206 | | /************************************************************************/ |
7207 | | /* GDALTranspose2D() */ |
7208 | | /************************************************************************/ |
7209 | | |
7210 | | /** |
7211 | | * Transpose a 2D array in a efficient (cache-oblivious) way. |
7212 | | * |
7213 | | * @param pSrc Source array of width = nSrcWidth and height = nSrcHeight. |
7214 | | * @param eSrcType Data type of pSrc. |
7215 | | * @param pDst Destination transposed array of width = nSrcHeight and height = nSrcWidth. |
7216 | | * @param eDstType Data type of pDst. |
7217 | | * @param nSrcWidth Width of pSrc array. |
7218 | | * @param nSrcHeight Height of pSrc array. |
7219 | | * @since GDAL 3.11 |
7220 | | */ |
7221 | | |
7222 | | void GDALTranspose2D(const void *pSrc, GDALDataType eSrcType, void *pDst, |
7223 | | GDALDataType eDstType, size_t nSrcWidth, size_t nSrcHeight) |
7224 | 0 | { |
7225 | 0 | if (eSrcType == eDstType && (eSrcType == GDT_UInt8 || eSrcType == GDT_Int8)) |
7226 | 0 | { |
7227 | 0 | if (nSrcHeight == 2) |
7228 | 0 | { |
7229 | 0 | GDALInterleave2Byte(static_cast<const uint8_t *>(pSrc), |
7230 | 0 | static_cast<uint8_t *>(pDst), nSrcWidth); |
7231 | 0 | return; |
7232 | 0 | } |
7233 | 0 | if (nSrcHeight == 4) |
7234 | 0 | { |
7235 | 0 | GDALInterleave4Byte(static_cast<const uint8_t *>(pSrc), |
7236 | 0 | static_cast<uint8_t *>(pDst), nSrcWidth); |
7237 | 0 | return; |
7238 | 0 | } |
7239 | 0 | #if (defined(HAVE_SSSE3_AT_COMPILE_TIME) && \ |
7240 | 0 | (defined(__x86_64) || defined(_M_X64))) |
7241 | 0 | if (CPLHaveRuntimeSSSE3()) |
7242 | 0 | { |
7243 | 0 | GDALTranspose2D_Byte_SSSE3(static_cast<const uint8_t *>(pSrc), |
7244 | 0 | static_cast<uint8_t *>(pDst), nSrcWidth, |
7245 | 0 | nSrcHeight); |
7246 | 0 | return; |
7247 | 0 | } |
7248 | | #elif defined(USE_NEON_OPTIMIZATIONS) |
7249 | | { |
7250 | | GDALTranspose2D_Byte_SSSE3(static_cast<const uint8_t *>(pSrc), |
7251 | | static_cast<uint8_t *>(pDst), nSrcWidth, |
7252 | | nSrcHeight); |
7253 | | return; |
7254 | | } |
7255 | | #endif |
7256 | 0 | } |
7257 | | |
7258 | 0 | #define CALL_GDALTranspose2D_internal(DST_TYPE, DST_IS_COMPLEX) \ |
7259 | 0 | GDALTranspose2D<DST_TYPE, DST_IS_COMPLEX>( \ |
7260 | 0 | pSrc, eSrcType, static_cast<DST_TYPE *>(pDst), nSrcWidth, nSrcHeight) |
7261 | | |
7262 | | // clang-format off |
7263 | 0 | switch (eDstType) |
7264 | 0 | { |
7265 | 0 | case GDT_UInt8: CALL_GDALTranspose2D_internal(uint8_t, false); break; |
7266 | 0 | case GDT_Int8: CALL_GDALTranspose2D_internal(int8_t, false); break; |
7267 | 0 | case GDT_UInt16: CALL_GDALTranspose2D_internal(uint16_t, false); break; |
7268 | 0 | case GDT_Int16: CALL_GDALTranspose2D_internal(int16_t, false); break; |
7269 | 0 | case GDT_UInt32: CALL_GDALTranspose2D_internal(uint32_t, false); break; |
7270 | 0 | case GDT_Int32: CALL_GDALTranspose2D_internal(int32_t, false); break; |
7271 | 0 | case GDT_UInt64: CALL_GDALTranspose2D_internal(uint64_t, false); break; |
7272 | 0 | case GDT_Int64: CALL_GDALTranspose2D_internal(int64_t, false); break; |
7273 | 0 | case GDT_Float16: CALL_GDALTranspose2D_internal(GFloat16, false); break; |
7274 | 0 | case GDT_Float32: CALL_GDALTranspose2D_internal(float, false); break; |
7275 | 0 | case GDT_Float64: CALL_GDALTranspose2D_internal(double, false); break; |
7276 | 0 | case GDT_CInt16: CALL_GDALTranspose2D_internal(int16_t, true); break; |
7277 | 0 | case GDT_CInt32: CALL_GDALTranspose2D_internal(int32_t, true); break; |
7278 | 0 | case GDT_CFloat16: CALL_GDALTranspose2D_internal(GFloat16, true); break; |
7279 | 0 | case GDT_CFloat32: CALL_GDALTranspose2D_internal(float, true); break; |
7280 | 0 | case GDT_CFloat64: CALL_GDALTranspose2D_internal(double, true); break; |
7281 | 0 | case GDT_Unknown: |
7282 | 0 | case GDT_TypeCount: |
7283 | 0 | break; |
7284 | 0 | } |
7285 | | // clang-format on |
7286 | |
|
7287 | 0 | #undef CALL_GDALTranspose2D_internal |
7288 | 0 | } |
7289 | | |
7290 | | /************************************************************************/ |
7291 | | /* ExtractBitAndConvertTo255() */ |
7292 | | /************************************************************************/ |
7293 | | |
7294 | | #if defined(__GNUC__) || defined(_MSC_VER) |
7295 | | // Signedness of char implementation dependent, so be explicit. |
7296 | | // Assumes 2-complement integer types and sign extension of right shifting |
7297 | | // GCC guarantees such: |
7298 | | // https://gcc.gnu.org/onlinedocs/gcc/Integers-implementation.html#Integers-implementation |
7299 | | static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit) |
7300 | 0 | { |
7301 | 0 | return static_cast<GByte>(static_cast<signed char>(byVal << (7 - nBit)) >> |
7302 | 0 | 7); |
7303 | 0 | } |
7304 | | #else |
7305 | | // Portable way |
7306 | | static inline GByte ExtractBitAndConvertTo255(GByte byVal, int nBit) |
7307 | | { |
7308 | | return (byVal & (1 << nBit)) ? 255 : 0; |
7309 | | } |
7310 | | #endif |
7311 | | |
7312 | | /************************************************************************/ |
7313 | | /* ExpandEightPackedBitsToByteAt255() */ |
7314 | | /************************************************************************/ |
7315 | | |
7316 | | static inline void ExpandEightPackedBitsToByteAt255(GByte byVal, |
7317 | | GByte abyOutput[8]) |
7318 | 0 | { |
7319 | 0 | abyOutput[0] = ExtractBitAndConvertTo255(byVal, 7); |
7320 | 0 | abyOutput[1] = ExtractBitAndConvertTo255(byVal, 6); |
7321 | 0 | abyOutput[2] = ExtractBitAndConvertTo255(byVal, 5); |
7322 | 0 | abyOutput[3] = ExtractBitAndConvertTo255(byVal, 4); |
7323 | 0 | abyOutput[4] = ExtractBitAndConvertTo255(byVal, 3); |
7324 | 0 | abyOutput[5] = ExtractBitAndConvertTo255(byVal, 2); |
7325 | 0 | abyOutput[6] = ExtractBitAndConvertTo255(byVal, 1); |
7326 | 0 | abyOutput[7] = ExtractBitAndConvertTo255(byVal, 0); |
7327 | 0 | } |
7328 | | |
7329 | | /************************************************************************/ |
7330 | | /* GDALExpandPackedBitsToByteAt0Or255() */ |
7331 | | /************************************************************************/ |
7332 | | |
7333 | | /** Expand packed-bits (ordered from most-significant bit to least one) |
7334 | | into a byte each, where a bit at 0 is expanded to a byte at 0, and a bit |
7335 | | at 1 to a byte at 255. |
7336 | | |
7337 | | The function does (in a possibly more optimized way) the following: |
7338 | | \code{.cpp} |
7339 | | for (size_t i = 0; i < nInputBits; ++i ) |
7340 | | { |
7341 | | pabyOutput[i] = (pabyInput[i / 8] & (1 << (7 - (i % 8)))) ? 255 : 0; |
7342 | | } |
7343 | | \endcode |
7344 | | |
7345 | | @param pabyInput Input array of (nInputBits + 7) / 8 bytes. |
7346 | | @param pabyOutput Output array of nInputBits bytes. |
7347 | | @param nInputBits Number of valid bits in pabyInput. |
7348 | | |
7349 | | @since 3.11 |
7350 | | */ |
7351 | | |
7352 | | void GDALExpandPackedBitsToByteAt0Or255(const GByte *CPL_RESTRICT pabyInput, |
7353 | | GByte *CPL_RESTRICT pabyOutput, |
7354 | | size_t nInputBits) |
7355 | 0 | { |
7356 | 0 | const size_t nInputWholeBytes = nInputBits / 8; |
7357 | 0 | size_t iByte = 0; |
7358 | |
|
7359 | 0 | #ifdef HAVE_SSE2 |
7360 | | // Mask to isolate each bit |
7361 | 0 | const __m128i bit_mask = _mm_set_epi8(1, 2, 4, 8, 16, 32, 64, -128, 1, 2, 4, |
7362 | 0 | 8, 16, 32, 64, -128); |
7363 | 0 | const __m128i zero = _mm_setzero_si128(); |
7364 | 0 | const __m128i all_ones = _mm_set1_epi8(-1); |
7365 | | #ifdef __SSSE3__ |
7366 | | const __m128i dispatch_two_bytes = |
7367 | | _mm_set_epi8(1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0); |
7368 | | #endif |
7369 | 0 | constexpr size_t SSE_REG_SIZE = sizeof(bit_mask); |
7370 | 0 | for (; iByte + SSE_REG_SIZE <= nInputWholeBytes; iByte += SSE_REG_SIZE) |
7371 | 0 | { |
7372 | 0 | __m128i reg_ori = _mm_loadu_si128( |
7373 | 0 | reinterpret_cast<const __m128i *>(pabyInput + iByte)); |
7374 | |
|
7375 | 0 | constexpr int NUM_PROCESSED_BYTES_PER_REG = 2; |
7376 | 0 | for (size_t k = 0; k < SSE_REG_SIZE / NUM_PROCESSED_BYTES_PER_REG; ++k) |
7377 | 0 | { |
7378 | | // Given reg_ori = (A, B, ... 14 other bytes ...), |
7379 | | // expand to (A, A, A, A, A, A, A, A, B, B, B, B, B, B, B, B) |
7380 | | #ifdef __SSSE3__ |
7381 | | __m128i reg = _mm_shuffle_epi8(reg_ori, dispatch_two_bytes); |
7382 | | #else |
7383 | 0 | __m128i reg = _mm_unpacklo_epi8(reg_ori, reg_ori); |
7384 | 0 | reg = _mm_unpacklo_epi16(reg, reg); |
7385 | 0 | reg = _mm_unpacklo_epi32(reg, reg); |
7386 | 0 | #endif |
7387 | | |
7388 | | // Test if bits of interest are set |
7389 | 0 | reg = _mm_and_si128(reg, bit_mask); |
7390 | | |
7391 | | // Now test if those bits are set, by comparing to zero. So the |
7392 | | // result will be that bytes where bits are set will be at 0, and |
7393 | | // ones where they are cleared will be at 0xFF. So the inverse of |
7394 | | // the end result we want! |
7395 | 0 | reg = _mm_cmpeq_epi8(reg, zero); |
7396 | | |
7397 | | // Invert the result |
7398 | 0 | reg = _mm_andnot_si128(reg, all_ones); |
7399 | |
|
7400 | 0 | _mm_storeu_si128(reinterpret_cast<__m128i *>(pabyOutput), reg); |
7401 | |
|
7402 | 0 | pabyOutput += SSE_REG_SIZE; |
7403 | | |
7404 | | // Right-shift of 2 bytes |
7405 | 0 | reg_ori = _mm_bsrli_si128(reg_ori, NUM_PROCESSED_BYTES_PER_REG); |
7406 | 0 | } |
7407 | 0 | } |
7408 | |
|
7409 | 0 | #endif // HAVE_SSE2 |
7410 | |
|
7411 | 0 | for (; iByte < nInputWholeBytes; ++iByte) |
7412 | 0 | { |
7413 | 0 | ExpandEightPackedBitsToByteAt255(pabyInput[iByte], pabyOutput); |
7414 | 0 | pabyOutput += 8; |
7415 | 0 | } |
7416 | 0 | for (int iBit = 0; iBit < static_cast<int>(nInputBits % 8); ++iBit) |
7417 | 0 | { |
7418 | 0 | *pabyOutput = ExtractBitAndConvertTo255(pabyInput[iByte], 7 - iBit); |
7419 | 0 | ++pabyOutput; |
7420 | 0 | } |
7421 | 0 | } |
7422 | | |
7423 | | /************************************************************************/ |
7424 | | /* ExpandEightPackedBitsToByteAt1() */ |
7425 | | /************************************************************************/ |
7426 | | |
7427 | | static inline void ExpandEightPackedBitsToByteAt1(GByte byVal, |
7428 | | GByte abyOutput[8]) |
7429 | 0 | { |
7430 | 0 | abyOutput[0] = (byVal >> 7) & 0x1; |
7431 | 0 | abyOutput[1] = (byVal >> 6) & 0x1; |
7432 | 0 | abyOutput[2] = (byVal >> 5) & 0x1; |
7433 | 0 | abyOutput[3] = (byVal >> 4) & 0x1; |
7434 | 0 | abyOutput[4] = (byVal >> 3) & 0x1; |
7435 | 0 | abyOutput[5] = (byVal >> 2) & 0x1; |
7436 | 0 | abyOutput[6] = (byVal >> 1) & 0x1; |
7437 | 0 | abyOutput[7] = (byVal >> 0) & 0x1; |
7438 | 0 | } |
7439 | | |
7440 | | /************************************************************************/ |
7441 | | /* GDALExpandPackedBitsToByteAt0Or1() */ |
7442 | | /************************************************************************/ |
7443 | | |
7444 | | /** Expand packed-bits (ordered from most-significant bit to least one) |
7445 | | into a byte each, where a bit at 0 is expanded to a byte at 0, and a bit |
7446 | | at 1 to a byte at 1. |
7447 | | |
7448 | | The function does (in a possibly more optimized way) the following: |
7449 | | \code{.cpp} |
7450 | | for (size_t i = 0; i < nInputBits; ++i ) |
7451 | | { |
7452 | | pabyOutput[i] = (pabyInput[i / 8] & (1 << (7 - (i % 8)))) ? 1 : 0; |
7453 | | } |
7454 | | \endcode |
7455 | | |
7456 | | @param pabyInput Input array of (nInputBits + 7) / 8 bytes. |
7457 | | @param pabyOutput Output array of nInputBits bytes. |
7458 | | @param nInputBits Number of valid bits in pabyInput. |
7459 | | |
7460 | | @since 3.11 |
7461 | | */ |
7462 | | |
7463 | | void GDALExpandPackedBitsToByteAt0Or1(const GByte *CPL_RESTRICT pabyInput, |
7464 | | GByte *CPL_RESTRICT pabyOutput, |
7465 | | size_t nInputBits) |
7466 | 0 | { |
7467 | 0 | const size_t nInputWholeBytes = nInputBits / 8; |
7468 | 0 | size_t iByte = 0; |
7469 | 0 | for (; iByte < nInputWholeBytes; ++iByte) |
7470 | 0 | { |
7471 | 0 | ExpandEightPackedBitsToByteAt1(pabyInput[iByte], pabyOutput); |
7472 | 0 | pabyOutput += 8; |
7473 | 0 | } |
7474 | 0 | for (int iBit = 0; iBit < static_cast<int>(nInputBits % 8); ++iBit) |
7475 | 0 | { |
7476 | 0 | *pabyOutput = (pabyInput[iByte] >> (7 - iBit)) & 0x1; |
7477 | 0 | ++pabyOutput; |
7478 | 0 | } |
7479 | 0 | } |