Coverage Report

Created: 2025-06-13 06:29

/src/gdal/gcore/gdalvirtualmem.cpp
Line
Count
Source (jump to first uncovered line)
1
/**********************************************************************
2
 *
3
 * Name:     gdalvirtualmem.cpp
4
 * Project:  GDAL
5
 * Purpose:  Dataset and rasterband exposed as a virtual memory mapping.
6
 * Author:   Even Rouault, <even dot rouault at spatialys.com>
7
 *
8
 **********************************************************************
9
 * Copyright (c) 2014, Even Rouault <even dot rouault at spatialys.com>
10
 *
11
 * SPDX-License-Identifier: MIT
12
 ****************************************************************************/
13
14
#include "cpl_port.h"
15
#include "gdal.h"
16
#include "gdal_priv.h"
17
18
#include <cstddef>
19
#include <cstring>
20
21
#include <algorithm>
22
23
#include "cpl_conv.h"
24
#include "cpl_error.h"
25
#include "cpl_virtualmem.h"
26
27
// To be changed if we go to 64-bit RasterIO coordinates and spacing.
28
using coord_type = int;
29
using spacing_type = int;
30
31
/************************************************************************/
32
/*                            GDALVirtualMem                            */
33
/************************************************************************/
34
35
class GDALVirtualMem
36
{
37
    GDALDatasetH hDS = nullptr;
38
    GDALRasterBandH hBand = nullptr;
39
    coord_type nXOff = 0;
40
    coord_type nYOff = 0;
41
    // int nXSize;
42
    // int nYSize;
43
    coord_type nBufXSize = 0;
44
    coord_type nBufYSize = 0;
45
    GDALDataType eBufType = GDT_Byte;
46
    int nBandCount = 0;
47
    int *panBandMap = nullptr;
48
    int nPixelSpace = 0;
49
    GIntBig nLineSpace = 0;
50
    GIntBig nBandSpace = 0;
51
52
    bool bIsCompact = false;
53
    bool bIsBandSequential = false;
54
55
    bool IsCompact() const
56
0
    {
57
0
        return bIsCompact;
58
0
    }
59
60
    bool IsBandSequential() const
61
0
    {
62
0
        return bIsBandSequential;
63
0
    }
64
65
    void GetXYBand(size_t nOffset, coord_type &x, coord_type &y,
66
                   int &band) const;
67
    size_t GetOffset(const coord_type &x, const coord_type &y, int band) const;
68
    bool GotoNextPixel(coord_type &x, coord_type &y, int &band) const;
69
70
    void DoIOBandSequential(GDALRWFlag eRWFlag, size_t nOffset, void *pPage,
71
                            size_t nBytes) const;
72
    void DoIOPixelInterleaved(GDALRWFlag eRWFlag, size_t nOffset, void *pPage,
73
                              size_t nBytes) const;
74
75
    CPL_DISALLOW_COPY_ASSIGN(GDALVirtualMem)
76
77
  public:
78
    GDALVirtualMem(GDALDatasetH hDS, GDALRasterBandH hBand,
79
                   const coord_type &nXOff, const coord_type &nYOff,
80
                   const coord_type &nXSize, const coord_type &nYSize,
81
                   const coord_type &nBufXSize, const coord_type &nBufYSize,
82
                   GDALDataType eBufType, int nBandCount,
83
                   const int *panBandMapIn, int nPixelSpace, GIntBig nLineSpace,
84
                   GIntBig nBandSpace);
85
    ~GDALVirtualMem();
86
87
    static void FillCacheBandSequential(CPLVirtualMem *ctxt, size_t nOffset,
88
                                        void *pPageToFill, size_t nToFill,
89
                                        void *pUserData);
90
    static void SaveFromCacheBandSequential(CPLVirtualMem *ctxt, size_t nOffset,
91
                                            const void *pPageToBeEvicted,
92
                                            size_t nToEvicted, void *pUserData);
93
94
    static void FillCachePixelInterleaved(CPLVirtualMem *ctxt, size_t nOffset,
95
                                          void *pPageToFill, size_t nToFill,
96
                                          void *pUserData);
97
    static void SaveFromCachePixelInterleaved(CPLVirtualMem *ctxt,
98
                                              size_t nOffset,
99
                                              const void *pPageToBeEvicted,
100
                                              size_t nToEvicted,
101
                                              void *pUserData);
102
103
    static void Destroy(void *pUserData);
104
};
105
106
/************************************************************************/
107
/*                             GDALVirtualMem()                         */
108
/************************************************************************/
109
110
GDALVirtualMem::GDALVirtualMem(
111
    GDALDatasetH hDSIn, GDALRasterBandH hBandIn, const coord_type &nXOffIn,
112
    const coord_type &nYOffIn, const coord_type & /* nXSize */,
113
    const coord_type & /* nYSize */, const coord_type &nBufXSizeIn,
114
    const coord_type &nBufYSizeIn, GDALDataType eBufTypeIn, int nBandCountIn,
115
    const int *panBandMapIn, int nPixelSpaceIn, GIntBig nLineSpaceIn,
116
    GIntBig nBandSpaceIn)
117
0
    : hDS(hDSIn), hBand(hBandIn), nXOff(nXOffIn), nYOff(nYOffIn),
118
      // TODO(schwehr): Why not used or removed?
119
      // nXSize(nXSize),
120
      // nYSize(nYSize),
121
0
      nBufXSize(nBufXSizeIn), nBufYSize(nBufYSizeIn), eBufType(eBufTypeIn),
122
0
      nBandCount(nBandCountIn), nPixelSpace(nPixelSpaceIn),
123
0
      nLineSpace(nLineSpaceIn), nBandSpace(nBandSpaceIn)
124
0
{
125
0
    if (hDS != nullptr)
126
0
    {
127
0
        panBandMap = static_cast<int *>(CPLMalloc(nBandCount * sizeof(int)));
128
0
        if (panBandMapIn)
129
0
        {
130
0
            memcpy(panBandMap, panBandMapIn, nBandCount * sizeof(int));
131
0
        }
132
0
        else
133
0
        {
134
0
            for (int i = 0; i < nBandCount; i++)
135
0
                panBandMap[i] = i + 1;
136
0
        }
137
0
    }
138
0
    else
139
0
    {
140
0
        panBandMap = nullptr;
141
0
        nBandCount = 1;
142
0
    }
143
144
0
    const int nDataTypeSize = GDALGetDataTypeSizeBytes(eBufType);
145
0
    if (nPixelSpace == nDataTypeSize &&
146
0
        nLineSpace == static_cast<GIntBig>(nBufXSize) * nPixelSpace &&
147
0
        nBandSpace == nBufYSize * nLineSpace)
148
0
        bIsCompact = true;
149
0
    else if (nBandSpace == nDataTypeSize &&
150
0
             nPixelSpace == nBandCount * nBandSpace &&
151
0
             nLineSpace == static_cast<GIntBig>(nBufXSize) * nPixelSpace)
152
0
        bIsCompact = true;
153
0
    else
154
0
        bIsCompact = false;
155
156
0
    bIsBandSequential = nBandSpace >= nBufYSize * nLineSpace;
157
0
}
158
159
/************************************************************************/
160
/*                            ~GDALVirtualMem()                         */
161
/************************************************************************/
162
163
GDALVirtualMem::~GDALVirtualMem()
164
0
{
165
0
    CPLFree(panBandMap);
166
0
}
167
168
/************************************************************************/
169
/*                              GetXYBand()                             */
170
/************************************************************************/
171
172
void GDALVirtualMem::GetXYBand(size_t nOffset, coord_type &x, coord_type &y,
173
                               int &band) const
174
0
{
175
0
    if (IsBandSequential())
176
0
    {
177
0
        if (nBandCount == 1)
178
0
            band = 0;
179
0
        else
180
0
            band = static_cast<int>(nOffset / nBandSpace);
181
0
        y = static_cast<coord_type>((nOffset - band * nBandSpace) / nLineSpace);
182
0
        x = static_cast<coord_type>(
183
0
            (nOffset - band * nBandSpace - y * nLineSpace) / nPixelSpace);
184
0
    }
185
0
    else
186
0
    {
187
0
        y = static_cast<coord_type>(nOffset / nLineSpace);
188
0
        x = static_cast<coord_type>((nOffset - y * nLineSpace) / nPixelSpace);
189
0
        if (nBandCount == 1)
190
0
            band = 0;
191
0
        else
192
0
            band = static_cast<int>((nOffset - y * nLineSpace -
193
0
                                     static_cast<size_t>(x) * nPixelSpace) /
194
0
                                    nBandSpace);
195
0
    }
196
0
}
197
198
/************************************************************************/
199
/*                            GotoNextPixel()                           */
200
/************************************************************************/
201
202
bool GDALVirtualMem::GotoNextPixel(coord_type &x, coord_type &y,
203
                                   int &band) const
204
0
{
205
0
    if (IsBandSequential())
206
0
    {
207
0
        ++x;
208
0
        if (x == nBufXSize)
209
0
        {
210
0
            x = 0;
211
0
            ++y;
212
0
        }
213
0
        if (y == nBufYSize)
214
0
        {
215
0
            y = 0;
216
0
            band++;
217
0
            if (band == nBandCount)
218
0
                return false;
219
0
        }
220
0
    }
221
0
    else
222
0
    {
223
0
        band++;
224
0
        if (band == nBandCount)
225
0
        {
226
0
            band = 0;
227
0
            ++x;
228
0
        }
229
0
        if (x == nBufXSize)
230
0
        {
231
0
            x = 0;
232
0
            ++y;
233
0
            if (y == nBufYSize)
234
0
                return false;
235
0
        }
236
0
    }
237
0
    return true;
238
0
}
239
240
/************************************************************************/
241
/*                              GetOffset()                             */
242
/************************************************************************/
243
244
size_t GDALVirtualMem::GetOffset(const coord_type &x, const coord_type &y,
245
                                 int band) const
246
0
{
247
0
    return static_cast<size_t>(static_cast<size_t>(x) * nPixelSpace +
248
0
                               y * nLineSpace + band * nBandSpace);
249
0
}
250
251
/************************************************************************/
252
/*                          DoIOPixelInterleaved()                      */
253
/************************************************************************/
254
255
void GDALVirtualMem::DoIOPixelInterleaved(GDALRWFlag eRWFlag,
256
                                          const size_t nOffset, void *pPage,
257
                                          size_t nBytes) const
258
0
{
259
0
    coord_type x = 0;
260
0
    coord_type y = 0;
261
0
    int band = 0;
262
263
0
    GetXYBand(nOffset, x, y, band);
264
#ifdef DEBUG_VERBOSE
265
    fprintf(stderr, "eRWFlag=%d, nOffset=%d, x=%d, y=%d, band=%d\n", /*ok*/
266
            eRWFlag, static_cast<int>(nOffset), x, y, band);
267
#endif
268
269
0
    if (eRWFlag == GF_Read && !IsCompact())
270
0
        memset(pPage, 0, nBytes);
271
272
0
    if (band >= nBandCount)
273
0
    {
274
0
        band = nBandCount - 1;
275
0
        if (!GotoNextPixel(x, y, band))
276
0
            return;
277
0
    }
278
0
    else if (x >= nBufXSize)
279
0
    {
280
0
        x = nBufXSize - 1;
281
0
        band = nBandCount - 1;
282
0
        if (!GotoNextPixel(x, y, band))
283
0
            return;
284
0
    }
285
286
0
    size_t nOffsetRecompute = GetOffset(x, y, band);
287
0
    CPLAssert(nOffsetRecompute >= nOffset);
288
0
    size_t nOffsetShift = nOffsetRecompute - nOffset;
289
0
    if (nOffsetShift >= nBytes)
290
0
        return;
291
292
    // If we don't start at the first band for that given pixel, load/store
293
    // the remaining bands
294
0
    if (band > 0)
295
0
    {
296
0
        size_t nEndOffsetEndOfPixel = GetOffset(x, y, nBandCount);
297
0
        int bandEnd = nBandCount;
298
        // Check that we have enough space to load/store until last band
299
        // Should be always OK unless the number of bands is really huge
300
0
        if (nEndOffsetEndOfPixel - nOffset > nBytes)
301
0
        {
302
            // Not enough space: find last possible band
303
0
            coord_type xEnd, yEnd;
304
0
            GetXYBand(nOffset + nBytes, xEnd, yEnd, bandEnd);
305
0
            CPLAssert(x == xEnd);
306
0
            CPLAssert(y == yEnd);
307
0
        }
308
309
        // Finish reading/writing the remaining bands for that pixel
310
0
        CPL_IGNORE_RET_VAL(GDALDatasetRasterIO(
311
0
            hDS, eRWFlag, nXOff + x, nYOff + y, 1, 1,
312
0
            static_cast<char *>(pPage) + nOffsetShift, 1, 1, eBufType,
313
0
            bandEnd - band, panBandMap + band, nPixelSpace,
314
0
            static_cast<spacing_type>(nLineSpace),
315
0
            static_cast<spacing_type>(nBandSpace)));
316
317
0
        if (bandEnd < nBandCount)
318
0
            return;
319
320
0
        band = nBandCount - 1;
321
0
        if (!GotoNextPixel(x, y, band))
322
0
            return;
323
0
        nOffsetRecompute = GetOffset(x, y, 0);
324
0
        nOffsetShift = nOffsetRecompute - nOffset;
325
0
        if (nOffsetShift >= nBytes)
326
0
            return;
327
0
    }
328
329
    // Is there enough place to store/load up to the end of current line ?
330
0
    size_t nEndOffsetEndOfLine = GetOffset(nBufXSize - 1, y, nBandCount);
331
0
    if (nEndOffsetEndOfLine - nOffset > nBytes)
332
0
    {
333
        // No : read/write as many pixels on this line as possible
334
0
        coord_type xEnd, yEnd;
335
0
        int bandEnd;
336
0
        GetXYBand(nOffset + nBytes, xEnd, yEnd, bandEnd);
337
0
        CPLAssert(y == yEnd);
338
339
0
        if (x < xEnd)
340
0
        {
341
0
            CPL_IGNORE_RET_VAL(GDALDatasetRasterIO(
342
0
                hDS, eRWFlag, nXOff + x, nYOff + y, xEnd - x, 1,
343
0
                static_cast<char *>(pPage) + nOffsetShift, xEnd - x, 1,
344
0
                eBufType, nBandCount, panBandMap, nPixelSpace,
345
0
                static_cast<spacing_type>(nLineSpace),
346
0
                static_cast<spacing_type>(nBandSpace)));
347
0
        }
348
349
        // Are there partial bands to read/write for the last pixel ?
350
0
        if (bandEnd > 0)
351
0
        {
352
0
            x = xEnd;
353
0
            nOffsetRecompute = GetOffset(x, y, 0);
354
0
            nOffsetShift = nOffsetRecompute - nOffset;
355
0
            if (nOffsetShift >= nBytes)
356
0
                return;
357
358
0
            if (bandEnd >= nBandCount)
359
0
                bandEnd = nBandCount;
360
361
0
            CPL_IGNORE_RET_VAL(GDALDatasetRasterIO(
362
0
                hDS, eRWFlag, nXOff + x, nYOff + y, 1, 1,
363
0
                static_cast<char *>(pPage) + nOffsetShift, 1, 1, eBufType,
364
0
                bandEnd, panBandMap, nPixelSpace,
365
0
                static_cast<spacing_type>(nLineSpace),
366
0
                static_cast<spacing_type>(nBandSpace)));
367
0
        }
368
369
0
        return;
370
0
    }
371
372
    // Yes, enough place to read/write until end of line
373
0
    if (x > 0 || nBytes - nOffsetShift < static_cast<size_t>(nLineSpace))
374
0
    {
375
0
        CPL_IGNORE_RET_VAL(GDALDatasetRasterIO(
376
0
            hDS, eRWFlag, nXOff + x, nYOff + y, nBufXSize - x, 1,
377
0
            static_cast<char *>(pPage) + nOffsetShift, nBufXSize - x, 1,
378
0
            eBufType, nBandCount, panBandMap, nPixelSpace,
379
0
            static_cast<spacing_type>(nLineSpace),
380
0
            static_cast<spacing_type>(nBandSpace)));
381
382
        // Go to beginning of next line
383
0
        x = nBufXSize - 1;
384
0
        band = nBandCount - 1;
385
0
        if (!GotoNextPixel(x, y, band))
386
0
            return;
387
0
        nOffsetRecompute = GetOffset(x, y, 0);
388
0
        nOffsetShift = nOffsetRecompute - nOffset;
389
0
        if (nOffsetShift >= nBytes)
390
0
            return;
391
0
    }
392
393
    // How many whole lines can we store/load ?
394
0
    coord_type nLineCount =
395
0
        static_cast<coord_type>((nBytes - nOffsetShift) / nLineSpace);
396
0
    if (y + nLineCount > nBufYSize)
397
0
        nLineCount = nBufYSize - y;
398
0
    if (nLineCount > 0)
399
0
    {
400
0
        CPL_IGNORE_RET_VAL(GDALDatasetRasterIO(
401
0
            hDS, eRWFlag, nXOff + 0, nYOff + y, nBufXSize, nLineCount,
402
0
            static_cast<GByte *>(pPage) + nOffsetShift, nBufXSize, nLineCount,
403
0
            eBufType, nBandCount, panBandMap, nPixelSpace,
404
0
            static_cast<spacing_type>(nLineSpace),
405
0
            static_cast<spacing_type>(nBandSpace)));
406
407
0
        y += nLineCount;
408
0
        if (y == nBufYSize)
409
0
            return;
410
0
        nOffsetRecompute = GetOffset(x, y, 0);
411
0
        nOffsetShift = nOffsetRecompute - nOffset;
412
0
    }
413
414
0
    if (nOffsetShift < nBytes)
415
0
    {
416
0
        DoIOPixelInterleaved(eRWFlag, nOffsetRecompute,
417
0
                             static_cast<char *>(pPage) + nOffsetShift,
418
0
                             nBytes - nOffsetShift);
419
0
    }
420
0
}
421
422
/************************************************************************/
423
/*                          DoIOPixelInterleaved()                      */
424
/************************************************************************/
425
426
void GDALVirtualMem::DoIOBandSequential(GDALRWFlag eRWFlag,
427
                                        const size_t nOffset, void *pPage,
428
                                        size_t nBytes) const
429
0
{
430
0
    coord_type x = 0;
431
0
    coord_type y = 0;
432
433
0
    int band = 0;
434
0
    GetXYBand(nOffset, x, y, band);
435
#if DEBUG_VERBOSE
436
    fprintf(stderr, "eRWFlag=%d, nOffset=%d, x=%d, y=%d, band=%d\n", /*ok*/
437
            eRWFlag, static_cast<int>(nOffset), x, y, band);
438
#endif
439
440
0
    if (eRWFlag == GF_Read && !IsCompact())
441
0
        memset(pPage, 0, nBytes);
442
443
0
    if (x >= nBufXSize)
444
0
    {
445
0
        x = nBufXSize - 1;
446
0
        if (!GotoNextPixel(x, y, band))
447
0
            return;
448
0
    }
449
0
    else if (y >= nBufYSize)
450
0
    {
451
0
        x = nBufXSize - 1;
452
0
        y = nBufYSize - 1;
453
0
        if (!GotoNextPixel(x, y, band))
454
0
            return;
455
0
    }
456
457
0
    size_t nOffsetRecompute = GetOffset(x, y, band);
458
0
    CPLAssert(nOffsetRecompute >= nOffset);
459
0
    size_t nOffsetShift = nOffsetRecompute - nOffset;
460
0
    if (nOffsetShift >= nBytes)
461
0
        return;
462
463
    // Is there enough place to store/load up to the end of current line?
464
0
    size_t nEndOffsetEndOfLine = GetOffset(nBufXSize, y, band);
465
0
    if (nEndOffsetEndOfLine - nOffset > nBytes)
466
0
    {
467
        // No : read/write as many pixels on this line as possible
468
0
        coord_type xEnd, yEnd;
469
0
        int bandEnd;
470
0
        GetXYBand(nOffset + nBytes, xEnd, yEnd, bandEnd);
471
0
        CPLAssert(y == yEnd);
472
0
        CPLAssert(band == bandEnd);
473
0
        CPL_IGNORE_RET_VAL(GDALRasterIO(
474
0
            hBand ? hBand : GDALGetRasterBand(hDS, panBandMap[band]), eRWFlag,
475
0
            nXOff + x, nYOff + y, xEnd - x, 1,
476
0
            static_cast<char *>(pPage) + nOffsetShift, xEnd - x, 1, eBufType,
477
0
            nPixelSpace, static_cast<spacing_type>(nLineSpace)));
478
479
0
        return;
480
0
    }
481
482
    // Yes, enough place to read/write until end of line
483
0
    if (x > 0 || nBytes - nOffsetShift < static_cast<size_t>(nLineSpace))
484
0
    {
485
0
        CPL_IGNORE_RET_VAL(GDALRasterIO(
486
0
            hBand ? hBand : GDALGetRasterBand(hDS, panBandMap[band]), eRWFlag,
487
0
            nXOff + x, nYOff + y, nBufXSize - x, 1,
488
0
            static_cast<char *>(pPage) + nOffsetShift, nBufXSize - x, 1,
489
0
            eBufType, nPixelSpace, static_cast<spacing_type>(nLineSpace)));
490
491
        // Go to beginning of next line
492
0
        x = nBufXSize - 1;
493
0
        if (!GotoNextPixel(x, y, band))
494
0
            return;
495
0
        nOffsetRecompute = GetOffset(x, y, band);
496
0
        nOffsetShift = nOffsetRecompute - nOffset;
497
0
        if (nOffsetShift >= nBytes)
498
0
            return;
499
0
    }
500
501
    // How many whole lines can we store/load ?
502
0
    coord_type nLineCount =
503
0
        static_cast<coord_type>((nBytes - nOffsetShift) / nLineSpace);
504
0
    if (y + nLineCount > nBufYSize)
505
0
        nLineCount = nBufYSize - y;
506
0
    if (nLineCount > 0)
507
0
    {
508
0
        CPL_IGNORE_RET_VAL(GDALRasterIO(
509
0
            hBand ? hBand : GDALGetRasterBand(hDS, panBandMap[band]), eRWFlag,
510
0
            nXOff + 0, nYOff + y, nBufXSize, nLineCount,
511
0
            static_cast<GByte *>(pPage) + nOffsetShift, nBufXSize, nLineCount,
512
0
            eBufType, nPixelSpace, static_cast<spacing_type>(nLineSpace)));
513
514
0
        y += nLineCount;
515
0
        if (y == nBufYSize)
516
0
        {
517
0
            y = 0;
518
0
            band++;
519
0
            if (band == nBandCount)
520
0
                return;
521
0
        }
522
0
        nOffsetRecompute = GetOffset(x, y, band);
523
0
        nOffsetShift = nOffsetRecompute - nOffset;
524
0
    }
525
526
0
    if (nOffsetShift < nBytes)
527
0
    {
528
0
        DoIOBandSequential(eRWFlag, nOffsetRecompute,
529
0
                           static_cast<char *>(pPage) + nOffsetShift,
530
0
                           nBytes - nOffsetShift);
531
0
    }
532
0
}
533
534
/************************************************************************/
535
/*                    FillCacheBandSequential()                        */
536
/************************************************************************/
537
538
void GDALVirtualMem::FillCacheBandSequential(CPLVirtualMem *, size_t nOffset,
539
                                             void *pPageToFill, size_t nToFill,
540
                                             void *pUserData)
541
0
{
542
0
    const GDALVirtualMem *psParams = static_cast<GDALVirtualMem *>(pUserData);
543
0
    psParams->DoIOBandSequential(GF_Read, nOffset, pPageToFill, nToFill);
544
0
}
545
546
/************************************************************************/
547
/*                    SaveFromCacheBandSequential()                    */
548
/************************************************************************/
549
550
void GDALVirtualMem::SaveFromCacheBandSequential(CPLVirtualMem *,
551
                                                 size_t nOffset,
552
                                                 const void *pPageToBeEvicted,
553
                                                 size_t nToEvicted,
554
                                                 void *pUserData)
555
0
{
556
0
    const GDALVirtualMem *psParams = static_cast<GDALVirtualMem *>(pUserData);
557
0
    psParams->DoIOBandSequential(
558
0
        GF_Write, nOffset, const_cast<void *>(pPageToBeEvicted), nToEvicted);
559
0
}
560
561
/************************************************************************/
562
/*                     FillCachePixelInterleaved()                      */
563
/************************************************************************/
564
565
void GDALVirtualMem::FillCachePixelInterleaved(CPLVirtualMem *, size_t nOffset,
566
                                               void *pPageToFill,
567
                                               size_t nToFill, void *pUserData)
568
0
{
569
0
    const GDALVirtualMem *psParams = static_cast<GDALVirtualMem *>(pUserData);
570
0
    psParams->DoIOPixelInterleaved(GF_Read, nOffset, pPageToFill, nToFill);
571
0
}
572
573
/************************************************************************/
574
/*                     SaveFromCachePixelInterleaved()                  */
575
/************************************************************************/
576
577
void GDALVirtualMem::SaveFromCachePixelInterleaved(CPLVirtualMem *,
578
                                                   size_t nOffset,
579
                                                   const void *pPageToBeEvicted,
580
                                                   size_t nToEvicted,
581
                                                   void *pUserData)
582
0
{
583
0
    const GDALVirtualMem *psParams = static_cast<GDALVirtualMem *>(pUserData);
584
0
    psParams->DoIOPixelInterleaved(
585
0
        GF_Write, nOffset, const_cast<void *>(pPageToBeEvicted), nToEvicted);
586
0
}
587
588
/************************************************************************/
589
/*                                Destroy()                             */
590
/************************************************************************/
591
592
void GDALVirtualMem::Destroy(void *pUserData)
593
0
{
594
0
    GDALVirtualMem *psParams = static_cast<GDALVirtualMem *>(pUserData);
595
0
    delete psParams;
596
0
}
597
598
/************************************************************************/
599
/*                      GDALCheckBandParameters()                       */
600
/************************************************************************/
601
602
static bool GDALCheckBandParameters(GDALDatasetH hDS, int nBandCount,
603
                                    int *panBandMap)
604
0
{
605
0
    if (nBandCount == 0)
606
0
    {
607
0
        CPLError(CE_Failure, CPLE_AppDefined, "nBandCount == 0");
608
0
        return false;
609
0
    }
610
611
0
    if (panBandMap != nullptr)
612
0
    {
613
0
        for (int i = 0; i < nBandCount; i++)
614
0
        {
615
0
            if (panBandMap[i] < 1 || panBandMap[i] > GDALGetRasterCount(hDS))
616
0
            {
617
0
                CPLError(CE_Failure, CPLE_AppDefined, "panBandMap[%d]=%d", i,
618
0
                         panBandMap[i]);
619
0
                return false;
620
0
            }
621
0
        }
622
0
    }
623
0
    else if (nBandCount > GDALGetRasterCount(hDS))
624
0
    {
625
0
        CPLError(CE_Failure, CPLE_AppDefined,
626
0
                 "nBandCount > GDALGetRasterCount(hDS)");
627
0
        return false;
628
0
    }
629
0
    return true;
630
0
}
631
632
/************************************************************************/
633
/*                          GDALGetVirtualMem()                         */
634
/************************************************************************/
635
636
static CPLVirtualMem *
637
GDALGetVirtualMem(GDALDatasetH hDS, GDALRasterBandH hBand, GDALRWFlag eRWFlag,
638
                  coord_type nXOff, coord_type nYOff, coord_type nXSize,
639
                  coord_type nYSize, coord_type nBufXSize, coord_type nBufYSize,
640
                  GDALDataType eBufType, int nBandCount, int *panBandMap,
641
                  int nPixelSpace, GIntBig nLineSpace, GIntBig nBandSpace,
642
                  size_t nCacheSize, size_t nPageSizeHint,
643
                  int bSingleThreadUsage, CSLConstList /*papszOptions*/)
644
0
{
645
0
    CPLVirtualMem *view = nullptr;
646
0
    GDALVirtualMem *psParams = nullptr;
647
0
    GUIntBig nReqMem = 0;
648
649
0
    if (nXSize != nBufXSize || nYSize != nBufYSize)
650
0
    {
651
0
        CPLError(CE_Failure, CPLE_NotSupported,
652
0
                 "nXSize != nBufXSize || nYSize != nBufYSize");
653
0
        return nullptr;
654
0
    }
655
656
0
    int nRasterXSize =
657
0
        hDS ? GDALGetRasterXSize(hDS) : GDALGetRasterBandXSize(hBand);
658
0
    int nRasterYSize =
659
0
        hDS ? GDALGetRasterYSize(hDS) : GDALGetRasterBandYSize(hBand);
660
661
0
    if (nXOff < 0 || nYOff < 0 || nXSize == 0 || nYSize == 0 || nBufXSize < 0 ||
662
0
        nBufYSize < 0 || nXOff + nXSize > nRasterXSize ||
663
0
        nYOff + nYSize > nRasterYSize)
664
0
    {
665
0
        CPLError(CE_Failure, CPLE_AppDefined, "Invalid window request");
666
0
        return nullptr;
667
0
    }
668
669
0
    if (nPixelSpace < 0 || nLineSpace < 0 || nBandSpace < 0)
670
0
    {
671
0
        CPLError(CE_Failure, CPLE_NotSupported,
672
0
                 "nPixelSpace < 0 || nLineSpace < 0 || nBandSpace < 0");
673
0
        return nullptr;
674
0
    }
675
676
0
    if (hDS != nullptr && !GDALCheckBandParameters(hDS, nBandCount, panBandMap))
677
0
        return nullptr;
678
679
0
    const int nDataTypeSize = GDALGetDataTypeSizeBytes(eBufType);
680
0
    if (nPixelSpace == 0)
681
0
        nPixelSpace = nDataTypeSize;
682
0
    if (nLineSpace == 0)
683
0
        nLineSpace = static_cast<GIntBig>(nBufXSize) * nPixelSpace;
684
0
    if (nBandSpace == 0)
685
0
        nBandSpace = static_cast<GIntBig>(nBufYSize) * nLineSpace;
686
687
    // OFFSET = offset(x,y,band) = x * nPixelSpace + y * nLineSpace + band *
688
    // nBandSpace where 0 <= x < nBufXSize and 0 <= y < nBufYSize and 0 <= band
689
    // < nBandCount if nPixelSpace, nLineSpace and nBandSpace can have arbitrary
690
    // values, there is no way of finding a unique(x,y,band) solution. We need
691
    // to restrict the space of possibilities strongly.
692
    // if nBandSpace >= nBufYSize * nLineSpace and
693
    //   nLineSpace >= nBufXSize * nPixelSpace,           INTERLEAVE = BAND
694
    //      band = OFFSET / nBandSpace
695
    //      y = (OFFSET - band * nBandSpace) / nLineSpace
696
    //      x = (OFFSET - band * nBandSpace - y * nLineSpace) / nPixelSpace
697
    // else if nPixelSpace >= nBandCount * nBandSpace and
698
    //   nLineSpace >= nBufXSize * nPixelSpace,    INTERLEAVE = PIXEL
699
    //      y = OFFSET / nLineSpace
700
    //      x = (OFFSET - y * nLineSpace) / nPixelSpace
701
    //      band = (OFFSET - y * nLineSpace - x * nPixelSpace) / nBandSpace
702
703
0
    if (nDataTypeSize == 0 || /* to please Coverity. not needed */
704
0
        nLineSpace < static_cast<GIntBig>(nBufXSize) * nPixelSpace ||
705
0
        (nBandCount > 1 &&
706
0
         (nBandSpace == nPixelSpace ||
707
0
          (nBandSpace < nPixelSpace &&
708
0
           (nBandSpace < nDataTypeSize ||
709
0
            nPixelSpace < nBandCount * nBandSpace)) ||
710
0
          (nBandSpace > nPixelSpace && (nPixelSpace < nDataTypeSize ||
711
0
                                        nBandSpace < nBufYSize * nLineSpace)))))
712
0
    {
713
0
        CPLError(CE_Failure, CPLE_NotSupported,
714
0
                 "Only pixel interleaving or band interleaving are supported");
715
0
        return nullptr;
716
0
    }
717
718
    /* Avoid odd spacings that would complicate I/O operations */
719
    /* Ensuring they are multiple of nDataTypeSize should be fine, because */
720
    /* the page size is a power of 2 that is also a multiple of nDataTypeSize */
721
0
    if ((nPixelSpace % nDataTypeSize) != 0 ||
722
0
        (nLineSpace % nDataTypeSize) != 0 || (nBandSpace % nDataTypeSize) != 0)
723
0
    {
724
0
        CPLError(CE_Failure, CPLE_NotSupported, "Unsupported spacing");
725
0
        return nullptr;
726
0
    }
727
728
0
    bool bIsBandSequential = nBandSpace >= nBufYSize * nLineSpace;
729
0
    if (bIsBandSequential)
730
0
        nReqMem = nBandCount * nBandSpace;
731
0
    else
732
0
        nReqMem = nBufYSize * nLineSpace;
733
0
    if (nReqMem != static_cast<GUIntBig>(static_cast<size_t>(nReqMem)))
734
0
    {
735
0
        CPLError(CE_Failure, CPLE_OutOfMemory,
736
0
                 "Cannot reserve " CPL_FRMT_GUIB " bytes", nReqMem);
737
0
        return nullptr;
738
0
    }
739
740
0
    psParams = new GDALVirtualMem(
741
0
        hDS, hBand, nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize,
742
0
        eBufType, nBandCount, panBandMap, nPixelSpace, nLineSpace, nBandSpace);
743
744
0
    view = CPLVirtualMemNew(
745
0
        static_cast<size_t>(nReqMem), nCacheSize, nPageSizeHint,
746
0
        bSingleThreadUsage,
747
0
        eRWFlag == GF_Read ? VIRTUALMEM_READONLY_ENFORCED
748
0
                           : VIRTUALMEM_READWRITE,
749
0
        bIsBandSequential ? GDALVirtualMem::FillCacheBandSequential
750
0
                          : GDALVirtualMem::FillCachePixelInterleaved,
751
0
        bIsBandSequential ? GDALVirtualMem::SaveFromCacheBandSequential
752
0
                          : GDALVirtualMem::SaveFromCachePixelInterleaved,
753
0
        GDALVirtualMem::Destroy, psParams);
754
755
0
    if (view == nullptr)
756
0
    {
757
0
        delete psParams;
758
0
    }
759
760
0
    return view;
761
0
}
762
763
/************************************************************************/
764
/*                       GDALDatasetGetVirtualMem()                     */
765
/************************************************************************/
766
767
/** Create a CPLVirtualMem object from a GDAL dataset object.
768
 *
769
 * Only supported on Linux for now.
770
 *
771
 * This method allows creating a virtual memory object for a region of one
772
 * or more GDALRasterBands from  this dataset. The content of the virtual
773
 * memory object is automatically filled from dataset content when a virtual
774
 * memory page is first accessed, and it is released (or flushed in case of a
775
 * "dirty" page) when the cache size limit has been reached.
776
 *
777
 * The pointer to access the virtual memory object is obtained with
778
 * CPLVirtualMemGetAddr(). It remains valid until CPLVirtualMemFree() is called.
779
 * CPLVirtualMemFree() must be called before the dataset object is destroyed.
780
 *
781
 * If p is such a pointer and base_type the C type matching eBufType, for
782
 * default values of spacing parameters, the element of image coordinates (x, y)
783
 * (relative to xOff, yOff) for band b can be accessed with
784
 * ((base_type*)p)[x + y * nBufXSize + (b-1)*nBufXSize*nBufYSize].
785
 *
786
 * Note that the mechanism used to transparently fill memory pages when they are
787
 * accessed is the same (but in a controlled way) than what occurs when a memory
788
 * error occurs in a program. Debugging software will generally interrupt
789
 * program execution when that happens. If needed, CPLVirtualMemPin() can be
790
 * used to avoid that by ensuring memory pages are allocated before being
791
 * accessed.
792
 *
793
 * The size of the region that can be mapped as a virtual memory object depends
794
 * on hardware and operating system limitations.
795
 * On Linux AMD64 platforms, the maximum value is 128 TB.
796
 * On Linux x86 platforms, the maximum value is 2 GB.
797
 *
798
 * Data type translation is automatically done if the data type
799
 * (eBufType) of the buffer is different than
800
 * that of the GDALRasterBand.
801
 *
802
 * Image decimation / replication is currently not supported, i.e. if the
803
 * size of the region being accessed (nXSize x nYSize) is different from the
804
 * buffer size (nBufXSize x nBufYSize).
805
 *
806
 * The nPixelSpace, nLineSpace and nBandSpace parameters allow reading into or
807
 * writing from various organization of buffers. Arbitrary values for the
808
 * spacing parameters are not supported. Those values must be multiple of the
809
 * size of thebuffer data type, and must be either band sequential
810
 * organization (typically nPixelSpace = GDALGetDataTypeSizeBytes(eBufType),
811
 * nLineSpace = nPixelSpace * nBufXSize,
812
 * nBandSpace = nLineSpace * nBufYSize), or pixel-interleaved organization
813
 * (typically nPixelSpace = nBandSpace * nBandCount,
814
 * nLineSpace = nPixelSpace * nBufXSize,
815
 * nBandSpace = GDALGetDataTypeSizeBytes(eBufType))
816
 *
817
 * @param hDS Dataset object
818
 *
819
 * @param eRWFlag Either GF_Read to read a region of data, or GF_Write to
820
 * write a region of data.
821
 *
822
 * @param nXOff The pixel offset to the top left corner of the region
823
 * of the band to be accessed.  This would be zero to start from the left side.
824
 *
825
 * @param nYOff The line offset to the top left corner of the region
826
 * of the band to be accessed.  This would be zero to start from the top.
827
 *
828
 * @param nXSize The width of the region of the band to be accessed in pixels.
829
 *
830
 * @param nYSize The height of the region of the band to be accessed in lines.
831
 *
832
 * @param nBufXSize the width of the buffer image into which the desired region
833
 * is to be read, or from which it is to be written.
834
 *
835
 * @param nBufYSize the height of the buffer image into which the desired
836
 * region is to be read, or from which it is to be written.
837
 *
838
 * @param eBufType the type of the pixel values in the data buffer. The
839
 * pixel values will automatically be translated to/from the GDALRasterBand
840
 * data type as needed.
841
 *
842
 * @param nBandCount the number of bands being read or written.
843
 *
844
 * @param panBandMap the list of nBandCount band numbers being read/written.
845
 * Note band numbers are 1 based. This may be NULL to select the first
846
 * nBandCount bands.
847
 *
848
 * @param nPixelSpace The byte offset from the start of one pixel value in
849
 * the buffer to the start of the next pixel value within a scanline. If
850
 * defaulted (0) the size of the datatype eBufType is used.
851
 *
852
 * @param nLineSpace The byte offset from the start of one scanline in
853
 * the buffer to the start of the next. If defaulted (0) the size of the
854
 * datatype eBufType * nBufXSize is used.
855
 *
856
 * @param nBandSpace the byte offset from the start of one bands data to the
857
 * start of the next. If defaulted (0) the value will be
858
 * nLineSpace * nBufYSize implying band sequential organization
859
 * of the data buffer.
860
 *
861
 * @param nCacheSize   size in bytes of the maximum memory that will be really
862
 *                     allocated (must ideally fit into RAM)
863
 *
864
 * @param nPageSizeHint hint for the page size. Must be a multiple of the
865
 *                      system page size, returned by CPLGetPageSize().
866
 *                      Minimum value is generally 4096. Might be set to 0 to
867
 *                      let the function determine a default page size.
868
 *
869
 * @param bSingleThreadUsage set to TRUE if there will be no concurrent threads
870
 *                           that will access the virtual memory mapping. This
871
 *                           can optimize performance a bit. If set to FALSE,
872
 *                           CPLVirtualMemDeclareThread() must be called.
873
 *
874
 * @param papszOptions NULL terminated list of options. Unused for now.
875
 *
876
 * @return a virtual memory object that must be freed by CPLVirtualMemFree(),
877
 *         or NULL in case of failure.
878
 *
879
 * @since GDAL 1.11
880
 */
881
882
CPLVirtualMem *GDALDatasetGetVirtualMem(
883
    GDALDatasetH hDS, GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize,
884
    int nYSize, int nBufXSize, int nBufYSize, GDALDataType eBufType,
885
    int nBandCount, int *panBandMap, int nPixelSpace, GIntBig nLineSpace,
886
    GIntBig nBandSpace, size_t nCacheSize, size_t nPageSizeHint,
887
    int bSingleThreadUsage, CSLConstList papszOptions)
888
0
{
889
0
    return GDALGetVirtualMem(hDS, nullptr, eRWFlag, nXOff, nYOff, nXSize,
890
0
                             nYSize, nBufXSize, nBufYSize, eBufType, nBandCount,
891
0
                             panBandMap, nPixelSpace, nLineSpace, nBandSpace,
892
0
                             nCacheSize, nPageSizeHint, bSingleThreadUsage,
893
0
                             papszOptions);
894
0
}
895
896
/************************************************************************/
897
/*                     GDALRasterBandGetVirtualMem()                    */
898
/************************************************************************/
899
900
/** Create a CPLVirtualMem object from a GDAL raster band object.
901
 *
902
 * Only supported on Linux for now.
903
 *
904
 * This method allows creating a virtual memory object for a region of a
905
 * GDALRasterBand. The content of the virtual
906
 * memory object is automatically filled from dataset content when a virtual
907
 * memory page is first accessed, and it is released (or flushed in case of a
908
 * "dirty" page) when the cache size limit has been reached.
909
 *
910
 * The pointer to access the virtual memory object is obtained with
911
 * CPLVirtualMemGetAddr(). It remains valid until CPLVirtualMemFree() is called.
912
 * CPLVirtualMemFree() must be called before the raster band object is
913
 * destroyed.
914
 *
915
 * If p is such a pointer and base_type the C type matching eBufType, for
916
 * values of spacing parameters, the element of image coordinates (x, y)
917
 * default (relative to xOff, yOff) can be accessed with
918
 * ((base_type*)p)[x + y * nBufXSize].
919
 *
920
 * Note that the mechanism used to transparently fill memory pages when they are
921
 * accessed is the same (but in a controlled way) than what occurs when a memory
922
 * error occurs in a program. Debugging software will generally interrupt
923
 * program execution when that happens. If needed, CPLVirtualMemPin() can be
924
 * used to avoid that by ensuring memory pages are allocated before being
925
 * accessed.
926
 *
927
 * The size of the region that can be mapped as a virtual memory object depends
928
 * on hardware and operating system limitations.
929
 * On Linux AMD64 platforms, the maximum value is 128 TB.
930
 * On Linux x86 platforms, the maximum value is 2 GB.
931
 *
932
 * Data type translation is automatically done if the data type
933
 * (eBufType) of the buffer is different than
934
 * that of the GDALRasterBand.
935
 *
936
 * Image decimation / replication is currently not supported, i.e. if the
937
 * size of the region being accessed (nXSize x nYSize) is different from the
938
 * buffer size (nBufXSize x nBufYSize).
939
 *
940
 * The nPixelSpace and nLineSpace parameters allow reading into or
941
 * writing from various organization of buffers. Arbitrary values for the
942
 * spacing parameters are not supported. Those values must be multiple of the
943
 * size of the buffer data type and must be such that nLineSpace >=
944
 * nPixelSpace * nBufXSize.
945
 *
946
 * @param hBand Rasterband object
947
 *
948
 * @param eRWFlag Either GF_Read to read a region of data, or GF_Write to
949
 * write a region of data.
950
 *
951
 * @param nXOff The pixel offset to the top left corner of the region
952
 * of the band to be accessed.  This would be zero to start from the left side.
953
 *
954
 * @param nYOff The line offset to the top left corner of the region
955
 * of the band to be accessed.  This would be zero to start from the top.
956
 *
957
 * @param nXSize The width of the region of the band to be accessed in pixels.
958
 *
959
 * @param nYSize The height of the region of the band to be accessed in lines.
960
 *
961
 * @param nBufXSize the width of the buffer image into which the desired region
962
 * is to be read, or from which it is to be written.
963
 *
964
 * @param nBufYSize the height of the buffer image into which the desired
965
 * region is to be read, or from which it is to be written.
966
 *
967
 * @param eBufType the type of the pixel values in the data buffer. The
968
 * pixel values will automatically be translated to/from the GDALRasterBand
969
 * data type as needed.
970
 *
971
 * @param nPixelSpace The byte offset from the start of one pixel value in the
972
 * buffer to the start of the next pixel value within a scanline. If defaulted
973
 * (0) the size of the datatype eBufType is used.
974
 *
975
 * @param nLineSpace The byte offset from the start of one scanline in the
976
 * buffer to the start of the next. If defaulted (0) the size of the datatype
977
 * eBufType * nBufXSize is used.
978
 *
979
 * @param nCacheSize   size in bytes of the maximum memory that will be really
980
 *                     allocated (must ideally fit into RAM)
981
 *
982
 * @param nPageSizeHint hint for the page size. Must be a multiple of the
983
 *                      system page size, returned by CPLGetPageSize().
984
 *                      Minimum value is generally 4096. Might be set to 0 to
985
 *                      let the function determine a default page size.
986
 *
987
 * @param bSingleThreadUsage set to TRUE if there will be no concurrent threads
988
 *                           that will access the virtual memory mapping. This
989
 *                           can optimize performance a bit. If set to FALSE,
990
 *                           CPLVirtualMemDeclareThread() must be called.
991
 *
992
 * @param papszOptions NULL terminated list of options. Unused for now.
993
 *
994
 * @return a virtual memory object that must be freed by CPLVirtualMemFree(),
995
 *         or NULL in case of failure.
996
 *
997
 * @since GDAL 1.11
998
 */
999
1000
CPLVirtualMem *GDALRasterBandGetVirtualMem(
1001
    GDALRasterBandH hBand, GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize,
1002
    int nYSize, int nBufXSize, int nBufYSize, GDALDataType eBufType,
1003
    int nPixelSpace, GIntBig nLineSpace, size_t nCacheSize,
1004
    size_t nPageSizeHint, int bSingleThreadUsage, CSLConstList papszOptions)
1005
0
{
1006
0
    return GDALGetVirtualMem(nullptr, hBand, eRWFlag, nXOff, nYOff, nXSize,
1007
0
                             nYSize, nBufXSize, nBufYSize, eBufType, 1, nullptr,
1008
0
                             nPixelSpace, nLineSpace, 0, nCacheSize,
1009
0
                             nPageSizeHint, bSingleThreadUsage, papszOptions);
1010
0
}
1011
1012
/************************************************************************/
1013
/*                        GDALTiledVirtualMem                           */
1014
/************************************************************************/
1015
1016
class GDALTiledVirtualMem
1017
{
1018
    GDALDatasetH hDS = nullptr;
1019
    GDALRasterBandH hBand = nullptr;
1020
    int nXOff = 0;
1021
    int nYOff = 0;
1022
    int nXSize = 0;
1023
    int nYSize = 0;
1024
    int nTileXSize = 0;
1025
    int nTileYSize = 0;
1026
    GDALDataType eBufType = GDT_Byte;
1027
    int nBandCount = 0;
1028
    int *panBandMap = nullptr;
1029
    GDALTileOrganization eTileOrganization = GTO_TIP;
1030
1031
    void DoIO(GDALRWFlag eRWFlag, size_t nOffset, void *pPage,
1032
              size_t nBytes) const;
1033
1034
    CPL_DISALLOW_COPY_ASSIGN(GDALTiledVirtualMem)
1035
1036
  public:
1037
    GDALTiledVirtualMem(GDALDatasetH hDS, GDALRasterBandH hBand, int nXOff,
1038
                        int nYOff, int nXSize, int nYSize, int nTileXSize,
1039
                        int nTileYSize, GDALDataType eBufType, int nBandCount,
1040
                        const int *panBandMapIn,
1041
                        GDALTileOrganization eTileOrganization);
1042
    ~GDALTiledVirtualMem();
1043
1044
    static void FillCache(CPLVirtualMem *ctxt, size_t nOffset,
1045
                          void *pPageToFill, size_t nPageSize, void *pUserData);
1046
    static void SaveFromCache(CPLVirtualMem *ctxt, size_t nOffset,
1047
                              const void *pPageToBeEvicted, size_t nToEvicted,
1048
                              void *pUserData);
1049
1050
    static void Destroy(void *pUserData);
1051
};
1052
1053
/************************************************************************/
1054
/*                        GDALTiledVirtualMem()                         */
1055
/************************************************************************/
1056
1057
GDALTiledVirtualMem::GDALTiledVirtualMem(
1058
    GDALDatasetH hDSIn, GDALRasterBandH hBandIn, int nXOffIn, int nYOffIn,
1059
    int nXSizeIn, int nYSizeIn, int nTileXSizeIn, int nTileYSizeIn,
1060
    GDALDataType eBufTypeIn, int nBandCountIn, const int *panBandMapIn,
1061
    GDALTileOrganization eTileOrganizationIn)
1062
0
    : hDS(hDSIn), hBand(hBandIn), nXOff(nXOffIn), nYOff(nYOffIn),
1063
0
      nXSize(nXSizeIn), nYSize(nYSizeIn), nTileXSize(nTileXSizeIn),
1064
0
      nTileYSize(nTileYSizeIn), eBufType(eBufTypeIn), nBandCount(nBandCountIn),
1065
0
      eTileOrganization(eTileOrganizationIn)
1066
0
{
1067
0
    if (hDS != nullptr)
1068
0
    {
1069
0
        panBandMap = static_cast<int *>(CPLMalloc(nBandCount * sizeof(int)));
1070
0
        if (panBandMapIn)
1071
0
        {
1072
0
            memcpy(panBandMap, panBandMapIn, nBandCount * sizeof(int));
1073
0
        }
1074
0
        else
1075
0
        {
1076
0
            for (int i = 0; i < nBandCount; i++)
1077
0
                panBandMap[i] = i + 1;
1078
0
        }
1079
0
    }
1080
0
    else
1081
0
    {
1082
0
        panBandMap = nullptr;
1083
0
        nBandCount = 1;
1084
0
    }
1085
0
}
1086
1087
/************************************************************************/
1088
/*                       ~GDALTiledVirtualMem()                         */
1089
/************************************************************************/
1090
1091
GDALTiledVirtualMem::~GDALTiledVirtualMem()
1092
0
{
1093
0
    CPLFree(panBandMap);
1094
0
}
1095
1096
/************************************************************************/
1097
/*                                DoIO()                                */
1098
/************************************************************************/
1099
1100
void GDALTiledVirtualMem::DoIO(GDALRWFlag eRWFlag, size_t nOffset, void *pPage,
1101
                               size_t nBytes) const
1102
0
{
1103
0
    const int nDataTypeSize = GDALGetDataTypeSizeBytes(eBufType);
1104
0
    const int nTilesPerRow = DIV_ROUND_UP(nXSize, nTileXSize);
1105
0
    const int nTilesPerCol = DIV_ROUND_UP(nYSize, nTileYSize);
1106
0
    size_t nPageSize =
1107
0
        static_cast<size_t>(nTileXSize) * nTileYSize * nDataTypeSize;
1108
0
    if (eTileOrganization != GTO_BSQ)
1109
0
        nPageSize *= nBandCount;
1110
0
    CPLAssert((nOffset % nPageSize) == 0);
1111
0
    CPLAssert(nBytes == nPageSize);
1112
0
    size_t nTile = 0;
1113
0
    int band = 0;
1114
0
    int nPixelSpace = 0;
1115
0
    int nLineSpace = 0;
1116
0
    int nBandSpace = 0;
1117
0
    if (eTileOrganization == GTO_TIP)
1118
0
    {
1119
0
        nTile = nOffset / nPageSize;
1120
0
        band = 0;
1121
0
        nPixelSpace = nDataTypeSize * nBandCount;
1122
0
        nLineSpace = nPixelSpace * nTileXSize;
1123
0
        nBandSpace = nDataTypeSize;
1124
0
    }
1125
0
    else if (eTileOrganization == GTO_BIT)
1126
0
    {
1127
0
        nTile = nOffset / nPageSize;
1128
0
        band = 0;
1129
0
        nPixelSpace = nDataTypeSize;
1130
0
        nLineSpace = nPixelSpace * nTileXSize;
1131
0
        nBandSpace = nLineSpace * nTileYSize;
1132
0
    }
1133
0
    else
1134
0
    {
1135
        // offset = nPageSize * (band * nTilesPerRow * nTilesPerCol + nTile)
1136
0
        band = static_cast<int>(nOffset / (static_cast<size_t>(nPageSize) *
1137
0
                                           nTilesPerRow * nTilesPerCol));
1138
0
        nTile = nOffset / nPageSize -
1139
0
                static_cast<size_t>(band) * nTilesPerRow * nTilesPerCol;
1140
0
        nPixelSpace = nDataTypeSize;
1141
0
        nLineSpace = nPixelSpace * nTileXSize;
1142
0
        nBandSpace = 0;
1143
0
        band++;
1144
0
    }
1145
0
    size_t nYTile = nTile / nTilesPerRow;
1146
0
    size_t nXTile = nTile - nYTile * nTilesPerRow;
1147
1148
0
    int nReqXSize =
1149
0
        std::min(nTileXSize, nXSize - static_cast<int>(nXTile * nTileXSize));
1150
0
    int nReqYSize =
1151
0
        std::min(nTileYSize, nYSize - static_cast<int>(nYTile * nTileYSize));
1152
0
    if (eRWFlag == GF_Read &&
1153
0
        (nReqXSize < nTileXSize || nReqYSize < nTileYSize))
1154
0
        memset(pPage, 0, nBytes);
1155
0
    if (hDS != nullptr)
1156
0
    {
1157
0
        CPL_IGNORE_RET_VAL(GDALDatasetRasterIO(
1158
0
            hDS, eRWFlag, static_cast<int>(nXOff + nXTile * nTileXSize),
1159
0
            static_cast<int>(nYOff + nYTile * nTileYSize), nReqXSize, nReqYSize,
1160
0
            pPage, nReqXSize, nReqYSize, eBufType,
1161
0
            eTileOrganization != GTO_BSQ ? nBandCount : 1,
1162
0
            eTileOrganization != GTO_BSQ ? panBandMap : &band, nPixelSpace,
1163
0
            nLineSpace, nBandSpace));
1164
0
    }
1165
0
    else
1166
0
    {
1167
0
        CPL_IGNORE_RET_VAL(GDALRasterIO(
1168
0
            hBand, eRWFlag, static_cast<int>(nXOff + nXTile * nTileXSize),
1169
0
            static_cast<int>(nYOff + nYTile * nTileYSize), nReqXSize, nReqYSize,
1170
0
            pPage, nReqXSize, nReqYSize, eBufType, nPixelSpace, nLineSpace));
1171
0
    }
1172
0
}
1173
1174
/************************************************************************/
1175
/*                           FillCache()                                */
1176
/************************************************************************/
1177
1178
void GDALTiledVirtualMem::FillCache(CPLVirtualMem *, size_t nOffset,
1179
                                    void *pPageToFill, size_t nToFill,
1180
                                    void *pUserData)
1181
0
{
1182
0
    const GDALTiledVirtualMem *psParams =
1183
0
        static_cast<GDALTiledVirtualMem *>(pUserData);
1184
0
    psParams->DoIO(GF_Read, nOffset, pPageToFill, nToFill);
1185
0
}
1186
1187
/************************************************************************/
1188
/*                          SaveFromCache()                             */
1189
/************************************************************************/
1190
1191
void GDALTiledVirtualMem::SaveFromCache(CPLVirtualMem *, size_t nOffset,
1192
                                        const void *pPageToBeEvicted,
1193
                                        size_t nToEvicted, void *pUserData)
1194
0
{
1195
0
    const GDALTiledVirtualMem *psParams =
1196
0
        static_cast<GDALTiledVirtualMem *>(pUserData);
1197
0
    psParams->DoIO(GF_Write, nOffset, const_cast<void *>(pPageToBeEvicted),
1198
0
                   nToEvicted);
1199
0
}
1200
1201
/************************************************************************/
1202
/*                                Destroy()                             */
1203
/************************************************************************/
1204
1205
void GDALTiledVirtualMem::Destroy(void *pUserData)
1206
0
{
1207
0
    GDALTiledVirtualMem *psParams =
1208
0
        static_cast<GDALTiledVirtualMem *>(pUserData);
1209
0
    delete psParams;
1210
0
}
1211
1212
/************************************************************************/
1213
/*                      GDALGetTiledVirtualMem()                        */
1214
/************************************************************************/
1215
1216
static CPLVirtualMem *GDALGetTiledVirtualMem(
1217
    GDALDatasetH hDS, GDALRasterBandH hBand, GDALRWFlag eRWFlag, int nXOff,
1218
    int nYOff, int nXSize, int nYSize, int nTileXSize, int nTileYSize,
1219
    GDALDataType eBufType, int nBandCount, int *panBandMap,
1220
    GDALTileOrganization eTileOrganization, size_t nCacheSize,
1221
    int bSingleThreadUsage, CSLConstList /* papszOptions */)
1222
0
{
1223
0
    CPLVirtualMem *view;
1224
0
    GDALTiledVirtualMem *psParams;
1225
1226
0
    size_t nPageSize = CPLGetPageSize();
1227
0
    if (nPageSize == 0)
1228
0
    {
1229
0
        CPLError(CE_Failure, CPLE_NotSupported,
1230
0
                 "GDALGetTiledVirtualMem() unsupported on this "
1231
0
                 "operating system / configuration");
1232
0
        return nullptr;
1233
0
    }
1234
1235
0
    int nRasterXSize =
1236
0
        hDS ? GDALGetRasterXSize(hDS) : GDALGetRasterBandXSize(hBand);
1237
0
    int nRasterYSize =
1238
0
        hDS ? GDALGetRasterYSize(hDS) : GDALGetRasterBandYSize(hBand);
1239
1240
0
    if (nXOff < 0 || nYOff < 0 || nTileXSize <= 0 || nTileYSize <= 0 ||
1241
0
        nXOff + nXSize > nRasterXSize || nYOff + nYSize > nRasterYSize)
1242
0
    {
1243
0
        CPLError(CE_Failure, CPLE_AppDefined, "Invalid window request");
1244
0
        return nullptr;
1245
0
    }
1246
1247
0
    if (hDS != nullptr && !GDALCheckBandParameters(hDS, nBandCount, panBandMap))
1248
0
        return nullptr;
1249
1250
0
    const int nDataTypeSize = GDALGetDataTypeSizeBytes(eBufType);
1251
0
    int nTilesPerRow = DIV_ROUND_UP(nXSize, nTileXSize);
1252
0
    int nTilesPerCol = DIV_ROUND_UP(nYSize, nTileYSize);
1253
0
    GUIntBig nReqMem = static_cast<GUIntBig>(nTilesPerRow) * nTilesPerCol *
1254
0
                       nTileXSize * nTileYSize * nBandCount * nDataTypeSize;
1255
#if SIZEOF_SIZE_T == 4
1256
    if (nReqMem != static_cast<GUIntBig>(static_cast<size_t>(nReqMem)))
1257
    {
1258
        CPLError(CE_Failure, CPLE_OutOfMemory,
1259
                 "Cannot reserve " CPL_FRMT_GUIB " bytes", nReqMem);
1260
        return nullptr;
1261
    }
1262
#endif
1263
1264
0
    size_t nPageSizeHint =
1265
0
        static_cast<size_t>(nTileXSize) * nTileYSize * nDataTypeSize;
1266
0
    if (eTileOrganization != GTO_BSQ)
1267
0
        nPageSizeHint *= nBandCount;
1268
0
    if ((nPageSizeHint % nPageSize) != 0)
1269
0
    {
1270
0
        CPLError(CE_Failure, CPLE_AppDefined,
1271
0
                 "Tile dimensions incompatible with page size");
1272
0
        return nullptr;
1273
0
    }
1274
1275
0
    psParams = new GDALTiledVirtualMem(
1276
0
        hDS, hBand, nXOff, nYOff, nXSize, nYSize, nTileXSize, nTileYSize,
1277
0
        eBufType, nBandCount, panBandMap, eTileOrganization);
1278
1279
0
    view = CPLVirtualMemNew(static_cast<size_t>(nReqMem), nCacheSize,
1280
0
                            nPageSizeHint, bSingleThreadUsage,
1281
0
                            eRWFlag == GF_Read ? VIRTUALMEM_READONLY_ENFORCED
1282
0
                                               : VIRTUALMEM_READWRITE,
1283
0
                            GDALTiledVirtualMem::FillCache,
1284
0
                            GDALTiledVirtualMem::SaveFromCache,
1285
0
                            GDALTiledVirtualMem::Destroy, psParams);
1286
1287
0
    if (view == nullptr)
1288
0
    {
1289
0
        delete psParams;
1290
0
    }
1291
0
    else if (CPLVirtualMemGetPageSize(view) != nPageSizeHint)
1292
0
    {
1293
0
        CPLError(CE_Failure, CPLE_AppDefined,
1294
0
                 "Did not get expected page size : %d vs %d",
1295
0
                 static_cast<int>(CPLVirtualMemGetPageSize(view)),
1296
0
                 static_cast<int>(nPageSizeHint));
1297
0
        CPLVirtualMemFree(view);
1298
0
        return nullptr;
1299
0
    }
1300
1301
0
    return view;
1302
0
}
1303
1304
/************************************************************************/
1305
/*                   GDALDatasetGetTiledVirtualMem()                    */
1306
/************************************************************************/
1307
1308
/** Create a CPLVirtualMem object from a GDAL dataset object, with tiling
1309
 * organization
1310
 *
1311
 * Only supported on Linux for now.
1312
 *
1313
 * This method allows creating a virtual memory object for a region of one
1314
 * or more GDALRasterBands from  this dataset. The content of the virtual
1315
 * memory object is automatically filled from dataset content when a virtual
1316
 * memory page is first accessed, and it is released (or flushed in case of a
1317
 * "dirty" page) when the cache size limit has been reached.
1318
 *
1319
 * Contrary to GDALDatasetGetVirtualMem(), pixels will be organized by tiles
1320
 * instead of scanlines. Different ways of organizing pixel within/across tiles
1321
 * can be selected with the eTileOrganization parameter.
1322
 *
1323
 * If nXSize is not a multiple of nTileXSize or nYSize is not a multiple of
1324
 * nTileYSize, partial tiles will exists at the right and/or bottom of the
1325
 * region of interest. Those partial tiles will also have nTileXSize *
1326
 * nTileYSize dimension, with padding pixels.
1327
 *
1328
 * The pointer to access the virtual memory object is obtained with
1329
 * CPLVirtualMemGetAddr(). It remains valid until CPLVirtualMemFree() is called.
1330
 * CPLVirtualMemFree() must be called before the dataset object is destroyed.
1331
 *
1332
 * If p is such a pointer and base_type the C type matching eBufType, for
1333
 * default values of spacing parameters, the element of image coordinates (x, y)
1334
 * (relative to xOff, yOff) for band b can be accessed with:
1335
 *  - for eTileOrganization = GTO_TIP,
1336
 *        ((base_type*)p)[tile_number(x,y)*nBandCount*tile_size +
1337
 *                        offset_in_tile(x,y)*nBandCount + (b-1)].
1338
 *  - for eTileOrganization = GTO_BIT,
1339
 *        ((base_type*)p)[(tile_number(x,y)*nBandCount +
1340
 *                        (b-1)) * tile_size + offset_in_tile(x,y)].
1341
 *  - for eTileOrganization = GTO_BSQ,
1342
 *        ((base_type*)p)[(tile_number(x,y) +
1343
 *                        (b-1)*nTilesCount) * tile_size + offset_in_tile(x,y)].
1344
 *
1345
 * where nTilesPerRow = ceil(nXSize / nTileXSize)
1346
 *       nTilesPerCol = ceil(nYSize / nTileYSize)
1347
 *       nTilesCount = nTilesPerRow * nTilesPerCol
1348
 *       tile_number(x,y) = (y / nTileYSize) * nTilesPerRow + (x / nTileXSize)
1349
 *       offset_in_tile(x,y) = (y % nTileYSize) * nTileXSize  + (x % nTileXSize)
1350
 *       tile_size = nTileXSize * nTileYSize
1351
 *
1352
 * Note that for a single band request, all tile organizations are equivalent.
1353
 *
1354
 * Note that the mechanism used to transparently fill memory pages when they are
1355
 * accessed is the same (but in a controlled way) than what occurs when a memory
1356
 * error occurs in a program. Debugging software will generally interrupt
1357
 * program execution when that happens. If needed, CPLVirtualMemPin() can be
1358
 * used to avoid that by ensuring memory pages are allocated before being
1359
 * accessed.
1360
 *
1361
 * The size of the region that can be mapped as a virtual memory object depends
1362
 * on hardware and operating system limitations.
1363
 * On Linux AMD64 platforms, the maximum value is 128 TB.
1364
 * On Linux x86 platforms, the maximum value is 2 GB.
1365
 *
1366
 * Data type translation is automatically done if the data type
1367
 * (eBufType) of the buffer is different than
1368
 * that of the GDALRasterBand.
1369
 *
1370
 * @param hDS Dataset object
1371
 *
1372
 * @param eRWFlag Either GF_Read to read a region of data, or GF_Write to
1373
 * write a region of data.
1374
 *
1375
 * @param nXOff The pixel offset to the top left corner of the region
1376
 * of the band to be accessed.  This would be zero to start from the left side.
1377
 *
1378
 * @param nYOff The line offset to the top left corner of the region
1379
 * of the band to be accessed.  This would be zero to start from the top.
1380
 *
1381
 * @param nXSize The width of the region of the band to be accessed in pixels.
1382
 *
1383
 * @param nYSize The height of the region of the band to be accessed in lines.
1384
 *
1385
 * @param nTileXSize the width of the tiles.
1386
 *
1387
 * @param nTileYSize the height of the tiles.
1388
 *
1389
 * @param eBufType the type of the pixel values in the data buffer. The
1390
 * pixel values will automatically be translated to/from the GDALRasterBand
1391
 * data type as needed.
1392
 *
1393
 * @param nBandCount the number of bands being read or written.
1394
 *
1395
 * @param panBandMap the list of nBandCount band numbers being read/written.
1396
 * Note band numbers are 1 based. This may be NULL to select the first
1397
 * nBandCount bands.
1398
 *
1399
 * @param eTileOrganization tile organization.
1400
 *
1401
 * @param nCacheSize   size in bytes of the maximum memory that will be really
1402
 *                     allocated (must ideally fit into RAM)
1403
 *
1404
 * @param bSingleThreadUsage set to TRUE if there will be no concurrent threads
1405
 *                           that will access the virtual memory mapping. This
1406
 *                           can optimize performance a bit. If set to FALSE,
1407
 *                           CPLVirtualMemDeclareThread() must be called.
1408
 *
1409
 * @param papszOptions NULL terminated list of options. Unused for now.
1410
 *
1411
 * @return a virtual memory object that must be freed by CPLVirtualMemFree(),
1412
 *         or NULL in case of failure.
1413
 *
1414
 * @since GDAL 1.11
1415
 */
1416
1417
CPLVirtualMem *GDALDatasetGetTiledVirtualMem(
1418
    GDALDatasetH hDS, GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize,
1419
    int nYSize, int nTileXSize, int nTileYSize, GDALDataType eBufType,
1420
    int nBandCount, int *panBandMap, GDALTileOrganization eTileOrganization,
1421
    size_t nCacheSize, int bSingleThreadUsage, CSLConstList papszOptions)
1422
0
{
1423
0
    return GDALGetTiledVirtualMem(hDS, nullptr, eRWFlag, nXOff, nYOff, nXSize,
1424
0
                                  nYSize, nTileXSize, nTileYSize, eBufType,
1425
0
                                  nBandCount, panBandMap, eTileOrganization,
1426
0
                                  nCacheSize, bSingleThreadUsage, papszOptions);
1427
0
}
1428
1429
/************************************************************************/
1430
/*                   GDALRasterBandGetTiledVirtualMem()                 */
1431
/************************************************************************/
1432
1433
/** Create a CPLVirtualMem object from a GDAL rasterband object, with tiling
1434
 * organization
1435
 *
1436
 * Only supported on Linux for now.
1437
 *
1438
 * This method allows creating a virtual memory object for a region of one
1439
 * GDALRasterBand. The content of the virtual
1440
 * memory object is automatically filled from dataset content when a virtual
1441
 * memory page is first accessed, and it is released (or flushed in case of a
1442
 * "dirty" page) when the cache size limit has been reached.
1443
 *
1444
 * Contrary to GDALDatasetGetVirtualMem(), pixels will be organized by tiles
1445
 * instead of scanlines.
1446
 *
1447
 * If nXSize is not a multiple of nTileXSize or nYSize is not a multiple of
1448
 * nTileYSize, partial tiles will exists at the right and/or bottom of the
1449
 * region of interest. Those partial tiles will also have nTileXSize *
1450
 * nTileYSize dimension, with padding pixels.
1451
 *
1452
 * The pointer to access the virtual memory object is obtained with
1453
 * CPLVirtualMemGetAddr(). It remains valid until CPLVirtualMemFree() is called.
1454
 * CPLVirtualMemFree() must be called before the raster band object is
1455
 * destroyed.
1456
 *
1457
 * If p is such a pointer and base_type the C type matching eBufType, for
1458
 * default values of spacing parameters, the element of image coordinates (x, y)
1459
 * (relative to xOff, yOff) can be accessed with:
1460
 *  ((base_type*)p)[tile_number(x,y)*tile_size + offset_in_tile(x,y)].
1461
 *
1462
 * where nTilesPerRow = ceil(nXSize / nTileXSize)
1463
 *       nTilesCount = nTilesPerRow * nTilesPerCol
1464
 *       tile_number(x,y) = (y / nTileYSize) * nTilesPerRow + (x / nTileXSize)
1465
 *       offset_in_tile(x,y) = (y % nTileYSize) * nTileXSize  + (x % nTileXSize)
1466
 *       tile_size = nTileXSize * nTileYSize
1467
 *
1468
 * Note that the mechanism used to transparently fill memory pages when they are
1469
 * accessed is the same (but in a controlled way) than what occurs when a memory
1470
 * error occurs in a program. Debugging software will generally interrupt
1471
 * program execution when that happens. If needed, CPLVirtualMemPin() can be
1472
 * used to avoid that by ensuring memory pages are allocated before being
1473
 * accessed.
1474
 *
1475
 * The size of the region that can be mapped as a virtual memory object depends
1476
 * on hardware and operating system limitations.
1477
 * On Linux AMD64 platforms, the maximum value is 128 TB.
1478
 * On Linux x86 platforms, the maximum value is 2 GB.
1479
 *
1480
 * Data type translation is automatically done if the data type
1481
 * (eBufType) of the buffer is different than
1482
 * that of the GDALRasterBand.
1483
 *
1484
 * @param hBand Rasterband object
1485
 *
1486
 * @param eRWFlag Either GF_Read to read a region of data, or GF_Write to
1487
 * write a region of data.
1488
 *
1489
 * @param nXOff The pixel offset to the top left corner of the region
1490
 * of the band to be accessed.  This would be zero to start from the left side.
1491
 *
1492
 * @param nYOff The line offset to the top left corner of the region
1493
 * of the band to be accessed.  This would be zero to start from the top.
1494
 *
1495
 * @param nXSize The width of the region of the band to be accessed in pixels.
1496
 *
1497
 * @param nYSize The height of the region of the band to be accessed in lines.
1498
 *
1499
 * @param nTileXSize the width of the tiles.
1500
 *
1501
 * @param nTileYSize the height of the tiles.
1502
 *
1503
 * @param eBufType the type of the pixel values in the data buffer. The
1504
 * pixel values will automatically be translated to/from the GDALRasterBand
1505
 * data type as needed.
1506
 *
1507
 * @param nCacheSize   size in bytes of the maximum memory that will be really
1508
 *                     allocated (must ideally fit into RAM)
1509
 *
1510
 * @param bSingleThreadUsage set to TRUE if there will be no concurrent threads
1511
 *                           that will access the virtual memory mapping. This
1512
 *                           can optimize performance a bit. If set to FALSE,
1513
 *                           CPLVirtualMemDeclareThread() must be called.
1514
 *
1515
 * @param papszOptions NULL terminated list of options. Unused for now.
1516
 *
1517
 * @return a virtual memory object that must be freed by CPLVirtualMemFree(),
1518
 *         or NULL in case of failure.
1519
 *
1520
 * @since GDAL 1.11
1521
 */
1522
1523
CPLVirtualMem *GDALRasterBandGetTiledVirtualMem(
1524
    GDALRasterBandH hBand, GDALRWFlag eRWFlag, int nXOff, int nYOff, int nXSize,
1525
    int nYSize, int nTileXSize, int nTileYSize, GDALDataType eBufType,
1526
    size_t nCacheSize, int bSingleThreadUsage, CSLConstList papszOptions)
1527
0
{
1528
0
    return GDALGetTiledVirtualMem(nullptr, hBand, eRWFlag, nXOff, nYOff, nXSize,
1529
0
                                  nYSize, nTileXSize, nTileYSize, eBufType, 1,
1530
0
                                  nullptr, GTO_BSQ, nCacheSize,
1531
0
                                  bSingleThreadUsage, papszOptions);
1532
0
}