Coverage Report

Created: 2022-11-18 06:58

/src/netcdf-c/libnczarr/zwalk.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
Additonal optimizations:
3
1. slice covers all of exactly one chunk: we can just tranfer whole chunk to/from memory
4
5
*/
6
/*********************************************************************
7
 *   Copyright 2018, UCAR/Unidata
8
 *   See netcdf/COPYRIGHT file for copying and redistribution conditions.
9
 *********************************************************************/
10
#include "zincludes.h"
11
12
#define WDEBUG
13
#undef DFALTOPTIMIZE
14
15
static int initialized = 0;
16
17
static unsigned int optimize = 0;
18
19
extern int NCZ_buildchunkkey(size_t R, const size64_t* chunkindices, char** keyp);
20
21
/* 0 => no debug */
22
static unsigned int wdebug = 1;
23
24
/* Forward */
25
static int NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, void* chunkdata);
26
static int rangecount(NCZChunkRange range);
27
static int readfromcache(void* source, size64_t* chunkindices, void** chunkdata);
28
static int iswholechunk(struct Common* common,NCZSlice*);
29
static int wholechunk_indices(struct Common* common, NCZSlice* slices, size64_t* chunkindices);
30
31
const char*
32
astype(int typesize, void* ptr)
33
0
{
34
0
    switch(typesize) {
35
0
    case 4: {
36
0
  static char is[8];
37
0
  snprintf(is,sizeof(is),"%u",*((unsigned int*)ptr));
38
0
  return is;
39
0
        } break;
40
0
    default: break;
41
0
    }
42
0
    return "?";
43
0
}
44
45
/**************************************************/
46
int
47
ncz_chunking_init(void)
48
0
{
49
0
    const char* val = NULL;
50
#ifdef DFALTOPTIMIZE
51
    val = getenv("NCZ_NOOPTIMIZATION");
52
    optimize = (val == NULL ? 1 : 0);
53
#else
54
0
    optimize = 0;
55
0
#endif
56
0
    val = getenv("NCZ_WDEBUG");
57
0
    wdebug = (val == NULL ? 0 : atoi(val));
58
0
#ifdef WDEBUG
59
0
    if(wdebug > 0) fprintf(stderr,"wdebug=%u\n",wdebug);
60
0
#endif
61
0
    initialized = 1;
62
0
    return NC_NOERR;
63
0
}
64
65
/**************************************************/
66
67
/**
68
Goal: Given the slices being applied to the variable, create
69
and walk all possible combinations of projection vectors that
70
can be evaluated to provide the output data.
71
Note that we do not actually pass NCZSlice but rather
72
(start,count,stride) vectors.
73
74
@param var Controlling variable
75
@param usreading reading vs writing
76
@param start start vector
77
@param stop stop vector
78
@param stride stride vector
79
@param memory target or source of data
80
@param typecode nc_type of type being written
81
@param walkfcn fcn parameter to actually transfer data
82
*/
83
84
int
85
NCZ_transferslice(NC_VAR_INFO_T* var, int reading,
86
      size64_t* start, size64_t* count, size64_t* stride,
87
      void* memory, nc_type typecode)
88
0
{
89
0
    int r,stat = NC_NOERR;
90
0
    size64_t dimlens[NC_MAX_VAR_DIMS];
91
0
    size64_t chunklens[NC_MAX_VAR_DIMS];
92
0
    size64_t memshape[NC_MAX_VAR_DIMS];
93
0
    NCZSlice slices[NC_MAX_VAR_DIMS];
94
0
    struct Common common;
95
0
    NCZ_FILE_INFO_T* zfile = NULL;
96
0
    NCZ_VAR_INFO_T* zvar = NULL;
97
0
    size_t typesize;
98
99
0
    if(!initialized) ncz_chunking_init();
100
101
0
    if((stat = NC4_inq_atomic_type(typecode, NULL, &typesize))) goto done;
102
103
0
    if(wdebug >= 1) {
104
0
        size64_t stopvec[NC_MAX_VAR_DIMS];
105
0
  for(r=0;r<var->ndims;r++) stopvec[r] = start[r]+(count[r]*stride[r]);
106
0
        fprintf(stderr,"var: name=%s",var->hdr.name);
107
0
        fprintf(stderr," start=%s",nczprint_vector(var->ndims,start));
108
0
        fprintf(stderr," count=%s",nczprint_vector(var->ndims,count));
109
0
        fprintf(stderr," stop=%s",nczprint_vector(var->ndims,stopvec));
110
0
        fprintf(stderr," stride=%s\n",nczprint_vector(var->ndims,stride));
111
0
    }
112
113
    /* Fill in common */
114
0
    memset(&common,0,sizeof(common));
115
0
    common.var = var;
116
0
    common.file = (var->container)->nc4_info;
117
0
    zfile = common.file->format_file_info;
118
0
    zvar = common.var->format_var_info;
119
120
0
    common.reading = reading;
121
0
    common.memory = memory;
122
0
    common.typesize = typesize;
123
0
    common.cache = zvar->cache;
124
125
    /* We need to talk scalar into account */
126
0
    common.rank = var->ndims;
127
0
    common.scalar = zvar->scalar;
128
0
    common.swap = (zfile->native_endianness == var->endianness ? 0 : 1);
129
130
0
    common.chunkcount = 1;
131
0
    if(common.scalar) {
132
0
  dimlens[0] = 1;
133
0
  chunklens[0] = 1;
134
0
  slices[0].start = 0;
135
0
  slices[0].stride = 1;
136
0
  slices[0].stop = 0;
137
0
  slices[0].len = 1;
138
0
  common.chunkcount = 1;
139
0
  memshape[0] = 1;
140
0
    } else for(r=0;r<common.rank;r++) {
141
0
      dimlens[r] = var->dim[r]->len;
142
0
  chunklens[r] = var->chunksizes[r];
143
0
  slices[r].start = start[r];
144
0
  slices[r].stride = stride[r];
145
0
  slices[r].stop = minimum(start[r]+(count[r]*stride[r]),dimlens[r]);
146
0
  slices[r].len = dimlens[r];
147
0
  common.chunkcount *= chunklens[r];
148
0
  memshape[r] = count[r];
149
0
    }
150
151
0
    if(wdebug >= 1) {
152
0
        fprintf(stderr,"\trank=%d",common.rank);
153
0
        if(!common.scalar) {
154
0
            fprintf(stderr," dimlens=%s",nczprint_vector(common.rank,dimlens));
155
0
            fprintf(stderr," chunklens=%s",nczprint_vector(common.rank,chunklens));
156
0
            fprintf(stderr," memshape=%s",nczprint_vector(common.rank,memshape));
157
0
        }
158
0
  fprintf(stderr,"\n");
159
0
    }
160
0
    common.dimlens = dimlens; /* BAD: storing stack vector in a pointer; do not free */
161
0
    common.chunklens = chunklens; /* ditto */
162
0
    common.memshape = memshape; /* ditto */
163
0
    common.reader.source = ((NCZ_VAR_INFO_T*)(var->format_var_info))->cache;
164
0
    common.reader.read = readfromcache;
165
166
0
    if(common.scalar) {
167
0
        if((stat = NCZ_transferscalar(&common))) goto done;
168
0
    }
169
0
    else {
170
0
        if((stat = NCZ_transfer(&common, slices))) goto done;
171
0
    }
172
0
done:
173
0
    NCZ_clearcommon(&common);
174
0
    return stat;
175
0
}
176
177
/*
178
Walk the possible projections.
179
Broken out so we can use it for unit testing
180
@param common common parameters
181
@param slices
182
*/
183
int
184
NCZ_transfer(struct Common* common, NCZSlice* slices)
185
0
{
186
0
    int stat = NC_NOERR;
187
0
    NCZOdometer* chunkodom =  NULL;
188
0
    NCZOdometer* slpodom = NULL;
189
0
    NCZOdometer* memodom = NULL;
190
0
    void* chunkdata = NULL;
191
0
    int wholechunk = 0;
192
193
    /*
194
     We will need three sets of odometers.
195
     1. Chunk odometer to walk the chunk ranges to get all possible
196
        combinations of chunkranges over all dimensions.
197
     2. For each chunk odometer set of indices, we need a projection
198
        odometer that walks the set of projection slices for a given
199
        set of chunk ranges over all dimensions.
200
     3. A memory odometer that walks the memory data to specify
201
        the locations in memory for read/write
202
    */     
203
204
0
    if(wdebug >= 2) {
205
0
  fprintf(stderr,"slices=%s\n",nczprint_slices(common->rank,slices));
206
0
    }
207
208
0
    if((stat = NCZ_projectslices(common->dimlens, common->chunklens, slices,
209
0
      common, &chunkodom)))
210
0
  goto done;
211
212
0
    if(wdebug >= 4) {
213
0
  fprintf(stderr,"allprojections:\n%s",nczprint_allsliceprojections(common->rank,common->allprojections)); fflush(stderr);
214
0
    }
215
216
0
    wholechunk = iswholechunk(common,slices);
217
218
0
    if(wholechunk) {
219
        /* Implement a whole chunk read optimization; this is a rare occurrence
220
           where the the slices cover all of a single chunk.
221
        */
222
0
  size64_t chunkindices[NC_MAX_VAR_DIMS];
223
0
  unsigned char* memptr;
224
0
        unsigned char* slpptr;
225
226
  /* Which chunk are we getting? */
227
0
  if((stat=wholechunk_indices(common,slices,chunkindices))) goto done;
228
0
  if(wdebug >= 1)
229
0
      fprintf(stderr,"case: wholechunk: chunkindices: %s\n",nczprint_vector(common->rank,chunkindices));
230
  /* Read the chunk; handles fixed vs char* strings*/
231
0
        switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
232
0
        case NC_EEMPTY: /* cache created the chunk */
233
0
      break;
234
0
        case NC_NOERR: break;
235
0
        default: goto done;
236
0
        }
237
        /* Figure out memory address */
238
0
  memptr = ((unsigned char*)common->memory);
239
0
  slpptr = ((unsigned char*)chunkdata);
240
0
  if(common->reading) {
241
0
      if((stat=NCZ_copy_data(common->file,common->var->type_info,slpptr,common->chunkcount,!ZCLEAR,memptr))) goto done;
242
0
  } else {
243
0
      if((stat=NCZ_copy_data(common->file,common->var->type_info,memptr,common->chunkcount,ZCLEAR,slpptr))) goto done;
244
0
  }
245
//        transfern(common,slpptr,memptr,common->chunkcount,1,chunkdata);
246
0
        if(zutest && zutest->tests & UTEST_WHOLECHUNK)
247
0
      zutest->print(UTEST_WHOLECHUNK, common, chunkindices);
248
0
  goto done;
249
0
    }
250
251
    /* iterate over the odometer: all combination of chunk
252
       indices in the projections */
253
0
    for(;nczodom_more(chunkodom);) {
254
0
  int r;
255
0
  size64_t* chunkindices = NULL;
256
0
        NCZSlice slpslices[NC_MAX_VAR_DIMS];
257
0
        NCZSlice memslices[NC_MAX_VAR_DIMS];
258
0
        NCZProjection* proj[NC_MAX_VAR_DIMS];
259
0
  size64_t shape[NC_MAX_VAR_DIMS];
260
261
0
  chunkindices = nczodom_indices(chunkodom);
262
0
  if(wdebug >= 1) {
263
0
      fprintf(stderr,"chunkindices: %s\n",nczprint_vector(common->rank,chunkindices));
264
0
  }
265
266
0
  for(r=0;r<common->rank;r++) {
267
0
      NCZSliceProjections* slp = &common->allprojections[r];
268
0
      NCZProjection* projlist = slp->projections;
269
0
      size64_t indexr = chunkindices[r];
270
        /* use chunkindices[r] to find the corresponding projection slice */
271
      /* We must take into account that the chunkindex of projlist[r]
272
               may be greater than zero */
273
      /* note the 2 level indexing */
274
0
      indexr -= slp->range.start;
275
0
      NCZProjection* pr = &projlist[indexr];
276
0
      proj[r] = pr;
277
0
  }
278
279
0
  if(wdebug > 0) {
280
0
        fprintf(stderr,"Selected projections:\n");
281
0
      for(r=0;r<common->rank;r++) {
282
0
            fprintf(stderr,"\t[%d] %s\n",r,nczprint_projection(*proj[r]));
283
0
    shape[r] = proj[r]->iocount;
284
0
      }
285
0
      fprintf(stderr,"\tshape=%s\n",nczprint_vector(common->rank,shape));
286
0
  }
287
288
  /* See if any of the projections is a skip; if so, then move to the next chunk indices */
289
0
  for(r=0;r<common->rank;r++) {
290
0
      if(proj[r]->skip) goto next;
291
0
  }
292
293
0
  for(r=0;r<common->rank;r++) {
294
0
      slpslices[r] = proj[r]->chunkslice;
295
0
      memslices[r] = proj[r]->memslice;
296
0
  }
297
0
  if(zutest && zutest->tests & UTEST_TRANSFER)
298
0
      zutest->print(UTEST_TRANSFER, common, chunkodom, slpslices, memslices);
299
300
        /* Read from cache */
301
0
        stat = common->reader.read(common->reader.source, chunkindices, &chunkdata);
302
0
  switch (stat) {
303
0
        case NC_EEMPTY: /* cache created the chunk */
304
0
      break;
305
0
        case NC_NOERR: break;
306
0
        default: goto done;
307
0
        }
308
309
0
  slpodom = nczodom_fromslices(common->rank,slpslices);
310
0
  memodom = nczodom_fromslices(common->rank,memslices);
311
312
0
  { /* walk with odometer */
313
0
      if(wdebug >= 1)
314
0
      fprintf(stderr,"case: odometer:\n");
315
        /* This is the key action: walk this set of slices and transfer data */
316
0
        if((stat = NCZ_walk(proj,chunkodom,slpodom,memodom,common,chunkdata))) goto done;
317
0
  }
318
0
next:
319
0
        nczodom_free(slpodom); slpodom = NULL;
320
0
        nczodom_free(memodom); memodom = NULL;
321
0
        nczodom_next(chunkodom);
322
0
    }
323
0
done:
324
0
    nczodom_free(slpodom);
325
0
    nczodom_free(memodom);
326
0
    nczodom_free(chunkodom);
327
0
    return stat;
328
0
}
329
330
331
#ifdef WDEBUG
332
static void
333
wdebug2(const struct Common* common, unsigned char* slpptr, unsigned char* memptr, size_t avail, size_t stride, void* chunkdata)
334
0
{
335
0
    unsigned char* slpbase = chunkdata;
336
0
    unsigned char* membase = common->memory;
337
0
    unsigned slpoff = (unsigned)(slpptr - slpbase);
338
0
    unsigned memoff = (unsigned)(memptr - membase);
339
0
    unsigned slpidx = slpoff / common->typesize;
340
0
    unsigned memidx = memoff / common->typesize;
341
0
    unsigned value;
342
343
0
    fprintf(stderr,"wdebug2: %s: [%u/%d] %u->%u",
344
0
      common->reading?"read":"write",
345
0
      (unsigned)avail,
346
0
          (unsigned)stride,
347
0
      (unsigned)(common->reading?slpidx:memidx),
348
0
      (unsigned)(common->reading?memidx:slpidx)
349
0
      );
350
0
    if(common->reading)
351
0
        value = ((unsigned*)slpptr)[0];
352
0
    else
353
0
        value = ((unsigned*)memptr)[0];
354
0
    fprintf(stderr,"; [%u]=%u",(unsigned)(common->reading?slpidx:memidx),value);
355
356
0
    fprintf(stderr,"\n");
357
0
}
358
#else
359
#define wdebug2(common,slpptr,memptr,avail,stride,chunkdata)
360
#endif
361
362
/*
363
@param projv
364
@param chunkodom
365
@param slpodom
366
@param memodom
367
@param common
368
@param chunkdata
369
@return NC_NOERR
370
*/
371
static int
372
NCZ_walk(NCZProjection** projv, NCZOdometer* chunkodom, NCZOdometer* slpodom, NCZOdometer* memodom, const struct Common* common, void* chunkdata)
373
0
{
374
0
    int stat = NC_NOERR;
375
376
0
    for(;;) {
377
0
  size64_t slpoffset = 0;
378
0
  size64_t memoffset = 0;
379
0
  size64_t slpavail = 0;
380
0
  size64_t memavail = 0;
381
0
  size64_t laststride = 0;
382
0
  unsigned char* memptr0 = NULL;
383
0
  unsigned char* slpptr0 = NULL;
384
  
385
386
0
        if(!nczodom_more(slpodom)) break;
387
  
388
0
            if(wdebug >= 3) {
389
0
    fprintf(stderr,"xx.slp: odom: %s\n",nczprint_odom(slpodom));
390
0
    fprintf(stderr,"xx.mem: odom: %s\n",nczprint_odom(memodom));
391
0
      }
392
393
            /* Convert the indices to a linear offset WRT to chunk indices */
394
0
            slpoffset = nczodom_offset(slpodom);
395
0
            memoffset = nczodom_offset(memodom);
396
397
            /* transfer data between these addresses */
398
0
            memptr0 = ((unsigned char*)common->memory)+(memoffset * common->typesize);
399
0
            slpptr0 = ((unsigned char*)chunkdata)+(slpoffset * common->typesize);
400
401
0
      LOG((1,"%s: slpptr0=%p memptr0=%p slpoffset=%llu memoffset=%lld",__func__,slpptr0,memptr0,slpoffset,memoffset));
402
0
      if(zutest && zutest->tests & UTEST_WALK)
403
0
    zutest->print(UTEST_WALK, common, chunkodom, slpodom, memodom);
404
      /* See if we can transfer multiple values at one shot */
405
0
      laststride = slpodom->stride[common->rank-1];
406
0
      if(laststride == 1) {
407
0
          slpavail = nczodom_avail(slpodom); /* How much can we read? */
408
0
          memavail = nczodom_avail(memodom);
409
0
    assert(memavail == slpavail);
410
0
    nczodom_skipavail(slpodom);
411
0
          nczodom_skipavail(memodom);
412
0
      } else {
413
0
          slpavail = 1;
414
0
    memavail = 1;
415
0
      }
416
0
        if(slpavail > 0) {
417
0
if(wdebug > 0) wdebug2(common,slpptr0,memptr0,slpavail,laststride,chunkdata);
418
0
      if(common->reading) {
419
0
        if((stat=NCZ_copy_data(common->file,common->var->type_info,slpptr0,slpavail,!ZCLEAR,memptr0))) goto done;
420
0
    } else {
421
0
        if((stat=NCZ_copy_data(common->file,common->var->type_info,memptr0,slpavail,ZCLEAR,slpptr0))) goto done;
422
0
    }
423
0
      }
424
//      if((stat = transfern(common,slpptr0,memptr0,avail,nczodom_laststride(slpodom),chunkdata)))goto done;
425
0
            nczodom_next(memodom);
426
0
            nczodom_next(slpodom);
427
0
    }
428
0
done:
429
0
    return stat;    
430
0
}
431
432
#if 0
433
static void
434
wdebug1(const struct Common* common, unsigned char* srcptr, unsigned char* dstptr, size_t count, size_t stride, void* chunkdata, const char* tag)
435
{
436
    unsigned char* dstbase = (common->reading?common->memory:chunkdata);
437
    unsigned char* srcbase = (common->reading?chunkdata:common->memory);
438
    unsigned dstoff = (unsigned)(dstptr - dstbase);
439
    unsigned srcoff = (unsigned)(srcptr - srcbase);
440
    unsigned srcidx = srcoff / sizeof(unsigned);
441
442
    fprintf(stderr,"%s: %s: [%u/%d] %u->%u",
443
      tag,
444
      common->reading?"read":"write",
445
      (unsigned)count,
446
          (unsigned)stride,
447
      (unsigned)(srcoff/common->typesize),
448
      (unsigned)(dstoff/common->typesize)
449
      );
450
#if 0
451
    fprintf(stderr,"\t%s[%u]=%u\n",(common->reading?"chunkdata":"memdata"),
452
//      0,((unsigned*)srcptr)[0]
453
        srcidx,((unsigned*)srcbase)[srcidx]
454
  );
455
#endif
456
#if 0
457
    { size_t len = common->typesize*count;
458
    fprintf(stderr," | [%u] %u->%u\n",(unsigned)len,(unsigned)srcoff,(unsigned)dstoff);
459
    }
460
#endif
461
    fprintf(stderr,"\n");
462
}
463
#else
464
#define wdebug1(common,srcptr,dstptr,count,srcstride,dststride,chunkdata,tag)
465
#endif
466
467
#if 0
468
static int
469
transfern(const struct Common* common, unsigned char* slpptr, unsigned char* memptr, size_t avail, size_t slpstride, void* chunkdata)
470
{
471
    int stat = NC_NOERR;
472
    size_t typesize = common->typesize;
473
    size_t len = typesize*avail;
474
    size_t m,s;
475
476
    if(common->reading) {
477
  if(slpstride == 1)
478
            memcpy(memptr,slpptr,len); /* straight copy */
479
  else {
480
      for(m=0,s=0;s<avail;s+=slpstride,m++) {
481
    size_t soffset = s*typesize;
482
    size_t moffset = m*typesize;
483
          memcpy(memptr+moffset,slpptr+soffset,typesize);
484
      }
485
  }
486
        if(common->swap)
487
            NCZ_swapatomicdata(len,memptr,common->typesize);
488
    } else { /*writing*/
489
unsigned char* srcbase = (common->reading?chunkdata:common->memory);
490
unsigned srcoff = (unsigned)(memptr - srcbase);
491
unsigned srcidx = srcoff / sizeof(unsigned); (void)srcidx;
492
  if(slpstride == 1)
493
            memcpy(slpptr,memptr,len); /* straight copy */
494
  else {
495
      for(m=0,s=0;s<avail;s+=slpstride,m++) {
496
    size_t soffset = s*typesize;
497
    size_t moffset = m*typesize;
498
          memcpy(slpptr+soffset,memptr+moffset,typesize);
499
      }
500
  }
501
        if(common->swap)
502
            NCZ_swapatomicdata(len,slpptr,common->typesize);
503
    }
504
    return THROW(stat);
505
}
506
#endif
507
508
#if 0
509
/* This function may not be necessary if code in zvar does it instead */
510
static int
511
NCZ_fillchunk(void* chunkdata, struct Common* common)
512
{
513
    int stat = NC_NOERR;    
514
515
    if(common->fillvalue == NULL) {
516
        memset(chunkdata,0,common->chunkcount*common->typesize);
517
  goto done;
518
    } 
519
520
    if(common->cache->fillchunk == NULL) {
521
        /* Get fill chunk*/
522
        if((stat = NCZ_create_fill_chunk(common->cache->chunksize, common->typesize, common->fillvalue, &common->cache->fillchunk)))
523
      goto done;
524
    }
525
    memcpy(chunkdata,common->cache->fillchunk,common->cache->chunksize);
526
done:
527
    return stat;
528
}
529
#endif
530
531
/* Break out this piece so we can use it for unit testing */
532
int
533
NCZ_projectslices(size64_t* dimlens,
534
                  size64_t* chunklens,
535
                  NCZSlice* slices,
536
                  struct Common* common, 
537
                  NCZOdometer** odomp)
538
0
{
539
0
    int stat = NC_NOERR;
540
0
    int r;
541
0
    NCZOdometer* odom = NULL;
542
0
    NCZSliceProjections* allprojections = NULL;
543
0
    NCZChunkRange ranges[NC_MAX_VAR_DIMS];
544
0
    size64_t start[NC_MAX_VAR_DIMS];
545
0
    size64_t stop[NC_MAX_VAR_DIMS];
546
0
    size64_t stride[NC_MAX_VAR_DIMS];
547
0
    size64_t len[NC_MAX_VAR_DIMS];
548
549
0
    if((allprojections = calloc(common->rank,sizeof(NCZSliceProjections))) == NULL)
550
0
        {stat = NC_ENOMEM; goto done;}
551
0
    memset(ranges,0,sizeof(ranges));
552
553
    /* Package common arguments */
554
0
    common->dimlens = dimlens;
555
0
    common->chunklens = chunklens;
556
    /* Compute the chunk ranges for each slice in a given dim */
557
0
    if((stat = NCZ_compute_chunk_ranges(common->rank,slices,common->chunklens,ranges)))
558
0
        goto done;
559
560
    /* Compute the slice index vector */
561
0
    if((stat=NCZ_compute_all_slice_projections(common,slices,ranges,allprojections)))
562
0
        goto done;
563
564
    /* Verify */
565
0
    for(r=0;r<common->rank;r++) {
566
0
        assert(rangecount(ranges[r]) == allprojections[r].count);
567
0
    }
568
569
    /* Compute the shape vector */
570
0
    for(r=0;r<common->rank;r++) {
571
0
        int j;
572
0
        size64_t iocount = 0;
573
0
        NCZProjection* projections = allprojections[r].projections;
574
0
        for(j=0;j<allprojections[r].count;j++) {
575
0
            NCZProjection* proj = &projections[j];
576
0
            iocount += proj->iocount;
577
0
        }
578
0
        common->shape[r] = iocount;
579
0
    }
580
0
    common->allprojections = allprojections;
581
0
    allprojections = NULL;
582
583
    /* Create an odometer to walk all the range combinations */
584
0
    for(r=0;r<common->rank;r++) {
585
0
        start[r] = ranges[r].start; 
586
0
        stop[r] = ranges[r].stop;
587
0
        stride[r] = 1;
588
0
        len[r] = ceildiv(common->dimlens[r],common->chunklens[r]);
589
0
    }   
590
591
0
    if((odom = nczodom_new(common->rank,start,stop,stride,len)) == NULL)
592
0
        {stat = NC_ENOMEM; goto done;}
593
0
    if(odomp) *odomp = odom;
594
595
0
done:
596
    /* reclaim allprojections if !NULL */
597
0
    if(allprojections != NULL) {
598
0
        NCZ_clearsliceprojections(common->rank,allprojections);
599
0
  nullfree(allprojections);
600
0
    }
601
0
    return stat;
602
0
}
603
604
/***************************************************/
605
/* Utilities */
606
607
static int
608
rangecount(NCZChunkRange range)
609
0
{
610
0
    return (range.stop - range.start);
611
0
}
612
613
/* Goal: Given a set of per-dimension indices,
614
     compute the corresponding linear position.
615
*/
616
size64_t
617
NCZ_computelinearoffset(size_t R, const size64_t* indices, const size64_t* dimlens)
618
0
{
619
0
      size64_t offset;
620
0
      int i;
621
622
0
      offset = 0;
623
0
      for(i=0;i<R;i++) {
624
0
          offset *= dimlens[i];
625
0
          offset += indices[i];
626
0
      } 
627
0
      return offset;
628
0
}
629
630
#if 0
631
/* Goal: Given a linear position
632
     compute the corresponding set of R indices
633
*/
634
void
635
NCZ_offset2indices(size_t R, size64_t offset, const size64_t* dimlens, size64_t* indices)
636
{
637
      int i;
638
639
      for(i=0;i<R;i++) {
640
          indices[i] = offset % dimlens[i];
641
          offset = offset / dimlens[i];
642
      } 
643
}
644
#endif
645
646
/**************************************************/
647
/* Unit test entry points */
648
649
int
650
NCZ_chunkindexodom(int rank, const NCZChunkRange* ranges, size64_t* chunkcounts, NCZOdometer** odomp)
651
0
{
652
0
    int stat = NC_NOERR;
653
0
    int r;
654
0
    NCZOdometer* odom = NULL;
655
0
    size64_t start[NC_MAX_VAR_DIMS];
656
0
    size64_t stop[NC_MAX_VAR_DIMS];
657
0
    size64_t stride[NC_MAX_VAR_DIMS];
658
0
    size64_t len[NC_MAX_VAR_DIMS];
659
660
0
    for(r=0;r<rank;r++) {
661
0
        start[r] = ranges[r].start; 
662
0
        stop[r] = ranges[r].stop;
663
0
        stride[r] = 1;
664
0
        len[r] = chunkcounts[r];
665
0
    }   
666
667
0
    if((odom = nczodom_new(rank, start, stop, stride, len))==NULL)
668
0
        {stat = NC_ENOMEM; goto done;}
669
670
0
    if(odomp) {*odomp = odom; odom = NULL;}
671
672
0
done:
673
0
    nczodom_free(odom);
674
0
    return stat;
675
0
}
676
677
static int
678
readfromcache(void* source, size64_t* chunkindices, void** chunkdatap)
679
0
{
680
0
    return NCZ_read_cache_chunk((struct NCZChunkCache*)source, chunkindices, chunkdatap);
681
0
}
682
683
void
684
NCZ_clearcommon(struct Common* common)
685
0
{
686
0
    NCZ_clearsliceprojections(common->rank,common->allprojections);
687
0
    nullfree(common->allprojections);
688
0
}
689
690
/* Does the User want all of one and only chunk? */
691
static int
692
iswholechunk(struct Common* common, NCZSlice* slices)
693
0
{
694
0
    int i;
695
    
696
    /* Check that slices cover a whole chunk */
697
0
    for(i=0;i<common->rank;i++) {
698
0
  if(!(slices[i].stride == 1                            /* no point skipping              */
699
0
     && (slices[i].start % common->chunklens[i]) == 0 /* starting at beginning of chunk */
700
0
     && (slices[i].stop - slices[i].start)            /* stop-start = edge length       */
701
0
        == common->chunklens[i]                       /* edge length == chunk length    */
702
0
     )) 
703
0
      return 0; /* slices do not cover a whole chunk */
704
0
    }
705
0
    return 1;
706
0
}
707
708
static int
709
wholechunk_indices(struct Common* common, NCZSlice* slices, size64_t* chunkindices)
710
0
{
711
0
    int i;
712
0
    for(i=0;i<common->rank;i++)
713
0
  chunkindices[i] = (slices[i].start / common->chunklens[i]);
714
0
    return NC_NOERR;
715
0
}
716
717
/**************************************************/
718
/* Scalar variable support */
719
720
/*
721
@param common common parameters
722
*/
723
724
int
725
NCZ_transferscalar(struct Common* common)
726
0
{
727
0
    int stat = NC_NOERR;
728
0
    void* chunkdata = NULL;
729
0
    size64_t chunkindices[NC_MAX_VAR_DIMS];
730
0
    unsigned char* memptr, *slpptr;
731
732
    /* Read from single chunk from cache */
733
0
    chunkindices[0] = 0;
734
0
    switch ((stat = common->reader.read(common->reader.source, chunkindices, &chunkdata))) {
735
0
    case NC_EEMPTY: /* cache created the chunk */
736
0
  break;
737
0
    case NC_NOERR: break;
738
0
    default: goto done;
739
0
    }
740
741
    /* Figure out memory address */
742
0
    memptr = ((unsigned char*)common->memory);
743
0
    slpptr = ((unsigned char*)chunkdata);
744
0
    if(common->reading) {
745
0
        if((stat=NCZ_copy_data(common->file,common->var->type_info,slpptr,common->chunkcount,!ZCLEAR,memptr))) goto done;
746
0
    } else {
747
0
        if((stat=NCZ_copy_data(common->file,common->var->type_info,memptr,common->chunkcount,ZCLEAR,slpptr))) goto done;
748
0
    }
749
750
0
done:
751
0
    return stat;
752
0
}
753
754
/* Debugging Interface: return the contents of a specified chunk */
755
EXTERNL int
756
NCZ_read_chunk(int ncid, int varid, size64_t* zindices, void* chunkdata)
757
0
{
758
0
    int stat = NC_NOERR;
759
0
    NC_VAR_INFO_T* var = NULL;
760
0
    NCZ_VAR_INFO_T* zvar = NULL;
761
0
    struct NCZChunkCache* cache = NULL;
762
0
    void* cachedata = NULL;
763
764
0
    if ((stat = nc4_find_grp_h5_var(ncid, varid, NULL, NULL, &var)))
765
0
  return THROW(stat);
766
0
    zvar = (NCZ_VAR_INFO_T*)var->format_var_info;
767
0
    cache = zvar->cache;
768
769
0
    if((stat = NCZ_read_cache_chunk(cache,zindices,&cachedata))) goto done;
770
0
    if(chunkdata) {
771
0
  if((stat = nc_copy_data(ncid,var->type_info->hdr.id,cachedata,cache->chunkcount,chunkdata))) goto done;
772
0
    } 
773
    
774
0
done:
775
0
    return stat;
776
0
}