/src/netcdf-c/libnczarr/zchunking.c
| Line | Count | Source | 
| 1 |  | /********************************************************************* | 
| 2 |  |  *   Copyright 2018, UCAR/Unidata | 
| 3 |  |  *   See netcdf/COPYRIGHT file for copying and redistribution conditions. | 
| 4 |  |  *********************************************************************/ | 
| 5 |  | #include "zincludes.h" | 
| 6 |  |  | 
| 7 |  | #define MAX(a,b) ((a)>(b)?(a):(b)) | 
| 8 |  | #define MIN(a,b) ((a)<(b)?(a):(b)) | 
| 9 |  |  | 
| 10 |  | static int pcounter = 0; | 
| 11 |  |  | 
| 12 |  | /* Forward */ | 
| 13 |  | static int compute_intersection(const NCZSlice* slice, size64_t chunklen, unsigned char isunlimited, NCZChunkRange* range); | 
| 14 |  | static void skipchunk(const NCZSlice* slice, NCZProjection* projection); | 
| 15 |  | static int verifyslice(const NCZSlice* slice); | 
| 16 |  |  | 
| 17 |  | /**************************************************/ | 
| 18 |  | /* Goal:create a vector of chunk ranges: one for each slice in | 
| 19 |  |    the top-level input. For each slice, compute the index (not | 
| 20 |  |    absolute position) of the first chunk that intersects the slice | 
| 21 |  |    and the index of the last chunk that intersects the slice. | 
| 22 |  |    In practice, the count = last - first + 1 is stored instead of the last index. | 
| 23 |  |    Note that this n-dim array of indices may have holes in it if the slice stride | 
| 24 |  |    is greater than the chunk length. | 
| 25 |  |    @param rank variable rank | 
| 26 |  |    @param slices the complete set of slices |slices| == R | 
| 27 |  |    @param ncr (out) the vector of computed chunk ranges. | 
| 28 |  |    @return NC_EXXX error code | 
| 29 |  | */ | 
| 30 |  | int | 
| 31 |  | NCZ_compute_chunk_ranges( | 
| 32 |  |   struct Common* common, | 
| 33 |  |         const NCZSlice* slices, /* the complete set of slices |slices| == R*/ | 
| 34 |  |         NCZChunkRange* ncr) | 
| 35 | 0 | { | 
| 36 | 0 |     int stat = NC_NOERR; | 
| 37 | 0 |     int i; | 
| 38 | 0 |     int rank = common->rank; | 
| 39 |  | 
 | 
| 40 | 0 |     for(i=0;i<rank;i++) { | 
| 41 | 0 |   if((stat = compute_intersection(&slices[i],common->chunklens[i],common->isunlimited[i],&ncr[i]))) | 
| 42 | 0 |       goto done; | 
| 43 | 0 |     } | 
| 44 |  |  | 
| 45 | 0 | done: | 
| 46 | 0 |     return stat; | 
| 47 | 0 | } | 
| 48 |  |  | 
| 49 |  | /** | 
| 50 |  | @param Compute chunk range for a single slice. | 
| 51 |  | @param chunklen size of the chunk | 
| 52 |  | @param isunlimited if corresponding dim is unlimited | 
| 53 |  | @param range (out) the range of chunks covered by this slice | 
| 54 |  | @return NC_EXX error code | 
| 55 |  | */ | 
| 56 |  | static int | 
| 57 |  | compute_intersection( | 
| 58 |  |         const NCZSlice* slice, | 
| 59 |  |   size64_t chunklen, | 
| 60 |  |   unsigned char isunlimited, | 
| 61 |  |         NCZChunkRange* range) | 
| 62 | 0 | { | 
| 63 | 0 |     range->start = floordiv(slice->start, chunklen); | 
| 64 | 0 |     range->stop = ceildiv(slice->stop, chunklen); | 
| 65 | 0 |     return NC_NOERR; | 
| 66 | 0 | } | 
| 67 |  |  | 
| 68 |  | /** | 
| 69 |  | Compute the projection of a slice as applied to n'th chunk. | 
| 70 |  | A projection defines the set of grid points touched within a | 
| 71 |  | chunk by a slice. This set of points is the "projection" | 
| 72 |  | of the slice onto the chunk. | 
| 73 |  | This is somewhat complex because: | 
| 74 |  | 1. for the first projection, the start is the slice start, | 
| 75 |  |    but after that, we have to take into account that for | 
| 76 |  |    a non-one stride, the start point in a projection may | 
| 77 |  |    be offset by some value in the range of 0..(slice.stride-1). | 
| 78 |  | 2. The stride might be so large as to completely skip some chunks. | 
| 79 |  |  | 
| 80 |  | @return NC_NOERR if ok | 
| 81 |  | @return NC_ERANGE if chunk skipped | 
| 82 |  | @return NC_EXXXX if failed | 
| 83 |  |  | 
| 84 |  | */ | 
| 85 |  |  | 
| 86 |  | int | 
| 87 |  | NCZ_compute_projections(struct Common* common,  int r, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections) | 
| 88 | 0 | { | 
| 89 | 0 |     int stat = NC_NOERR; | 
| 90 | 0 |     NCZProjection* projection = NULL; | 
| 91 | 0 |     NCZProjection* prev = NULL; | 
| 92 | 0 |     size64_t dimlen = common->dimlens[r]; /* the dimension length for r'th dimension */ | 
| 93 | 0 |     size64_t chunklen = common->chunklens[r]; /* the chunk length corresponding to the dimension */ | 
| 94 | 0 |     size64_t abslimit; | 
| 95 |  | 
 | 
| 96 | 0 |     projection = &projections[n]; | 
| 97 | 0 |     if(n > 0) { | 
| 98 |  |   /* Find last non-skipped projection */ | 
| 99 | 0 |   for(size_t i=n;i-->0;) { /* walk backward */ | 
| 100 | 0 |             if(!projections[i].skip) { | 
| 101 | 0 |           prev = &projections[i]; | 
| 102 | 0 |     break; | 
| 103 | 0 |       } | 
| 104 | 0 |   } | 
| 105 | 0 |   if(prev == NULL) {stat = NC_ENCZARR; goto done;} | 
| 106 | 0 |     } | 
| 107 |  |  | 
| 108 | 0 |     projection->id = ++pcounter; | 
| 109 | 0 |     projection->chunkindex = chunkindex; | 
| 110 |  | 
 | 
| 111 | 0 |     projection->offset = chunklen * chunkindex; /* with respect to dimension (WRD) */ | 
| 112 |  |  | 
| 113 |  |     /* limit in the n'th touched chunk, taking dimlen and stride->stop into account. */ | 
| 114 | 0 |     abslimit = (chunkindex + 1) * chunklen; | 
| 115 | 0 |     if(abslimit > slice->stop) abslimit = slice->stop; | 
| 116 | 0 |     if(abslimit > dimlen) abslimit = dimlen; | 
| 117 | 0 |     projection->limit = abslimit - projection->offset; | 
| 118 |  |  | 
| 119 |  |     /*  See if the next point after the last one in prev lands in the current projection. | 
| 120 |  |   If not, then we have skipped the current chunk. Also take limit into account. | 
| 121 |  |   Note by definition, n must be greater than zero because we always start in a relevant chunk. | 
| 122 |  |   */ | 
| 123 |  | 
 | 
| 124 | 0 |     if(n == 0) { | 
| 125 |  |   /*initial case: original slice start is in 1st projection */ | 
| 126 | 0 |   projection->first = slice->start - projection->offset; | 
| 127 | 0 |   projection->iopos = 0; | 
| 128 | 0 |     } else { /* n > 0 */ | 
| 129 |  |        /* Use absolute offsets for these computations to avoid negative values */ | 
| 130 | 0 |        size64_t abslastpoint, absnextpoint, absthislast; | 
| 131 |  |  | 
| 132 |  |         /* abs last point touched in prev projection */ | 
| 133 | 0 |         abslastpoint = prev->offset + prev->last; | 
| 134 |  |  | 
| 135 |  |   /* Compute the abs last touchable point in this chunk */ | 
| 136 | 0 |         absthislast = projection->offset + projection->limit; | 
| 137 |  |    | 
| 138 |  |         /* Compute next point touched after the last point touched in previous projection; | 
| 139 |  |      note that the previous projection might be wrt a chunk other than the immediately preceding | 
| 140 |  |      one (because the intermediate ones were skipped). | 
| 141 |  |         */ | 
| 142 | 0 |   absnextpoint = abslastpoint + slice->stride; /* abs next point to be touched */ | 
| 143 |  | 
 | 
| 144 | 0 |   if(absnextpoint >= absthislast) { /* this chunk is being skipped */ | 
| 145 | 0 |       skipchunk(slice,projection); | 
| 146 | 0 |       goto done; | 
| 147 | 0 |   } | 
| 148 |  |  | 
| 149 |  |         /* Compute start point in this chunk */ | 
| 150 |  |   /* basically absnextpoint - abs start of this projection */ | 
| 151 | 0 |   projection->first = absnextpoint - projection->offset; | 
| 152 |  |  | 
| 153 |  |   /* Compute the memory location of this first point in this chunk */ | 
| 154 | 0 |   projection->iopos = ceildiv((projection->offset - slice->start),slice->stride); | 
| 155 |  | 
 | 
| 156 | 0 |     } | 
| 157 | 0 |     if(slice->stop > abslimit) | 
| 158 | 0 |   projection->stop = chunklen; | 
| 159 | 0 |     else | 
| 160 | 0 |   projection->stop = slice->stop - projection->offset; | 
| 161 |  | 
 | 
| 162 | 0 |     projection->iocount = ceildiv((projection->stop - projection->first),slice->stride); | 
| 163 |  |  | 
| 164 |  |     /* Compute the slice relative to this chunk. | 
| 165 |  |        Recall the possibility that start+stride >= projection->limit */ | 
| 166 | 0 |     projection->chunkslice.start = projection->first; | 
| 167 | 0 |     projection->chunkslice.stop = projection->stop; | 
| 168 | 0 |     projection->chunkslice.stride = slice->stride; | 
| 169 | 0 |     projection->chunkslice.len = chunklen; | 
| 170 |  |  | 
| 171 |  |     /* Last place to be touched */ | 
| 172 | 0 |     projection->last = projection->first + (slice->stride * (projection->iocount - 1)); | 
| 173 |  | 
 | 
| 174 | 0 |     projection->memslice.start = projection->iopos; | 
| 175 | 0 |     projection->memslice.stop = projection->iopos + projection->iocount; | 
| 176 | 0 |     projection->memslice.stride = 1; | 
| 177 |  | //    projection->memslice.stride = slice->stride; | 
| 178 |  | //    projection->memslice.len = projection->memslice.stop; | 
| 179 | 0 |     projection->memslice.len = common->memshape[r]; | 
| 180 |  | #ifdef NEVERUSE | 
| 181 |  |    projection->memslice.len = dimlen; | 
| 182 |  |    projection->memslice.len = chunklen; | 
| 183 |  | #endif | 
| 184 |  | 
 | 
| 185 | 0 |     if(!verifyslice(&projection->memslice) || !verifyslice(&projection->chunkslice)) | 
| 186 | 0 |         {stat = NC_ECONSTRAINT; goto done;} | 
| 187 |  |  | 
| 188 | 0 | done: | 
| 189 | 0 |     return stat; | 
| 190 | 0 | } | 
| 191 |  |  | 
| 192 |  | static void | 
| 193 |  | skipchunk(const NCZSlice* slice, NCZProjection* projection) | 
| 194 | 0 | { | 
| 195 | 0 |     projection->skip = 1; | 
| 196 | 0 |     projection->first = 0; | 
| 197 | 0 |     projection->last = 0; | 
| 198 | 0 |     projection->iopos = ceildiv(projection->offset - slice->start, slice->stride); | 
| 199 | 0 |     projection->iocount = 0; | 
| 200 | 0 |     projection->chunkslice.start = 0; | 
| 201 | 0 |     projection->chunkslice.stop = 0; | 
| 202 | 0 |     projection->chunkslice.stride = 1; | 
| 203 | 0 |     projection->chunkslice.len = 0; | 
| 204 | 0 |     projection->memslice.start = 0; | 
| 205 | 0 |     projection->memslice.stop = 0; | 
| 206 | 0 |     projection->memslice.stride = 1; | 
| 207 | 0 |     projection->memslice.len = 0; | 
| 208 | 0 | } | 
| 209 |  |  | 
| 210 |  | /* Goal: | 
| 211 |  | Create a vector of projections wrt a slice and a sequence of chunks. | 
| 212 |  | */ | 
| 213 |  |  | 
| 214 |  | int | 
| 215 |  | NCZ_compute_per_slice_projections( | 
| 216 |  |   struct Common* common, | 
| 217 |  |   int r, /* which dimension are we projecting? */ | 
| 218 |  |         const NCZSlice* slice, /* the slice for which projections are computed */ | 
| 219 |  |   const NCZChunkRange* range, /* range */ | 
| 220 |  |   NCZSliceProjections* slp) | 
| 221 | 0 | { | 
| 222 | 0 |     int stat = NC_NOERR; | 
| 223 | 0 |     size64_t index,slicecount; | 
| 224 | 0 |     size_t n; | 
| 225 |  |  | 
| 226 |  |     /* Part fill the Slice Projections */ | 
| 227 | 0 |     slp->r = r; | 
| 228 | 0 |     slp->range = *range; | 
| 229 | 0 |     slp->count = range->stop - range->start; | 
| 230 | 0 |     if((slp->projections = calloc(slp->count,sizeof(NCZProjection))) == NULL) | 
| 231 | 0 |   {stat = NC_ENOMEM; goto done;} | 
| 232 |  |  | 
| 233 |  |     /* Compute the total number of output items defined by this slice | 
| 234 |  |            (equivalent to count as used by nc_get_vars) */ | 
| 235 | 0 |     slicecount = ceildiv((slice->stop - slice->start), slice->stride); | 
| 236 | 0 |     if(slicecount < 0) slicecount = 0; | 
| 237 |  |  | 
| 238 |  |     /* Iterate over each chunk that intersects slice to produce projection */ | 
| 239 | 0 |     for(n=0,index=range->start;index<range->stop;index++,n++) { | 
| 240 | 0 |   if((stat = NCZ_compute_projections(common, r, index, slice, n, slp->projections)))  | 
| 241 | 0 |       goto done; /* something went wrong */ | 
| 242 | 0 |     } | 
| 243 |  |  | 
| 244 | 0 | done: | 
| 245 | 0 |     return stat; | 
| 246 | 0 | } | 
| 247 |  |  | 
| 248 |  | /* Goal:create a vector of SliceProjection instances: one for each | 
| 249 |  |     slice in the top-level input. For each slice, compute a set | 
| 250 |  |     of projections from it wrt a dimension and a chunk size | 
| 251 |  |     associated with that dimension. | 
| 252 |  | */ | 
| 253 |  | int | 
| 254 |  | NCZ_compute_all_slice_projections( | 
| 255 |  |   struct Common* common, | 
| 256 |  |         const NCZSlice* slices, /* the complete set of slices |slices| == R*/ | 
| 257 |  |         const NCZChunkRange* ranges, | 
| 258 |  |         NCZSliceProjections* results) | 
| 259 | 0 | { | 
| 260 | 0 |     int stat = NC_NOERR; | 
| 261 | 0 |     int r;  | 
| 262 |  | 
 | 
| 263 | 0 |     for(r=0;r<common->rank;r++) { | 
| 264 |  |   /* Compute each of the rank SliceProjections instances */ | 
| 265 | 0 |   NCZSliceProjections* slp = &results[r]; | 
| 266 | 0 |         if((stat=NCZ_compute_per_slice_projections( | 
| 267 | 0 |           common, | 
| 268 | 0 |           r, | 
| 269 | 0 |           &slices[r], | 
| 270 | 0 |           &ranges[r], | 
| 271 | 0 |                                         slp))) goto done; | 
| 272 | 0 |     } | 
| 273 |  |  | 
| 274 | 0 | done: | 
| 275 | 0 |     return stat; | 
| 276 | 0 | } | 
| 277 |  |  | 
| 278 |  | /**************************************************/ | 
| 279 |  | /* Utilities */ | 
| 280 |  |      | 
| 281 |  | /* return 0 if slice is malformed; 1 otherwise */ | 
| 282 |  | static int | 
| 283 |  | verifyslice(const NCZSlice* slice) | 
| 284 | 0 | { | 
| 285 | 0 |     if(slice->stop < slice->start) return 0; | 
| 286 | 0 |     if(slice->stride <= 0) return 0; | 
| 287 | 0 |     if((slice->stop - slice->start) > slice->len) return 0; | 
| 288 | 0 |     return 1;     | 
| 289 | 0 | } | 
| 290 |  |  | 
| 291 |  | void | 
| 292 |  | NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv) | 
| 293 | 0 | { | 
| 294 | 0 |     if(slpv != NULL) { | 
| 295 | 0 |   int i; | 
| 296 | 0 |         for(i=0;i<count;i++) { | 
| 297 | 0 |       NCZSliceProjections* slp = &slpv[i]; | 
| 298 |  |       nullfree(slp->projections);  | 
| 299 | 0 |   } | 
| 300 | 0 |     } | 
| 301 | 0 | } | 
| 302 |  |  | 
| 303 |  | #if 0 | 
| 304 |  | static void | 
| 305 |  | clearallprojections(NCZAllProjections* nap) | 
| 306 |  | { | 
| 307 |  |     if(nap != NULL) { | 
| 308 |  |   int i; | 
| 309 |  |   for(i=0;i<nap->rank;i++)  | 
| 310 |  |       nclistfreeall(&nap->allprojections[i].projections); | 
| 311 |  |     } | 
| 312 |  | } | 
| 313 |  | #endif |