/src/netcdf-c/libnczarr/zchunking.c
Line | Count | Source |
1 | | /********************************************************************* |
2 | | * Copyright 2018, UCAR/Unidata |
3 | | * See netcdf/COPYRIGHT file for copying and redistribution conditions. |
4 | | *********************************************************************/ |
5 | | #include "zincludes.h" |
6 | | |
7 | | #define MAX(a,b) ((a)>(b)?(a):(b)) |
8 | | #define MIN(a,b) ((a)<(b)?(a):(b)) |
9 | | |
10 | | static int pcounter = 0; |
11 | | |
12 | | /* Forward */ |
13 | | static int compute_intersection(const NCZSlice* slice, size64_t chunklen, unsigned char isunlimited, NCZChunkRange* range); |
14 | | static void skipchunk(const NCZSlice* slice, NCZProjection* projection); |
15 | | static int verifyslice(const NCZSlice* slice); |
16 | | |
17 | | /**************************************************/ |
18 | | /* Goal:create a vector of chunk ranges: one for each slice in |
19 | | the top-level input. For each slice, compute the index (not |
20 | | absolute position) of the first chunk that intersects the slice |
21 | | and the index of the last chunk that intersects the slice. |
22 | | In practice, the count = last - first + 1 is stored instead of the last index. |
23 | | Note that this n-dim array of indices may have holes in it if the slice stride |
24 | | is greater than the chunk length. |
25 | | @param rank variable rank |
26 | | @param slices the complete set of slices |slices| == R |
27 | | @param ncr (out) the vector of computed chunk ranges. |
28 | | @return NC_EXXX error code |
29 | | */ |
30 | | int |
31 | | NCZ_compute_chunk_ranges( |
32 | | struct Common* common, |
33 | | const NCZSlice* slices, /* the complete set of slices |slices| == R*/ |
34 | | NCZChunkRange* ncr) |
35 | 0 | { |
36 | 0 | int stat = NC_NOERR; |
37 | 0 | int i; |
38 | 0 | int rank = common->rank; |
39 | |
|
40 | 0 | for(i=0;i<rank;i++) { |
41 | 0 | if((stat = compute_intersection(&slices[i],common->chunklens[i],common->isunlimited[i],&ncr[i]))) |
42 | 0 | goto done; |
43 | 0 | } |
44 | | |
45 | 0 | done: |
46 | 0 | return stat; |
47 | 0 | } |
48 | | |
49 | | /** |
50 | | @param Compute chunk range for a single slice. |
51 | | @param chunklen size of the chunk |
52 | | @param isunlimited if corresponding dim is unlimited |
53 | | @param range (out) the range of chunks covered by this slice |
54 | | @return NC_EXX error code |
55 | | */ |
56 | | static int |
57 | | compute_intersection( |
58 | | const NCZSlice* slice, |
59 | | size64_t chunklen, |
60 | | unsigned char isunlimited, |
61 | | NCZChunkRange* range) |
62 | 0 | { |
63 | 0 | range->start = floordiv(slice->start, chunklen); |
64 | 0 | range->stop = ceildiv(slice->stop, chunklen); |
65 | 0 | return NC_NOERR; |
66 | 0 | } |
67 | | |
68 | | /** |
69 | | Compute the projection of a slice as applied to n'th chunk. |
70 | | A projection defines the set of grid points touched within a |
71 | | chunk by a slice. This set of points is the "projection" |
72 | | of the slice onto the chunk. |
73 | | This is somewhat complex because: |
74 | | 1. for the first projection, the start is the slice start, |
75 | | but after that, we have to take into account that for |
76 | | a non-one stride, the start point in a projection may |
77 | | be offset by some value in the range of 0..(slice.stride-1). |
78 | | 2. The stride might be so large as to completely skip some chunks. |
79 | | |
80 | | @return NC_NOERR if ok |
81 | | @return NC_ERANGE if chunk skipped |
82 | | @return NC_EXXXX if failed |
83 | | |
84 | | */ |
85 | | |
86 | | int |
87 | | NCZ_compute_projections(struct Common* common, int r, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections) |
88 | 0 | { |
89 | 0 | int stat = NC_NOERR; |
90 | 0 | NCZProjection* projection = NULL; |
91 | 0 | NCZProjection* prev = NULL; |
92 | 0 | size64_t dimlen = common->dimlens[r]; /* the dimension length for r'th dimension */ |
93 | 0 | size64_t chunklen = common->chunklens[r]; /* the chunk length corresponding to the dimension */ |
94 | 0 | size64_t abslimit; |
95 | |
|
96 | 0 | projection = &projections[n]; |
97 | 0 | if(n > 0) { |
98 | | /* Find last non-skipped projection */ |
99 | 0 | for(size_t i=n;i-->0;) { /* walk backward */ |
100 | 0 | if(!projections[i].skip) { |
101 | 0 | prev = &projections[i]; |
102 | 0 | break; |
103 | 0 | } |
104 | 0 | } |
105 | 0 | if(prev == NULL) {stat = NC_ENCZARR; goto done;} |
106 | 0 | } |
107 | | |
108 | 0 | projection->id = ++pcounter; |
109 | 0 | projection->chunkindex = chunkindex; |
110 | |
|
111 | 0 | projection->offset = chunklen * chunkindex; /* with respect to dimension (WRD) */ |
112 | | |
113 | | /* limit in the n'th touched chunk, taking dimlen and stride->stop into account. */ |
114 | 0 | abslimit = (chunkindex + 1) * chunklen; |
115 | 0 | if(abslimit > slice->stop) abslimit = slice->stop; |
116 | 0 | if(abslimit > dimlen) abslimit = dimlen; |
117 | 0 | projection->limit = abslimit - projection->offset; |
118 | | |
119 | | /* See if the next point after the last one in prev lands in the current projection. |
120 | | If not, then we have skipped the current chunk. Also take limit into account. |
121 | | Note by definition, n must be greater than zero because we always start in a relevant chunk. |
122 | | */ |
123 | |
|
124 | 0 | if(n == 0) { |
125 | | /*initial case: original slice start is in 1st projection */ |
126 | 0 | projection->first = slice->start - projection->offset; |
127 | 0 | projection->iopos = 0; |
128 | 0 | } else { /* n > 0 */ |
129 | | /* Use absolute offsets for these computations to avoid negative values */ |
130 | 0 | size64_t abslastpoint, absnextpoint, absthislast; |
131 | | |
132 | | /* abs last point touched in prev projection */ |
133 | 0 | abslastpoint = prev->offset + prev->last; |
134 | | |
135 | | /* Compute the abs last touchable point in this chunk */ |
136 | 0 | absthislast = projection->offset + projection->limit; |
137 | | |
138 | | /* Compute next point touched after the last point touched in previous projection; |
139 | | note that the previous projection might be wrt a chunk other than the immediately preceding |
140 | | one (because the intermediate ones were skipped). |
141 | | */ |
142 | 0 | absnextpoint = abslastpoint + slice->stride; /* abs next point to be touched */ |
143 | |
|
144 | 0 | if(absnextpoint >= absthislast) { /* this chunk is being skipped */ |
145 | 0 | skipchunk(slice,projection); |
146 | 0 | goto done; |
147 | 0 | } |
148 | | |
149 | | /* Compute start point in this chunk */ |
150 | | /* basically absnextpoint - abs start of this projection */ |
151 | 0 | projection->first = absnextpoint - projection->offset; |
152 | | |
153 | | /* Compute the memory location of this first point in this chunk */ |
154 | 0 | projection->iopos = ceildiv((projection->offset - slice->start),slice->stride); |
155 | |
|
156 | 0 | } |
157 | 0 | if(slice->stop > abslimit) |
158 | 0 | projection->stop = chunklen; |
159 | 0 | else |
160 | 0 | projection->stop = slice->stop - projection->offset; |
161 | |
|
162 | 0 | projection->iocount = ceildiv((projection->stop - projection->first),slice->stride); |
163 | | |
164 | | /* Compute the slice relative to this chunk. |
165 | | Recall the possibility that start+stride >= projection->limit */ |
166 | 0 | projection->chunkslice.start = projection->first; |
167 | 0 | projection->chunkslice.stop = projection->stop; |
168 | 0 | projection->chunkslice.stride = slice->stride; |
169 | 0 | projection->chunkslice.len = chunklen; |
170 | | |
171 | | /* Last place to be touched */ |
172 | 0 | projection->last = projection->first + (slice->stride * (projection->iocount - 1)); |
173 | |
|
174 | 0 | projection->memslice.start = projection->iopos; |
175 | 0 | projection->memslice.stop = projection->iopos + projection->iocount; |
176 | 0 | projection->memslice.stride = 1; |
177 | | // projection->memslice.stride = slice->stride; |
178 | | // projection->memslice.len = projection->memslice.stop; |
179 | 0 | projection->memslice.len = common->memshape[r]; |
180 | | #ifdef NEVERUSE |
181 | | projection->memslice.len = dimlen; |
182 | | projection->memslice.len = chunklen; |
183 | | #endif |
184 | |
|
185 | 0 | if(!verifyslice(&projection->memslice) || !verifyslice(&projection->chunkslice)) |
186 | 0 | {stat = NC_ECONSTRAINT; goto done;} |
187 | | |
188 | 0 | done: |
189 | 0 | return stat; |
190 | 0 | } |
191 | | |
192 | | static void |
193 | | skipchunk(const NCZSlice* slice, NCZProjection* projection) |
194 | 0 | { |
195 | 0 | projection->skip = 1; |
196 | 0 | projection->first = 0; |
197 | 0 | projection->last = 0; |
198 | 0 | projection->iopos = ceildiv(projection->offset - slice->start, slice->stride); |
199 | 0 | projection->iocount = 0; |
200 | 0 | projection->chunkslice.start = 0; |
201 | 0 | projection->chunkslice.stop = 0; |
202 | 0 | projection->chunkslice.stride = 1; |
203 | 0 | projection->chunkslice.len = 0; |
204 | 0 | projection->memslice.start = 0; |
205 | 0 | projection->memslice.stop = 0; |
206 | 0 | projection->memslice.stride = 1; |
207 | 0 | projection->memslice.len = 0; |
208 | 0 | } |
209 | | |
210 | | /* Goal: |
211 | | Create a vector of projections wrt a slice and a sequence of chunks. |
212 | | */ |
213 | | |
214 | | int |
215 | | NCZ_compute_per_slice_projections( |
216 | | struct Common* common, |
217 | | int r, /* which dimension are we projecting? */ |
218 | | const NCZSlice* slice, /* the slice for which projections are computed */ |
219 | | const NCZChunkRange* range, /* range */ |
220 | | NCZSliceProjections* slp) |
221 | 0 | { |
222 | 0 | int stat = NC_NOERR; |
223 | 0 | size64_t index,slicecount; |
224 | 0 | size_t n; |
225 | | |
226 | | /* Part fill the Slice Projections */ |
227 | 0 | slp->r = r; |
228 | 0 | slp->range = *range; |
229 | 0 | slp->count = range->stop - range->start; |
230 | 0 | if((slp->projections = calloc(slp->count,sizeof(NCZProjection))) == NULL) |
231 | 0 | {stat = NC_ENOMEM; goto done;} |
232 | | |
233 | | /* Compute the total number of output items defined by this slice |
234 | | (equivalent to count as used by nc_get_vars) */ |
235 | 0 | slicecount = ceildiv((slice->stop - slice->start), slice->stride); |
236 | 0 | if(slicecount < 0) slicecount = 0; |
237 | | |
238 | | /* Iterate over each chunk that intersects slice to produce projection */ |
239 | 0 | for(n=0,index=range->start;index<range->stop;index++,n++) { |
240 | 0 | if((stat = NCZ_compute_projections(common, r, index, slice, n, slp->projections))) |
241 | 0 | goto done; /* something went wrong */ |
242 | 0 | } |
243 | | |
244 | 0 | done: |
245 | 0 | return stat; |
246 | 0 | } |
247 | | |
248 | | /* Goal:create a vector of SliceProjection instances: one for each |
249 | | slice in the top-level input. For each slice, compute a set |
250 | | of projections from it wrt a dimension and a chunk size |
251 | | associated with that dimension. |
252 | | */ |
253 | | int |
254 | | NCZ_compute_all_slice_projections( |
255 | | struct Common* common, |
256 | | const NCZSlice* slices, /* the complete set of slices |slices| == R*/ |
257 | | const NCZChunkRange* ranges, |
258 | | NCZSliceProjections* results) |
259 | 0 | { |
260 | 0 | int stat = NC_NOERR; |
261 | 0 | int r; |
262 | |
|
263 | 0 | for(r=0;r<common->rank;r++) { |
264 | | /* Compute each of the rank SliceProjections instances */ |
265 | 0 | NCZSliceProjections* slp = &results[r]; |
266 | 0 | if((stat=NCZ_compute_per_slice_projections( |
267 | 0 | common, |
268 | 0 | r, |
269 | 0 | &slices[r], |
270 | 0 | &ranges[r], |
271 | 0 | slp))) goto done; |
272 | 0 | } |
273 | | |
274 | 0 | done: |
275 | 0 | return stat; |
276 | 0 | } |
277 | | |
278 | | /**************************************************/ |
279 | | /* Utilities */ |
280 | | |
281 | | /* return 0 if slice is malformed; 1 otherwise */ |
282 | | static int |
283 | | verifyslice(const NCZSlice* slice) |
284 | 0 | { |
285 | 0 | if(slice->stop < slice->start) return 0; |
286 | 0 | if(slice->stride <= 0) return 0; |
287 | 0 | if((slice->stop - slice->start) > slice->len) return 0; |
288 | 0 | return 1; |
289 | 0 | } |
290 | | |
291 | | void |
292 | | NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv) |
293 | 0 | { |
294 | 0 | if(slpv != NULL) { |
295 | 0 | int i; |
296 | 0 | for(i=0;i<count;i++) { |
297 | 0 | NCZSliceProjections* slp = &slpv[i]; |
298 | | nullfree(slp->projections); |
299 | 0 | } |
300 | 0 | } |
301 | 0 | } |
302 | | |
303 | | #if 0 |
304 | | static void |
305 | | clearallprojections(NCZAllProjections* nap) |
306 | | { |
307 | | if(nap != NULL) { |
308 | | int i; |
309 | | for(i=0;i<nap->rank;i++) |
310 | | nclistfreeall(&nap->allprojections[i].projections); |
311 | | } |
312 | | } |
313 | | #endif |