/src/netcdf-c/libnczarr/zchunking.c
Line | Count | Source (jump to first uncovered line) |
1 | | /********************************************************************* |
2 | | * Copyright 2018, UCAR/Unidata |
3 | | * See netcdf/COPYRIGHT file for copying and redistribution conditions. |
4 | | *********************************************************************/ |
5 | | #include "zincludes.h" |
6 | | |
7 | | #define MAX(a,b) ((a)>(b)?(a):(b)) |
8 | | #define MIN(a,b) ((a)<(b)?(a):(b)) |
9 | | |
10 | | static int pcounter = 0; |
11 | | |
12 | | /* Forward */ |
13 | | static int compute_intersection(const NCZSlice* slice, const size64_t chunklen, NCZChunkRange* range); |
14 | | static void skipchunk(const NCZSlice* slice, NCZProjection* projection); |
15 | | static int verifyslice(const NCZSlice* slice); |
16 | | |
17 | | /**************************************************/ |
18 | | /* Goal:create a vector of chunk ranges: one for each slice in |
19 | | the top-level input. For each slice, compute the index (not |
20 | | absolute position) of the first chunk that intersects the slice |
21 | | and the index of the last chunk that intersects the slice. |
22 | | In practice, the count = last - first + 1 is stored instead of the last index. |
23 | | */ |
24 | | int |
25 | | NCZ_compute_chunk_ranges( |
26 | | int rank, /* variable rank */ |
27 | | const NCZSlice* slices, /* the complete set of slices |slices| == R*/ |
28 | | const size64_t* chunklen, /* the chunk length corresponding to the dimensions */ |
29 | | NCZChunkRange* ncr) |
30 | 0 | { |
31 | 0 | int stat = NC_NOERR; |
32 | 0 | int i; |
33 | |
|
34 | 0 | for(i=0;i<rank;i++) { |
35 | 0 | if((stat = compute_intersection(&slices[i],chunklen[i],&ncr[i]))) |
36 | 0 | goto done; |
37 | 0 | } |
38 | | |
39 | 0 | done: |
40 | 0 | return stat; |
41 | 0 | } |
42 | | |
43 | | static int |
44 | | compute_intersection( |
45 | | const NCZSlice* slice, |
46 | | const size64_t chunklen, |
47 | | NCZChunkRange* range) |
48 | 0 | { |
49 | 0 | range->start = floordiv(slice->start, chunklen); |
50 | 0 | range->stop = ceildiv(slice->stop, chunklen); |
51 | 0 | return NC_NOERR; |
52 | 0 | } |
53 | | |
54 | | /** |
55 | | Compute the projection of a slice as applied to n'th chunk. |
56 | | This is somewhat complex because: |
57 | | 1. for the first projection, the start is the slice start, |
58 | | but after that, we have to take into account that for |
59 | | a non-one stride, the start point in a projection may |
60 | | be offset by some value in the range of 0..(slice.stride-1). |
61 | | 2. The stride might be so large as to completely skip some chunks. |
62 | | |
63 | | @return NC_NOERR if ok |
64 | | @return NC_ERANGE if chunk skipped |
65 | | @return NC_EXXXX if failed |
66 | | |
67 | | */ |
68 | | |
69 | | int |
70 | | NCZ_compute_projections(struct Common* common, int r, size64_t chunkindex, const NCZSlice* slice, size_t n, NCZProjection* projections) |
71 | 0 | { |
72 | 0 | int stat = NC_NOERR; |
73 | 0 | NCZProjection* projection = NULL; |
74 | 0 | NCZProjection* prev = NULL; |
75 | 0 | size64_t dimlen = common->dimlens[r]; /* the dimension length for r'th dimension */ |
76 | 0 | size64_t chunklen = common->chunklens[r]; /* the chunk length corresponding to the dimension */ |
77 | 0 | size64_t abslimit; |
78 | |
|
79 | 0 | projection = &projections[n]; |
80 | 0 | if(n > 0) { |
81 | | /* Find last non-skipped projection */ |
82 | 0 | int i; |
83 | 0 | for(i=n-1;i>=0;i--) { /* walk backward */ |
84 | 0 | if(!projections[i].skip) { |
85 | 0 | prev = &projections[i]; |
86 | 0 | break; |
87 | 0 | } |
88 | 0 | } |
89 | 0 | if(prev == NULL) {stat = NC_ENCZARR; goto done;} |
90 | 0 | } |
91 | | |
92 | 0 | projection->id = ++pcounter; |
93 | 0 | projection->chunkindex = chunkindex; |
94 | |
|
95 | 0 | projection->offset = chunklen * chunkindex; /* with respect to dimension (WRD) */ |
96 | | |
97 | | /* limit in the n'th touched chunk, taking dimlen and stride->stop into account. */ |
98 | 0 | abslimit = (chunkindex + 1) * chunklen; |
99 | 0 | if(abslimit > slice->stop) abslimit = slice->stop; |
100 | 0 | if(abslimit > dimlen) abslimit = dimlen; |
101 | 0 | projection->limit = abslimit - projection->offset; |
102 | | |
103 | | /* See if the next point after the last one in prev lands in the current projection. |
104 | | If not, then we have skipped the current chunk. Also take limit into account. |
105 | | Note by definition, n must be greater than zero because we always start in a relevant chunk. |
106 | | */ |
107 | |
|
108 | 0 | if(n == 0) { |
109 | | /*initial case: original slice start is in 1st projection */ |
110 | 0 | projection->first = slice->start - projection->offset; |
111 | 0 | projection->iopos = 0; |
112 | 0 | } else { /* n > 0 */ |
113 | | /* Use absolute offsets for these computations to avoid negative values */ |
114 | 0 | size64_t abslastpoint, absnextpoint, absthislast; |
115 | | |
116 | | /* abs last point touched in prev projection */ |
117 | 0 | abslastpoint = prev->offset + prev->last; |
118 | | |
119 | | /* Compute the abs last touchable point in this chunk */ |
120 | 0 | absthislast = projection->offset + projection->limit; |
121 | | |
122 | | /* Compute next point touched after the last point touched in previous projection; |
123 | | note that the previous projection might be wrt a chunk other than the immediately preceding |
124 | | one (because the intermediate ones were skipped). |
125 | | */ |
126 | 0 | absnextpoint = abslastpoint + slice->stride; /* abs next point to be touched */ |
127 | |
|
128 | 0 | if(absnextpoint >= absthislast) { /* this chunk is being skipped */ |
129 | 0 | skipchunk(slice,projection); |
130 | 0 | goto done; |
131 | 0 | } |
132 | | |
133 | | /* Compute start point in this chunk */ |
134 | | /* basically absnextpoint - abs start of this projection */ |
135 | 0 | projection->first = absnextpoint - projection->offset; |
136 | | |
137 | | /* Compute the memory location of this first point in this chunk */ |
138 | 0 | projection->iopos = ceildiv((projection->offset - slice->start),slice->stride); |
139 | |
|
140 | 0 | } |
141 | 0 | if(slice->stop > abslimit) |
142 | 0 | projection->stop = chunklen; |
143 | 0 | else |
144 | 0 | projection->stop = slice->stop - projection->offset; |
145 | |
|
146 | 0 | projection->iocount = ceildiv((projection->stop - projection->first),slice->stride); |
147 | | |
148 | | /* Compute the slice relative to this chunk. |
149 | | Recall the possibility that start+stride >= projection->limit */ |
150 | 0 | projection->chunkslice.start = projection->first; |
151 | 0 | projection->chunkslice.stop = projection->stop; |
152 | 0 | projection->chunkslice.stride = slice->stride; |
153 | 0 | projection->chunkslice.len = chunklen; |
154 | | |
155 | | /* Last place to be touched */ |
156 | 0 | projection->last = projection->first + (slice->stride * (projection->iocount - 1)); |
157 | |
|
158 | 0 | projection->memslice.start = projection->iopos; |
159 | 0 | projection->memslice.stop = projection->iopos + projection->iocount; |
160 | 0 | projection->memslice.stride = 1; |
161 | | // projection->memslice.stride = slice->stride; |
162 | | // projection->memslice.len = projection->memslice.stop; |
163 | 0 | projection->memslice.len = common->memshape[r]; |
164 | | #ifdef NEVERUSE |
165 | | projection->memslice.len = dimlen; |
166 | | projection->memslice.len = chunklen; |
167 | | #endif |
168 | |
|
169 | 0 | if(!verifyslice(&projection->memslice) || !verifyslice(&projection->chunkslice)) |
170 | 0 | {stat = NC_ECONSTRAINT; goto done;} |
171 | | |
172 | 0 | done: |
173 | 0 | return stat; |
174 | 0 | } |
175 | | |
176 | | static void |
177 | | skipchunk(const NCZSlice* slice, NCZProjection* projection) |
178 | 0 | { |
179 | 0 | projection->skip = 1; |
180 | 0 | projection->first = 0; |
181 | 0 | projection->last = 0; |
182 | 0 | projection->iopos = ceildiv(projection->offset - slice->start, slice->stride); |
183 | 0 | projection->iocount = 0; |
184 | 0 | projection->chunkslice.start = 0; |
185 | 0 | projection->chunkslice.stop = 0; |
186 | 0 | projection->chunkslice.stride = 1; |
187 | 0 | projection->chunkslice.len = 0; |
188 | 0 | projection->memslice.start = 0; |
189 | 0 | projection->memslice.stop = 0; |
190 | 0 | projection->memslice.stride = 1; |
191 | 0 | projection->memslice.len = 0; |
192 | 0 | } |
193 | | |
194 | | /* Goal: |
195 | | Create a vector of projections wrt a slice and a sequence of chunks. |
196 | | */ |
197 | | |
198 | | int |
199 | | NCZ_compute_per_slice_projections( |
200 | | struct Common* common, |
201 | | int r, /* which dimension are we projecting? */ |
202 | | const NCZSlice* slice, /* the slice for which projections are computed */ |
203 | | const NCZChunkRange* range, /* range */ |
204 | | NCZSliceProjections* slp) |
205 | 0 | { |
206 | 0 | int stat = NC_NOERR; |
207 | 0 | size64_t index,slicecount; |
208 | 0 | size_t n; |
209 | | |
210 | | /* Part fill the Slice Projections */ |
211 | 0 | slp->r = r; |
212 | 0 | slp->range = *range; |
213 | 0 | slp->count = range->stop - range->start; |
214 | 0 | if((slp->projections = calloc(slp->count,sizeof(NCZProjection))) == NULL) |
215 | 0 | {stat = NC_ENOMEM; goto done;} |
216 | | |
217 | | /* Compute the total number of output items defined by this slice |
218 | | (equivalent to count as used by nc_get_vars) */ |
219 | 0 | slicecount = ceildiv((slice->stop - slice->start), slice->stride); |
220 | 0 | if(slicecount < 0) slicecount = 0; |
221 | | |
222 | | /* Iterate over each chunk that intersects slice to produce projection */ |
223 | 0 | for(n=0,index=range->start;index<range->stop;index++,n++) { |
224 | 0 | if((stat = NCZ_compute_projections(common, r, index, slice, n, slp->projections))) |
225 | 0 | goto done; /* something went wrong */ |
226 | 0 | } |
227 | | |
228 | 0 | done: |
229 | 0 | return stat; |
230 | 0 | } |
231 | | |
232 | | /* Goal:create a vector of SliceProjection instances: one for each |
233 | | slice in the top-level input. For each slice, compute a set |
234 | | of projections from it wrt a dimension and a chunk size |
235 | | associated with that dimension. |
236 | | */ |
237 | | int |
238 | | NCZ_compute_all_slice_projections( |
239 | | struct Common* common, |
240 | | const NCZSlice* slices, /* the complete set of slices |slices| == R*/ |
241 | | const NCZChunkRange* ranges, |
242 | | NCZSliceProjections* results) |
243 | 0 | { |
244 | 0 | int stat = NC_NOERR; |
245 | 0 | size64_t r; |
246 | |
|
247 | 0 | for(r=0;r<common->rank;r++) { |
248 | | /* Compute each of the rank SliceProjections instances */ |
249 | 0 | NCZSliceProjections* slp = &results[r]; |
250 | 0 | if((stat=NCZ_compute_per_slice_projections( |
251 | 0 | common, |
252 | 0 | r, |
253 | 0 | &slices[r], |
254 | 0 | &ranges[r], |
255 | 0 | slp))) goto done; |
256 | 0 | } |
257 | | |
258 | 0 | done: |
259 | 0 | return stat; |
260 | 0 | } |
261 | | |
262 | | /**************************************************/ |
263 | | /* Utilities */ |
264 | | |
265 | | /* return 0 if slice is malformed; 1 otherwise */ |
266 | | static int |
267 | | verifyslice(const NCZSlice* slice) |
268 | 0 | { |
269 | 0 | if(slice->stop < slice->start) return 0; |
270 | 0 | if(slice->stride <= 0) return 0; |
271 | 0 | if((slice->stop - slice->start) > slice->len) return 0; |
272 | 0 | return 1; |
273 | 0 | } |
274 | | |
275 | | void |
276 | | NCZ_clearsliceprojections(int count, NCZSliceProjections* slpv) |
277 | 0 | { |
278 | 0 | if(slpv != NULL) { |
279 | 0 | int i; |
280 | 0 | for(i=0;i<count;i++) { |
281 | 0 | NCZSliceProjections* slp = &slpv[i]; |
282 | 0 | nullfree(slp->projections); |
283 | 0 | } |
284 | 0 | } |
285 | 0 | } |
286 | | |
287 | | #if 0 |
288 | | static void |
289 | | clearallprojections(NCZAllProjections* nap) |
290 | | { |
291 | | if(nap != NULL) { |
292 | | int i; |
293 | | for(i=0;i<nap->rank;i++) |
294 | | nclistfreeall(&nap->allprojections[i].projections); |
295 | | } |
296 | | } |
297 | | #endif |
298 | | |