/src/netcdf-c/libnczarr/zarr.c
| Line | Count | Source | 
| 1 |  | /********************************************************************* | 
| 2 |  |  *   Copyright 2018, UCAR/Unidata | 
| 3 |  |  *   See netcdf/COPYRIGHT file for copying and redistribution conditions. | 
| 4 |  |  *********************************************************************/ | 
| 5 |  |  | 
| 6 |  | #include "zincludes.h" | 
| 7 |  | #include <stddef.h> | 
| 8 |  |  | 
| 9 |  | /**************************************************/ | 
| 10 |  | /* Forwards */ | 
| 11 |  |  | 
| 12 |  | static int applycontrols(NCZ_FILE_INFO_T* zinfo); | 
| 13 |  |  | 
| 14 |  | /***************************************************/ | 
| 15 |  | /* API */ | 
| 16 |  |  | 
| 17 |  | /** | 
| 18 |  | @internal Create the topmost dataset object and setup up | 
| 19 |  |           NCZ_FILE_INFO_T state. | 
| 20 |  | @param zinfo - [in] the internal state | 
| 21 |  | @return NC_NOERR | 
| 22 |  | @author Dennis Heimbigner | 
| 23 |  | */ | 
| 24 |  |  | 
| 25 |  | int | 
| 26 |  | ncz_create_dataset(NC_FILE_INFO_T* file, NC_GRP_INFO_T* root, NClist* controls) | 
| 27 | 0 | { | 
| 28 | 0 |     int stat = NC_NOERR; | 
| 29 | 0 |     NCZ_FILE_INFO_T* zinfo = NULL; | 
| 30 | 0 |     NCZ_GRP_INFO_T* zgrp = NULL; | 
| 31 | 0 |     NCURI* uri = NULL; | 
| 32 | 0 |     NC* nc = NULL; | 
| 33 | 0 |     NCjson* json = NULL; | 
| 34 | 0 |     char* key = NULL; | 
| 35 |  | 
 | 
| 36 | 0 |     ZTRACE(3,"file=%s root=%s controls=%s",file->hdr.name,root->hdr.name,(controls?nczprint_envv(controls):"null")); | 
| 37 |  | 
 | 
| 38 | 0 |     nc = (NC*)file->controller; | 
| 39 |  |  | 
| 40 |  |     /* Add struct to hold NCZ-specific file metadata. */ | 
| 41 | 0 |     if (!(zinfo = calloc(1, sizeof(NCZ_FILE_INFO_T)))) | 
| 42 | 0 |         {stat = NC_ENOMEM; goto done;} | 
| 43 | 0 |     file->format_file_info = zinfo; | 
| 44 | 0 |     zinfo->common.file = file; | 
| 45 |  |  | 
| 46 |  |     /* Add struct to hold NCZ-specific group info. */ | 
| 47 | 0 |     if (!(zgrp = calloc(1, sizeof(NCZ_GRP_INFO_T)))) | 
| 48 | 0 |         {stat = NC_ENOMEM; goto done;} | 
| 49 | 0 |     root->format_grp_info = zgrp; | 
| 50 | 0 |     zgrp->common.file = file; | 
| 51 |  |  | 
| 52 |  |     /* Fill in NCZ_FILE_INFO_T */ | 
| 53 | 0 |     zinfo->creating = 1; | 
| 54 | 0 |     zinfo->common.file = file; | 
| 55 | 0 |     zinfo->native_endianness = (NC_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); | 
| 56 | 0 |     if((zinfo->controllist=nclistclone(controls,1)) == NULL) | 
| 57 | 0 |   {stat = NC_ENOMEM; goto done;} | 
| 58 |  |  | 
| 59 |  |     /* fill in some of the zinfo and zroot fields */ | 
| 60 | 0 |     zinfo->zarr.zarr_version = atoi(ZARRVERSION); | 
| 61 | 0 |     sscanf(NCZARRVERSION,"%lu.%lu.%lu", | 
| 62 | 0 |      &zinfo->zarr.nczarr_version.major, | 
| 63 | 0 |      &zinfo->zarr.nczarr_version.minor, | 
| 64 | 0 |      &zinfo->zarr.nczarr_version.release); | 
| 65 |  | 
 | 
| 66 | 0 |     zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT; | 
| 67 |  |  | 
| 68 |  |     /* Apply client controls */ | 
| 69 | 0 |     if((stat = applycontrols(zinfo))) goto done; | 
| 70 |  |  | 
| 71 |  |     /* Load auth info from rc file */ | 
| 72 | 0 |     if((stat = ncuriparse(nc->path,&uri))) goto done; | 
| 73 | 0 |     if(uri) { | 
| 74 | 0 |   if((stat = NC_authsetup(&zinfo->auth, uri))) | 
| 75 | 0 |       goto done; | 
| 76 | 0 |     } | 
| 77 |  |  | 
| 78 |  |     /* initialize map handle*/ | 
| 79 | 0 |     if((stat = nczmap_create(zinfo->controls.mapimpl,nc->path,nc->mode,zinfo->controls.flags,NULL,&zinfo->map))) | 
| 80 | 0 |   goto done; | 
| 81 |  |  | 
| 82 | 0 | done: | 
| 83 | 0 |     ncurifree(uri); | 
| 84 | 0 |     NCJreclaim(json); | 
| 85 | 0 |     nullfree(key); | 
| 86 | 0 |     return ZUNTRACE(stat); | 
| 87 | 0 | } | 
| 88 |  |  | 
| 89 |  | /** | 
| 90 |  | @internal Open the topmost dataset object. | 
| 91 |  | @param file - [in] the file struct | 
| 92 |  | @param controls - [in] the fragment list in envv form from uri | 
| 93 |  | @return NC_NOERR | 
| 94 |  | @author Dennis Heimbigner | 
| 95 |  | */ | 
| 96 |  |  | 
| 97 |  | int | 
| 98 |  | ncz_open_dataset(NC_FILE_INFO_T* file, NClist* controls) | 
| 99 | 0 | { | 
| 100 | 0 |     int stat = NC_NOERR; | 
| 101 | 0 |     NC* nc = NULL; | 
| 102 | 0 |     NC_GRP_INFO_T* root = NULL; | 
| 103 | 0 |     NCURI* uri = NULL; | 
| 104 | 0 |     void* content = NULL; | 
| 105 | 0 |     NCjson* json = NULL; | 
| 106 | 0 |     NCZ_FILE_INFO_T* zinfo = NULL; | 
| 107 | 0 |     int mode; | 
| 108 | 0 |     NClist* modeargs = NULL; | 
| 109 | 0 |     char* nczarr_version = NULL; | 
| 110 | 0 |     char* zarr_format = NULL; | 
| 111 |  | 
 | 
| 112 | 0 |     ZTRACE(3,"file=%s controls=%s",file->hdr.name,(controls?nczprint_envv(controls):"null")); | 
| 113 |  |  | 
| 114 |  |     /* Extract info reachable via file */ | 
| 115 | 0 |     nc = (NC*)file->controller; | 
| 116 | 0 |     mode = nc->mode; | 
| 117 |  | 
 | 
| 118 | 0 |     root = file->root_grp; | 
| 119 | 0 |     assert(root != NULL && root->hdr.sort == NCGRP); | 
| 120 |  |  | 
| 121 |  |     /* Add struct to hold NCZ-specific file metadata. */ | 
| 122 | 0 |     if (!(file->format_file_info = calloc(1, sizeof(NCZ_FILE_INFO_T)))) | 
| 123 | 0 |         {stat = NC_ENOMEM; goto done;} | 
| 124 | 0 |     zinfo = file->format_file_info; | 
| 125 |  |  | 
| 126 |  |     /* Fill in NCZ_FILE_INFO_T */ | 
| 127 | 0 |     zinfo->creating = 0; | 
| 128 | 0 |     zinfo->common.file = file; | 
| 129 | 0 |     zinfo->native_endianness = (NC_isLittleEndian() ? NC_ENDIAN_LITTLE : NC_ENDIAN_BIG); | 
| 130 | 0 |     if((zinfo->controllist=nclistclone(controls,1)) == NULL) | 
| 131 | 0 |   {stat = NC_ENOMEM; goto done;} | 
| 132 | 0 |     zinfo->default_maxstrlen = NCZ_MAXSTR_DEFAULT; | 
| 133 |  |  | 
| 134 |  |     /* Add struct to hold NCZ-specific group info. */ | 
| 135 | 0 |     if (!(root->format_grp_info = calloc(1, sizeof(NCZ_GRP_INFO_T)))) | 
| 136 | 0 |         {stat = NC_ENOMEM; goto done;} | 
| 137 | 0 |     ((NCZ_GRP_INFO_T*)root->format_grp_info)->common.file = file; | 
| 138 |  |  | 
| 139 |  |     /* Apply client controls */ | 
| 140 | 0 |     if((stat = applycontrols(zinfo))) goto done; | 
| 141 |  |  | 
| 142 |  |     /* initialize map handle*/ | 
| 143 | 0 |     if((stat = nczmap_open(zinfo->controls.mapimpl,nc->path,mode,zinfo->controls.flags,NULL,&zinfo->map))) | 
| 144 | 0 |   goto done; | 
| 145 |  |  | 
| 146 |  |     /* Ok, try to read superblock */ | 
| 147 | 0 |     if((stat = ncz_read_superblock(file,&nczarr_version,&zarr_format))) goto done; | 
| 148 |  |  | 
| 149 | 0 |     if(nczarr_version == NULL) /* default */ | 
| 150 | 0 |         nczarr_version = strdup(NCZARRVERSION); | 
| 151 | 0 |     if(zarr_format == NULL) /* default */ | 
| 152 | 0 |        zarr_format = strdup(ZARRVERSION); | 
| 153 |  |     /* Extract the information from it */ | 
| 154 | 0 |     if(sscanf(zarr_format,"%d",&zinfo->zarr.zarr_version)!=1) | 
| 155 | 0 |   {stat = NC_ENCZARR; goto done;}    | 
| 156 | 0 |     if(sscanf(nczarr_version,"%lu.%lu.%lu", | 
| 157 | 0 |         &zinfo->zarr.nczarr_version.major, | 
| 158 | 0 |         &zinfo->zarr.nczarr_version.minor, | 
| 159 | 0 |         &zinfo->zarr.nczarr_version.release) == 0) | 
| 160 | 0 |   {stat = NC_ENCZARR; goto done;} | 
| 161 |  |  | 
| 162 |  |     /* Load auth info from rc file */ | 
| 163 | 0 |     if((stat = ncuriparse(nc->path,&uri))) goto done; | 
| 164 | 0 |     if(uri) { | 
| 165 | 0 |   if((stat = NC_authsetup(&zinfo->auth, uri))) | 
| 166 | 0 |       goto done; | 
| 167 | 0 |     } | 
| 168 |  |  | 
| 169 | 0 | done: | 
| 170 | 0 |     nullfree(zarr_format); | 
| 171 | 0 |     nullfree(nczarr_version); | 
| 172 | 0 |     ncurifree(uri); | 
| 173 | 0 |     nclistfreeall(modeargs); | 
| 174 | 0 |     if(json) NCJreclaim(json); | 
| 175 | 0 |     nullfree(content); | 
| 176 | 0 |     return ZUNTRACE(stat); | 
| 177 | 0 | } | 
| 178 |  |  | 
| 179 |  | /** | 
| 180 |  |  * @internal Determine whether file is netCDF-4. | 
| 181 |  |  * | 
| 182 |  |  * For libzarr, this is always true. | 
| 183 |  |  * | 
| 184 |  |  * @param h5 Pointer to HDF5 file info struct. | 
| 185 |  |  * | 
| 186 |  |  * @returns NC_NOERR No error. | 
| 187 |  |  * @author Dennis Heimbigner. | 
| 188 |  |  */ | 
| 189 |  | int | 
| 190 |  | NCZ_isnetcdf4(struct NC_FILE_INFO* h5) | 
| 191 | 0 | { | 
| 192 | 0 |     int isnc4 = 1; | 
| 193 | 0 |     NC_UNUSED(h5); | 
| 194 | 0 |     return isnc4; | 
| 195 | 0 | } | 
| 196 |  |  | 
| 197 |  | /** | 
| 198 |  |  * @internal Determine version info | 
| 199 |  |  * | 
| 200 |  |  * For libzarr, this is not well defined | 
| 201 |  |  * | 
| 202 |  |  * @param majorp Pointer to major version number | 
| 203 |  |  * @param minorp Pointer to minor version number | 
| 204 |  |  * @param releasep Pointer to release version number | 
| 205 |  |  * | 
| 206 |  |  * @returns NC_NOERR No error. | 
| 207 |  |  * @author Dennis Heimbigner. | 
| 208 |  |  */ | 
| 209 |  | int | 
| 210 |  | NCZ_get_libversion(unsigned long* majorp, unsigned long* minorp,unsigned long* releasep) | 
| 211 | 1 | { | 
| 212 | 1 |     unsigned long m0,m1,m2; | 
| 213 | 1 |     sscanf(NCZARRVERSION,"%lu.%lu.%lu",&m0,&m1,&m2); | 
| 214 | 1 |     if(majorp) *majorp = m0; | 
| 215 | 1 |     if(minorp) *minorp = m1; | 
| 216 | 1 |     if(releasep) *releasep = m2; | 
| 217 | 1 |     return NC_NOERR; | 
| 218 | 1 | } | 
| 219 |  |  | 
| 220 |  | /** | 
| 221 |  |  * @internal Determine "superblock" number. | 
| 222 |  |  * | 
| 223 |  |  * For libzarr, use the value of the major part of the nczarr version. | 
| 224 |  |  * | 
| 225 |  |  * @param superblocp Pointer to place to return superblock. | 
| 226 |  |  * use the nczarr format version major as the superblock number. | 
| 227 |  |  * | 
| 228 |  |  * @returns NC_NOERR No error. | 
| 229 |  |  * @author Dennis Heimbigner. | 
| 230 |  |  */ | 
| 231 |  | int | 
| 232 |  | NCZ_get_superblock(NC_FILE_INFO_T* file, int* superblockp) | 
| 233 | 0 | { | 
| 234 | 0 |     NCZ_FILE_INFO_T* zinfo = file->format_file_info; | 
| 235 | 0 |     if(superblockp) *superblockp = zinfo->zarr.nczarr_version.major; | 
| 236 | 0 |     return NC_NOERR; | 
| 237 | 0 | } | 
| 238 |  |  | 
| 239 |  | /**************************************************/ | 
| 240 |  | /* Utilities */ | 
| 241 |  |  | 
| 242 |  | static const char* | 
| 243 |  | controllookup(NClist* controls, const char* key) | 
| 244 | 0 | { | 
| 245 | 0 |     size_t i; | 
| 246 | 0 |     for(i=0;i<nclistlength(controls);i+=2) { | 
| 247 | 0 |         const char* p = (char*)nclistget(controls,i); | 
| 248 | 0 |   if(strcasecmp(key,p)==0) { | 
| 249 | 0 |       return (const char*)nclistget(controls,i+1); | 
| 250 | 0 |   } | 
| 251 | 0 |     } | 
| 252 | 0 |     return NULL; | 
| 253 | 0 | } | 
| 254 |  |  | 
| 255 |  |  | 
| 256 |  | static int | 
| 257 |  | applycontrols(NCZ_FILE_INFO_T* zinfo) | 
| 258 | 0 | { | 
| 259 | 0 |     size_t i; | 
| 260 | 0 |     int stat = NC_NOERR; | 
| 261 | 0 |     const char* value = NULL; | 
| 262 | 0 |     NClist* modelist = nclistnew(); | 
| 263 | 0 |     size64_t noflags = 0; /* track non-default negative flags */ | 
| 264 |  | 
 | 
| 265 | 0 |     if((value = controllookup(zinfo->controllist,"mode")) != NULL) { | 
| 266 | 0 |   if((stat = NCZ_comma_parse(value,modelist))) goto done; | 
| 267 | 0 |     } | 
| 268 |  |     /* Process the modelist first */ | 
| 269 | 0 |     zinfo->controls.mapimpl = NCZM_DEFAULT; | 
| 270 | 0 |     zinfo->controls.flags |= FLAG_XARRAYDIMS; /* Always support XArray convention where possible */ | 
| 271 | 0 |     for(i=0;i<nclistlength(modelist);i++) { | 
| 272 | 0 |         const char* p = nclistget(modelist,i); | 
| 273 | 0 |   if(strcasecmp(p,PUREZARRCONTROL)==0) | 
| 274 | 0 |       zinfo->controls.flags |= (FLAG_PUREZARR); | 
| 275 | 0 |   else if(strcasecmp(p,XARRAYCONTROL)==0) | 
| 276 | 0 |       zinfo->controls.flags |= FLAG_PUREZARR; | 
| 277 | 0 |   else if(strcasecmp(p,NOXARRAYCONTROL)==0) | 
| 278 | 0 |       noflags |= FLAG_XARRAYDIMS; | 
| 279 | 0 |   else if(strcasecmp(p,"zip")==0) zinfo->controls.mapimpl = NCZM_ZIP; | 
| 280 | 0 |   else if(strcasecmp(p,"file")==0) zinfo->controls.mapimpl = NCZM_FILE; | 
| 281 | 0 |   else if(strcasecmp(p,"s3")==0) zinfo->controls.mapimpl = NCZM_S3; | 
| 282 | 0 |     } | 
| 283 |  |     /* Apply negative controls by turning off negative flags */ | 
| 284 |  |     /* This is necessary to avoid order dependence of mode flags when both positive and negative flags are defined */ | 
| 285 | 0 |     zinfo->controls.flags &= (~noflags); | 
| 286 |  |  | 
| 287 |  |     /* Process other controls */ | 
| 288 | 0 |     if((value = controllookup(zinfo->controllist,"log")) != NULL) { | 
| 289 | 0 |   zinfo->controls.flags |= FLAG_LOGGING; | 
| 290 | 0 |         ncsetloglevel(NCLOGNOTE); | 
| 291 | 0 |     } | 
| 292 | 0 |     if((value = controllookup(zinfo->controllist,"show")) != NULL) { | 
| 293 | 0 |   if(strcasecmp(value,"fetch")==0) | 
| 294 | 0 |       zinfo->controls.flags |= FLAG_SHOWFETCH; | 
| 295 | 0 |     } | 
| 296 | 0 | done: | 
| 297 | 0 |     nclistfreeall(modelist); | 
| 298 | 0 |     return stat; | 
| 299 | 0 | } |