/src/netcdf-c/libsrc/httpio.c
Line | Count | Source (jump to first uncovered line) |
1 | | /********************************************************************* |
2 | | * Copyright 2018, UCAR/Unidata |
3 | | * See netcdf/COPYRIGHT file for copying and redistribution conditions. |
4 | | * ********************************************************************/ |
5 | | |
6 | | #if HAVE_CONFIG_H |
7 | | #include <config.h> |
8 | | #endif |
9 | | |
10 | | #include <assert.h> |
11 | | #include <stdlib.h> |
12 | | #include <errno.h> |
13 | | #include <string.h> |
14 | | #ifdef HAVE_FCNTL_H |
15 | | #include <fcntl.h> |
16 | | #endif |
17 | | #ifdef _MSC_VER /* Microsoft Compilers */ |
18 | | #include <io.h> |
19 | | #endif |
20 | | #ifdef HAVE_UNISTD_H |
21 | | #include <unistd.h> |
22 | | #endif |
23 | | #include "nc3internal.h" |
24 | | #include "nclist.h" |
25 | | #include "ncbytes.h" |
26 | | #include "ncuri.h" |
27 | | |
28 | | #undef DEBUG |
29 | | |
30 | | #ifdef DEBUG |
31 | | #include <stdio.h> |
32 | | #endif |
33 | | |
34 | | #include <curl/curl.h> |
35 | | |
36 | | #include "ncio.h" |
37 | | #include "fbits.h" |
38 | | #include "rnd.h" |
39 | | #include "ncbytes.h" |
40 | | #include "nchttp.h" |
41 | | |
42 | 0 | #define DEFAULTPAGESIZE 16384 |
43 | | |
44 | | /* Private data */ |
45 | | |
46 | | typedef struct NCHTTP { |
47 | | NC_HTTP_STATE* state; |
48 | | long long size; /* of the object */ |
49 | | NCbytes* region; |
50 | | } NCHTTP; |
51 | | |
52 | | /* Forward */ |
53 | | static int httpio_rel(ncio *const nciop, off_t offset, int rflags); |
54 | | static int httpio_get(ncio *const nciop, off_t offset, size_t extent, int rflags, void **const vpp); |
55 | | static int httpio_move(ncio *const nciop, off_t to, off_t from, size_t nbytes, int rflags); |
56 | | static int httpio_sync(ncio *const nciop); |
57 | | static int httpio_filesize(ncio* nciop, off_t* filesizep); |
58 | | static int httpio_pad_length(ncio* nciop, off_t length); |
59 | | static int httpio_close(ncio* nciop, int); |
60 | | |
61 | | static long pagesize = 0; |
62 | | |
63 | | /* Create a new ncio struct to hold info about the file. */ |
64 | | static int |
65 | | httpio_new(const char* path, int ioflags, ncio** nciopp, NCHTTP** hpp) |
66 | 0 | { |
67 | 0 | int status = NC_NOERR; |
68 | 0 | ncio* nciop = NULL; |
69 | 0 | NCHTTP* http = NULL; |
70 | |
|
71 | 0 | if(pagesize == 0) |
72 | 0 | pagesize = DEFAULTPAGESIZE; |
73 | 0 | errno = 0; |
74 | |
|
75 | 0 | nciop = (ncio* )calloc(1,sizeof(ncio)); |
76 | 0 | if(nciop == NULL) {status = NC_ENOMEM; goto fail;} |
77 | | |
78 | 0 | nciop->ioflags = ioflags; |
79 | |
|
80 | 0 | *((char**)&nciop->path) = strdup(path); |
81 | 0 | if(nciop->path == NULL) {status = NC_ENOMEM; goto fail;} |
82 | | |
83 | 0 | *((ncio_relfunc**)&nciop->rel) = httpio_rel; |
84 | 0 | *((ncio_getfunc**)&nciop->get) = httpio_get; |
85 | 0 | *((ncio_movefunc**)&nciop->move) = httpio_move; |
86 | 0 | *((ncio_syncfunc**)&nciop->sync) = httpio_sync; |
87 | 0 | *((ncio_filesizefunc**)&nciop->filesize) = httpio_filesize; |
88 | 0 | *((ncio_pad_lengthfunc**)&nciop->pad_length) = httpio_pad_length; |
89 | 0 | *((ncio_closefunc**)&nciop->close) = httpio_close; |
90 | |
|
91 | 0 | http = (NCHTTP*)calloc(1,sizeof(NCHTTP)); |
92 | 0 | if(http == NULL) {status = NC_ENOMEM; goto fail;} |
93 | 0 | *((void* *)&nciop->pvt) = http; |
94 | |
|
95 | 0 | if(nciopp) *nciopp = nciop; |
96 | 0 | if(hpp) *hpp = http; |
97 | |
|
98 | 0 | done: |
99 | 0 | return status; |
100 | | |
101 | 0 | fail: |
102 | 0 | if(http != NULL) { |
103 | 0 | if(http->region) |
104 | 0 | ncbytesfree(http->region); |
105 | 0 | free(http); |
106 | 0 | } |
107 | 0 | if(nciop != NULL) { |
108 | 0 | if(nciop->path != NULL) free((char*)nciop->path); |
109 | 0 | } |
110 | 0 | goto done; |
111 | 0 | } |
112 | | |
113 | | /* Create a file, and the ncio struct to go with it. This function is |
114 | | only called from nc__create_mp. |
115 | | |
116 | | path - path of file to create. |
117 | | ioflags - flags from nc_create |
118 | | initialsz - From the netcdf man page: "The argument |
119 | | Iinitialsize sets the initial size of the file at creation time." |
120 | | igeto - |
121 | | igetsz - |
122 | | sizehintp - the size of a page of data for buffered reads and writes. |
123 | | nciopp - pointer to a pointer that will get location of newly |
124 | | created and inited ncio struct. |
125 | | mempp - pointer to pointer to the initial memory read. |
126 | | */ |
127 | | int |
128 | | httpio_create(const char* path, int ioflags, |
129 | | size_t initialsz, |
130 | | off_t igeto, size_t igetsz, size_t* sizehintp, |
131 | | void* parameters, |
132 | | ncio* *nciopp, void** const mempp) |
133 | 0 | { |
134 | 0 | return NC_EPERM; |
135 | 0 | } |
136 | | |
137 | | /* This function opens the data file. It is only called from nc.c, |
138 | | from nc__open_mp and nc_delete_mp. |
139 | | |
140 | | path - path of data file. |
141 | | ioflags - flags passed into nc_open. |
142 | | igeto - looks like this function can do an initial page get, and |
143 | | igeto is going to be the offset for that. But it appears to be |
144 | | unused |
145 | | igetsz - the size in bytes of initial page get (a.k.a. extent). Not |
146 | | ever used in the library. |
147 | | sizehintp - the size of a page of data for buffered reads and writes. |
148 | | nciopp - pointer to pointer that will get address of newly created |
149 | | and inited ncio struct. |
150 | | mempp - pointer to pointer to the initial memory read. |
151 | | */ |
152 | | int |
153 | | httpio_open(const char* path, |
154 | | int ioflags, |
155 | | /* ignored */ off_t igeto, size_t igetsz, size_t* sizehintp, |
156 | | /* ignored */ void* parameters, |
157 | | ncio* *nciopp, |
158 | | /* ignored */ void** const mempp) |
159 | 0 | { |
160 | 0 | ncio* nciop; |
161 | 0 | int status; |
162 | 0 | NCHTTP* http = NULL; |
163 | 0 | size_t sizehint; |
164 | 0 | NCURI* uri = NULL; |
165 | |
|
166 | 0 | if(path == NULL ||* path == 0) |
167 | 0 | return EINVAL; |
168 | | |
169 | 0 | ncuriparse(path,&uri); |
170 | 0 | if(uri == NULL) {status = NC_EURL; goto done;} |
171 | | |
172 | | /* Create private data */ |
173 | 0 | if((status = httpio_new(path, ioflags, &nciop, &http))) goto done; |
174 | | /* Open the path and get curl handle and object size */ |
175 | 0 | if((status = nc_http_open(path,&http->state))) goto done; |
176 | 0 | if((status = nc_http_size(http->state,&http->size))) goto done; |
177 | | |
178 | 0 | sizehint = pagesize; |
179 | | |
180 | | /* sizehint must be multiple of 8 */ |
181 | 0 | sizehint = (sizehint / 8) * 8; |
182 | 0 | if(sizehint < 8) sizehint = 8; |
183 | |
|
184 | 0 | *sizehintp = sizehint; |
185 | 0 | *nciopp = nciop; |
186 | 0 | done: |
187 | 0 | if(status) |
188 | 0 | httpio_close(nciop,0); |
189 | 0 | return status; |
190 | 0 | } |
191 | | |
192 | | /* |
193 | | * Get file size in bytes. |
194 | | */ |
195 | | static int |
196 | | httpio_filesize(ncio* nciop, off_t* filesizep) |
197 | 0 | { |
198 | 0 | NCHTTP* http; |
199 | 0 | if(nciop == NULL || nciop->pvt == NULL) return NC_EINVAL; |
200 | 0 | http = (NCHTTP*)nciop->pvt; |
201 | 0 | if(filesizep != NULL) *filesizep = http->size; |
202 | 0 | return NC_NOERR; |
203 | 0 | } |
204 | | |
205 | | /* |
206 | | * Sync any changes to disk, then truncate or extend file so its size |
207 | | * is length. This is only intended to be called before close, if the |
208 | | * file is open for writing and the actual size does not match the |
209 | | * calculated size, perhaps as the result of having been previously |
210 | | * written in NOFILL mode. |
211 | | */ |
212 | | static int |
213 | | httpio_pad_length(ncio* nciop, off_t length) |
214 | 0 | { |
215 | 0 | return NC_NOERR; /* do nothing */ |
216 | 0 | } |
217 | | |
218 | | /* Write out any dirty buffers to disk and |
219 | | ensure that next read will get data from disk. |
220 | | Sync any changes, then close the open file associated with the ncio |
221 | | struct, and free its memory. |
222 | | nciop - pointer to ncio to close. |
223 | | doUnlink - if true, unlink file |
224 | | */ |
225 | | |
226 | | static int |
227 | | httpio_close(ncio* nciop, int doUnlink) |
228 | 0 | { |
229 | 0 | int status = NC_NOERR; |
230 | 0 | NCHTTP* http; |
231 | 0 | if(nciop == NULL || nciop->pvt == NULL) return NC_NOERR; |
232 | | |
233 | 0 | http = (NCHTTP*)nciop->pvt; |
234 | 0 | assert(http != NULL); |
235 | | |
236 | 0 | status = nc_http_close(http->state); |
237 | | |
238 | | /* do cleanup */ |
239 | 0 | if(http != NULL) { |
240 | 0 | ncbytesfree(http->region); |
241 | 0 | free(http); |
242 | 0 | } |
243 | 0 | if(nciop->path != NULL) free((char*)nciop->path); |
244 | 0 | free(nciop); |
245 | 0 | return status; |
246 | 0 | } |
247 | | |
248 | | /* |
249 | | * Request that the region (offset, extent) |
250 | | * be made available through *vpp. |
251 | | */ |
252 | | static int |
253 | | httpio_get(ncio* const nciop, off_t offset, size_t extent, int rflags, void** const vpp) |
254 | 0 | { |
255 | 0 | int status = NC_NOERR; |
256 | 0 | NCHTTP* http; |
257 | |
|
258 | 0 | if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;} |
259 | 0 | http = (NCHTTP*)nciop->pvt; |
260 | |
|
261 | 0 | assert(http->region == NULL); |
262 | 0 | http->region = ncbytesnew(); |
263 | 0 | ncbytessetalloc(http->region,(unsigned long)extent); |
264 | 0 | if((status = nc_http_read(http->state,offset,extent,http->region))) |
265 | 0 | goto done; |
266 | 0 | assert(ncbyteslength(http->region) == extent); |
267 | 0 | if(vpp) *vpp = ncbytescontents(http->region); |
268 | 0 | done: |
269 | 0 | return status; |
270 | 0 | } |
271 | | |
272 | | /* |
273 | | * Like memmove(), safely move possibly overlapping data. |
274 | | */ |
275 | | static int |
276 | | httpio_move(ncio* const nciop, off_t to, off_t from, size_t nbytes, int ignored) |
277 | 0 | { |
278 | 0 | return NC_EPERM; |
279 | 0 | } |
280 | | |
281 | | static int |
282 | | httpio_rel(ncio* const nciop, off_t offset, int rflags) |
283 | 0 | { |
284 | 0 | int status = NC_NOERR; |
285 | 0 | NCHTTP* http; |
286 | |
|
287 | 0 | if(nciop == NULL || nciop->pvt == NULL) {status = NC_EINVAL; goto done;} |
288 | 0 | http = (NCHTTP*)nciop->pvt; |
289 | 0 | ncbytesfree(http->region); |
290 | 0 | http->region = NULL; |
291 | 0 | done: |
292 | 0 | return status; |
293 | 0 | } |
294 | | |
295 | | /* |
296 | | * Write out any dirty buffers to disk and |
297 | | * ensure that next read will get data from disk. |
298 | | */ |
299 | | static int |
300 | | httpio_sync(ncio* const nciop) |
301 | 0 | { |
302 | 0 | return NC_NOERR; /* do nothing */ |
303 | 0 | } |