Line | Count | Source |
1 | | #include <assert.h> |
2 | | #include <errno.h> |
3 | | #include <limits.h> |
4 | | #include <string.h> |
5 | | #include <stdio.h> |
6 | | #include <stdlib.h> |
7 | | #include <stdint.h> |
8 | | #include <sys/stat.h> |
9 | | #include <libgen.h> |
10 | | |
11 | | #ifdef WIN32 |
12 | | #include <shlwapi.h> |
13 | | #endif |
14 | | |
15 | | #include "jq_parser.h" |
16 | | #include "locfile.h" |
17 | | #include "jv.h" |
18 | | #include "jq.h" |
19 | | #include "parser.h" |
20 | | #include "util.h" |
21 | | #include "compile.h" |
22 | | #include "jv_alloc.h" |
23 | | |
24 | | struct lib_loading_state { |
25 | | char **names; |
26 | | block *defs; |
27 | | uint64_t ct; |
28 | | }; |
29 | | static int load_library(jq_state *jq, jv lib_path, |
30 | | int is_data, int raw, int optional, |
31 | | const char *as, block *out_block, |
32 | | struct lib_loading_state *lib_state); |
33 | | |
34 | 0 | static int path_is_relative(jv p) { |
35 | 0 | const char *s = jv_string_value(p); |
36 | |
|
37 | | #ifdef WIN32 |
38 | | int res = PathIsRelativeA(s); |
39 | | #else |
40 | 0 | int res = *s != '/'; |
41 | 0 | #endif |
42 | 0 | jv_free(p); |
43 | 0 | return res; |
44 | 0 | } |
45 | | |
46 | | |
47 | | // Given a lib_path to search first, creates a chain of search paths |
48 | | // in the following order: |
49 | | // 1. lib_path |
50 | | // 2. -L paths passed in on the command line (from jq_state*) or builtin list |
51 | 0 | static jv build_lib_search_chain(jq_state *jq, jv search_path, jv jq_origin, jv lib_origin) { |
52 | 0 | assert(jv_get_kind(search_path) == JV_KIND_ARRAY); |
53 | 0 | jv expanded = jv_array(); |
54 | 0 | jv expanded_elt; |
55 | 0 | jv err = jv_null(); |
56 | 0 | jv_array_foreach(search_path, i, path) { |
57 | 0 | if (jv_get_kind(path) != JV_KIND_STRING) { |
58 | 0 | jv_free(path); |
59 | 0 | continue; |
60 | 0 | } |
61 | 0 | path = expand_path(path); |
62 | 0 | if (!jv_is_valid(path)) { |
63 | 0 | err = path; |
64 | 0 | path = jv_null(); |
65 | 0 | continue; |
66 | 0 | } |
67 | 0 | if (strcmp(".",jv_string_value(path)) == 0) { |
68 | 0 | expanded_elt = jv_copy(path); |
69 | 0 | } else if (strncmp("$ORIGIN/",jv_string_value(path),sizeof("$ORIGIN/") - 1) == 0) { |
70 | 0 | expanded_elt = jv_string_fmt("%s/%s", |
71 | 0 | jv_string_value(jq_origin), |
72 | 0 | jv_string_value(path) + sizeof ("$ORIGIN/") - 1); |
73 | 0 | } else if (jv_get_kind(lib_origin) == JV_KIND_STRING && |
74 | 0 | path_is_relative(jv_copy(path))) { |
75 | 0 | expanded_elt = jv_string_fmt("%s/%s", |
76 | 0 | jv_string_value(lib_origin), |
77 | 0 | jv_string_value(path)); |
78 | 0 | } else { |
79 | 0 | expanded_elt = path; |
80 | 0 | path = jv_invalid(); |
81 | 0 | } |
82 | 0 | expanded = jv_array_append(expanded, expanded_elt); |
83 | 0 | jv_free(path); |
84 | 0 | } |
85 | 0 | jv_free(jq_origin); |
86 | 0 | jv_free(lib_origin); |
87 | 0 | jv_free(search_path); |
88 | 0 | return JV_ARRAY(expanded, err); |
89 | 0 | } |
90 | | |
91 | | // Doesn't actually check that name not be an absolute path, and we |
92 | | // don't have to: we always append relative paths to others (with a '/' |
93 | | // in between). |
94 | 0 | static jv validate_relpath(jv name) { |
95 | 0 | const char *s = jv_string_value(name); |
96 | 0 | if (strchr(s, '\\')) { |
97 | 0 | jv res = jv_invalid_with_msg(jv_string_fmt("Modules must be named by relative paths using '/', not '\\' (%s)", s)); |
98 | 0 | jv_free(name); |
99 | 0 | return res; |
100 | 0 | } |
101 | 0 | jv components = jv_string_split(jv_copy(name), jv_string("/")); |
102 | 0 | jv_array_foreach(components, i, x) { |
103 | 0 | if (!strcmp(jv_string_value(x), "..")) { |
104 | 0 | jv_free(x); |
105 | 0 | jv_free(components); |
106 | 0 | jv res = jv_invalid_with_msg(jv_string_fmt("Relative paths to modules may not traverse to parent directories (%s)", s)); |
107 | 0 | jv_free(name); |
108 | 0 | return res; |
109 | 0 | } |
110 | 0 | if (i > 0 && jv_equal(jv_copy(x), jv_array_get(jv_copy(components), i - 1))) { |
111 | 0 | jv_free(x); |
112 | 0 | jv_free(components); |
113 | 0 | jv res = jv_invalid_with_msg(jv_string_fmt("module names must not have equal consecutive components: %s", |
114 | 0 | jv_string_value(name))); |
115 | 0 | jv_free(name); |
116 | 0 | return res; |
117 | 0 | } |
118 | 0 | jv_free(x); |
119 | 0 | } |
120 | 0 | jv_free(components); |
121 | 0 | return name; |
122 | 0 | } |
123 | | |
124 | | // Assumes name has been validated |
125 | 0 | static jv jv_basename(jv name) { |
126 | 0 | const char *s = jv_string_value(name); |
127 | 0 | const char *p = strrchr(s, '/'); |
128 | 0 | if (!p) |
129 | 0 | return name; |
130 | 0 | jv res = jv_string_fmt("%s", p); |
131 | 0 | jv_free(name); |
132 | 0 | return res; |
133 | 0 | } |
134 | | |
135 | | // Asummes validated relative path to module |
136 | 0 | static jv find_lib(jq_state *jq, jv rel_path, jv search, const char *suffix, jv jq_origin, jv lib_origin) { |
137 | 0 | if (!jv_is_valid(rel_path)) { |
138 | 0 | jv_free(search); |
139 | 0 | jv_free(jq_origin); |
140 | 0 | jv_free(lib_origin); |
141 | 0 | return rel_path; |
142 | 0 | } |
143 | 0 | if (jv_get_kind(rel_path) != JV_KIND_STRING) { |
144 | 0 | jv_free(rel_path); |
145 | 0 | jv_free(search); |
146 | 0 | jv_free(jq_origin); |
147 | 0 | jv_free(lib_origin); |
148 | 0 | return jv_invalid_with_msg(jv_string_fmt("Module path must be a string")); |
149 | 0 | } |
150 | 0 | if (jv_get_kind(search) != JV_KIND_ARRAY) { |
151 | 0 | jv_free(rel_path); |
152 | 0 | jv_free(search); |
153 | 0 | jv_free(jq_origin); |
154 | 0 | jv_free(lib_origin); |
155 | 0 | return jv_invalid_with_msg(jv_string_fmt("Module search path must be an array")); |
156 | 0 | } |
157 | | |
158 | 0 | struct stat st; |
159 | 0 | int ret; |
160 | | |
161 | | // Ideally we should cache this somewhere |
162 | 0 | search = build_lib_search_chain(jq, search, jq_origin, lib_origin); |
163 | 0 | jv err = jv_array_get(jv_copy(search), 1); |
164 | 0 | search = jv_array_get(search, 0); |
165 | |
|
166 | 0 | jv bname = jv_basename(jv_copy(rel_path)); |
167 | |
|
168 | 0 | jv_array_foreach(search, i, spath) { |
169 | 0 | if (jv_get_kind(spath) == JV_KIND_NULL) { |
170 | 0 | jv_free(spath); |
171 | 0 | break; |
172 | 0 | } |
173 | 0 | if (jv_get_kind(spath) != JV_KIND_STRING || |
174 | 0 | strcmp(jv_string_value(spath), "") == 0) { |
175 | 0 | jv_free(spath); |
176 | 0 | continue; /* XXX report non-strings in search path?? */ |
177 | 0 | } |
178 | | // Try ${search_dir}/${rel_path}.jq |
179 | 0 | jv testpath = jq_realpath(jv_string_fmt("%s/%s%s", |
180 | 0 | jv_string_value(spath), |
181 | 0 | jv_string_value(rel_path), |
182 | 0 | suffix)); |
183 | 0 | ret = stat(jv_string_value(testpath),&st); |
184 | 0 | if (ret == -1 && errno == ENOENT) { |
185 | 0 | jv_free(testpath); |
186 | | // Try ${search_dir}/$(dirname ${rel_path})/jq/main.jq |
187 | 0 | testpath = jq_realpath(jv_string_fmt("%s/%s/%s%s", |
188 | 0 | jv_string_value(spath), |
189 | 0 | jv_string_value(rel_path), |
190 | 0 | "jq/main", |
191 | 0 | suffix)); |
192 | 0 | ret = stat(jv_string_value(testpath),&st); |
193 | 0 | } |
194 | 0 | if (ret == -1 && errno == ENOENT) { |
195 | 0 | jv_free(testpath); |
196 | | // Try ${search_dir}/${rel_path}/$(basename ${rel_path}).jq |
197 | 0 | testpath = jq_realpath(jv_string_fmt("%s/%s/%s%s", |
198 | 0 | jv_string_value(spath), |
199 | 0 | jv_string_value(rel_path), |
200 | 0 | jv_string_value(bname), |
201 | 0 | suffix)); |
202 | 0 | ret = stat(jv_string_value(testpath),&st); |
203 | 0 | } |
204 | 0 | if (ret == 0) { |
205 | 0 | jv_free(err); |
206 | 0 | jv_free(rel_path); |
207 | 0 | jv_free(search); |
208 | 0 | jv_free(bname); |
209 | 0 | jv_free(spath); |
210 | 0 | return testpath; |
211 | 0 | } |
212 | 0 | jv_free(testpath); |
213 | 0 | jv_free(spath); |
214 | 0 | } |
215 | 0 | jv output; |
216 | 0 | if (!jv_is_valid(err)) { |
217 | 0 | err = jv_invalid_get_msg(err); |
218 | 0 | output = jv_invalid_with_msg(jv_string_fmt("module not found: %s (%s)", |
219 | 0 | jv_string_value(rel_path), |
220 | 0 | jv_string_value(err))); |
221 | 0 | } else { |
222 | 0 | output = jv_invalid_with_msg(jv_string_fmt("module not found: %s", |
223 | 0 | jv_string_value(rel_path))); |
224 | 0 | } |
225 | 0 | jv_free(err); |
226 | 0 | jv_free(rel_path); |
227 | 0 | jv_free(search); |
228 | 0 | jv_free(bname); |
229 | 0 | return output; |
230 | 0 | } |
231 | | |
232 | 0 | static jv default_search(jq_state *jq, jv value) { |
233 | 0 | if (!jv_is_valid(value)) { |
234 | | // dependent didn't say; prepend . to system search path listj |
235 | 0 | jv_free(value); |
236 | 0 | return jv_array_concat(JV_ARRAY(jv_string(".")), jq_get_lib_dirs(jq)); |
237 | 0 | } |
238 | 0 | if (jv_get_kind(value) != JV_KIND_ARRAY) |
239 | 0 | return JV_ARRAY(value); |
240 | 0 | return value; |
241 | 0 | } |
242 | | |
243 | | // XXX Split this into a util that takes a callback, and then... |
244 | 0 | static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block *src_block, struct lib_loading_state *lib_state) { |
245 | 0 | jv deps = block_take_imports(src_block); |
246 | 0 | block bk = *src_block; |
247 | 0 | int nerrors = 0; |
248 | | |
249 | | // XXX This is a backward jv_array_foreach because bindings go in reverse |
250 | 0 | for (int i = jv_array_length(jv_copy(deps)); i > 0; ) { |
251 | 0 | i--; |
252 | 0 | jv dep = jv_array_get(jv_copy(deps), i); |
253 | |
|
254 | 0 | const char *as_str = NULL; |
255 | 0 | int is_data = jv_get_kind(jv_object_get(jv_copy(dep), jv_string("is_data"))) == JV_KIND_TRUE; |
256 | 0 | int raw = 0; |
257 | 0 | jv v = jv_object_get(jv_copy(dep), jv_string("raw")); |
258 | 0 | if (jv_get_kind(v) == JV_KIND_TRUE) |
259 | 0 | raw = 1; |
260 | 0 | int optional = 0; |
261 | 0 | if (jv_get_kind(jv_object_get(jv_copy(dep), jv_string("optional"))) == JV_KIND_TRUE) |
262 | 0 | optional = 1; |
263 | 0 | jv_free(v); |
264 | 0 | jv relpath = validate_relpath(jv_object_get(jv_copy(dep), jv_string("relpath"))); |
265 | 0 | jv as = jv_object_get(jv_copy(dep), jv_string("as")); |
266 | 0 | assert(!jv_is_valid(as) || jv_get_kind(as) == JV_KIND_STRING); |
267 | 0 | if (jv_get_kind(as) == JV_KIND_STRING) |
268 | 0 | as_str = jv_string_value(as); |
269 | 0 | jv search = default_search(jq, jv_object_get(dep, jv_string("search"))); |
270 | | // dep is now freed; do not reuse |
271 | | |
272 | | // find_lib does a lot of work that could be cached... |
273 | 0 | jv resolved = find_lib(jq, relpath, search, is_data ? ".json" : ".jq", jv_copy(jq_origin), jv_copy(lib_origin)); |
274 | | // XXX ...move the rest of this into a callback. |
275 | 0 | if (!jv_is_valid(resolved)) { |
276 | 0 | jv_free(as); |
277 | 0 | if (optional) { |
278 | 0 | jv_free(resolved); |
279 | 0 | continue; |
280 | 0 | } |
281 | 0 | jv emsg = jv_invalid_get_msg(resolved); |
282 | 0 | jq_report_error(jq, jv_string_fmt("jq: error: %s\n",jv_string_value(emsg))); |
283 | 0 | jv_free(emsg); |
284 | 0 | jv_free(deps); |
285 | 0 | jv_free(jq_origin); |
286 | 0 | jv_free(lib_origin); |
287 | 0 | return 1; |
288 | 0 | } |
289 | | |
290 | 0 | if (is_data) { |
291 | | // Can't reuse data libs because the wrong name is bound |
292 | 0 | block dep_def_block; |
293 | 0 | nerrors += load_library(jq, resolved, is_data, raw, optional, as_str, &dep_def_block, lib_state); |
294 | 0 | if (nerrors == 0) { |
295 | | // Bind as both $data::data and $data for backward compatibility vs common sense |
296 | 0 | bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, as_str); |
297 | 0 | bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, NULL); |
298 | 0 | } |
299 | 0 | } else { |
300 | 0 | uint64_t state_idx = 0; |
301 | 0 | for (; state_idx < lib_state->ct; ++state_idx) { |
302 | 0 | if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0) |
303 | 0 | break; |
304 | 0 | } |
305 | |
|
306 | 0 | if (state_idx < lib_state->ct) { // Found |
307 | 0 | jv_free(resolved); |
308 | | // Bind the library to the program |
309 | 0 | bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, as_str); |
310 | 0 | } else { // Not found. Add it to the table before binding. |
311 | 0 | block dep_def_block = gen_noop(); |
312 | 0 | nerrors += load_library(jq, resolved, is_data, raw, optional, as_str, &dep_def_block, lib_state); |
313 | | // resolved has been freed |
314 | 0 | if (nerrors == 0) { |
315 | | // Bind the library to the program |
316 | 0 | bk = block_bind_library(dep_def_block, bk, OP_IS_CALL_PSEUDO, as_str); |
317 | 0 | } |
318 | 0 | } |
319 | 0 | } |
320 | |
|
321 | 0 | jv_free(as); |
322 | 0 | } |
323 | 0 | jv_free(lib_origin); |
324 | 0 | jv_free(jq_origin); |
325 | 0 | jv_free(deps); |
326 | 0 | return nerrors; |
327 | 0 | } |
328 | | |
329 | | // Loads the library at lib_path into lib_state, putting the library's defs |
330 | | // into *out_block |
331 | 0 | static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, int optional, const char *as, block *out_block, struct lib_loading_state *lib_state) { |
332 | 0 | int nerrors = 0; |
333 | 0 | struct locfile* src = NULL; |
334 | 0 | block program; |
335 | 0 | jv data; |
336 | 0 | if (is_data && !raw) |
337 | 0 | data = jv_load_file(jv_string_value(lib_path), 0); |
338 | 0 | else |
339 | 0 | data = jv_load_file(jv_string_value(lib_path), 1); |
340 | 0 | int state_idx; |
341 | 0 | if (!jv_is_valid(data)) { |
342 | 0 | program = gen_noop(); |
343 | 0 | if (!optional) { |
344 | 0 | if (jv_invalid_has_msg(jv_copy(data))) |
345 | 0 | data = jv_invalid_get_msg(data); |
346 | 0 | else |
347 | 0 | data = jv_string("unknown error"); |
348 | 0 | jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n", jv_string_value(lib_path), jv_string_value(data))); |
349 | 0 | nerrors++; |
350 | 0 | } |
351 | 0 | goto out; |
352 | 0 | } else if (is_data) { |
353 | | // import "foo" as $bar; |
354 | 0 | program = gen_const_global(jv_copy(data), as); |
355 | 0 | } else { |
356 | | // import "foo" as bar; |
357 | 0 | src = locfile_init(jq, jv_string_value(lib_path), jv_string_value(data), jv_string_length_bytes(jv_copy(data))); |
358 | 0 | nerrors += jq_parse_library(src, &program); |
359 | 0 | locfile_free(src); |
360 | 0 | if (nerrors == 0) { |
361 | 0 | char *lib_origin = strdup(jv_string_value(lib_path)); |
362 | 0 | nerrors += process_dependencies(jq, jq_get_jq_origin(jq), |
363 | 0 | jv_string(dirname(lib_origin)), |
364 | 0 | &program, lib_state); |
365 | 0 | free(lib_origin); |
366 | 0 | program = block_bind_self(program, OP_IS_CALL_PSEUDO); |
367 | 0 | } |
368 | 0 | } |
369 | 0 | state_idx = lib_state->ct++; |
370 | 0 | lib_state->names = jv_mem_realloc(lib_state->names, lib_state->ct * sizeof(const char *)); |
371 | 0 | lib_state->defs = jv_mem_realloc(lib_state->defs, lib_state->ct * sizeof(block)); |
372 | 0 | lib_state->names[state_idx] = strdup(jv_string_value(lib_path)); |
373 | 0 | lib_state->defs[state_idx] = program; |
374 | 0 | out: |
375 | 0 | *out_block = program; |
376 | 0 | jv_free(lib_path); |
377 | 0 | jv_free(data); |
378 | 0 | return nerrors; |
379 | 0 | } |
380 | | |
381 | | // FIXME It'd be nice to have an option to search the same search path |
382 | | // as we do in process_dependencies. |
383 | 0 | jv load_module_meta(jq_state *jq, jv mod_relpath) { |
384 | | // We can't know the caller's origin; we could though, if it was passed in |
385 | 0 | jv lib_path = find_lib(jq, validate_relpath(mod_relpath), jq_get_lib_dirs(jq), ".jq", jq_get_jq_origin(jq), jv_null()); |
386 | 0 | if (!jv_is_valid(lib_path)) |
387 | 0 | return lib_path; |
388 | 0 | jv meta = jv_null(); |
389 | 0 | jv data = jv_load_file(jv_string_value(lib_path), 1); |
390 | 0 | if (jv_is_valid(data)) { |
391 | 0 | block program; |
392 | 0 | struct locfile* src = locfile_init(jq, jv_string_value(lib_path), jv_string_value(data), jv_string_length_bytes(jv_copy(data))); |
393 | 0 | int nerrors = jq_parse_library(src, &program); |
394 | 0 | if (nerrors == 0) { |
395 | 0 | meta = block_module_meta(program); |
396 | 0 | if (jv_get_kind(meta) == JV_KIND_NULL) |
397 | 0 | meta = jv_object(); |
398 | 0 | meta = jv_object_set(meta, jv_string("deps"), block_take_imports(&program)); |
399 | 0 | meta = jv_object_set(meta, jv_string("defs"), block_list_funcs(program, 0)); |
400 | 0 | } |
401 | 0 | locfile_free(src); |
402 | 0 | block_free(program); |
403 | 0 | } |
404 | 0 | jv_free(lib_path); |
405 | 0 | jv_free(data); |
406 | 0 | return meta; |
407 | 0 | } |
408 | | |
409 | 0 | int load_program(jq_state *jq, struct locfile* src, block *out_block) { |
410 | 0 | int nerrors = 0; |
411 | 0 | block program; |
412 | 0 | struct lib_loading_state lib_state = {0,0,0}; |
413 | 0 | nerrors = jq_parse(src, &program); |
414 | 0 | if (nerrors) |
415 | 0 | return nerrors; |
416 | | |
417 | 0 | if (!block_has_main(program)) { |
418 | 0 | jq_report_error(jq, jv_string("jq: error: Top-level program not given (try \".\")")); |
419 | 0 | block_free(program); |
420 | 0 | return 1; |
421 | 0 | } |
422 | | |
423 | 0 | jv home = get_home(); |
424 | 0 | if (jv_is_valid(home)) { |
425 | | /* Import ~/.jq as a library named "" found in $HOME or %USERPROFILE% */ |
426 | 0 | block import = gen_import_meta(gen_import("", NULL, 0), |
427 | 0 | gen_const(JV_OBJECT( |
428 | 0 | jv_string("optional"), jv_true(), |
429 | 0 | jv_string("search"), home))); |
430 | 0 | program = BLOCK(import, program); |
431 | 0 | } else { // silently ignore if home dir not determined |
432 | 0 | jv_free(home); |
433 | 0 | } |
434 | |
|
435 | 0 | nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state); |
436 | 0 | block libs = gen_noop(); |
437 | 0 | for (uint64_t i = 0; i < lib_state.ct; ++i) { |
438 | 0 | free(lib_state.names[i]); |
439 | 0 | if (nerrors == 0 && !block_is_const(lib_state.defs[i])) |
440 | 0 | libs = block_join(libs, lib_state.defs[i]); |
441 | 0 | else |
442 | 0 | block_free(lib_state.defs[i]); |
443 | 0 | } |
444 | 0 | free(lib_state.names); |
445 | 0 | free(lib_state.defs); |
446 | 0 | if (nerrors) |
447 | 0 | block_free(program); |
448 | 0 | else |
449 | 0 | *out_block = block_drop_unreferenced(block_join(libs, program)); |
450 | |
|
451 | 0 | return nerrors; |
452 | 0 | } |