/src/r-source/src/main/dcf.c
Line | Count | Source |
1 | | /* |
2 | | * R : A Computer Language for Statistical Data Analysis |
3 | | * Copyright (C) 2001-2026 The R Core Team. |
4 | | * |
5 | | * This program is free software; you can redistribute it and/or modify |
6 | | * it under the terms of the GNU General Public License as published by |
7 | | * the Free Software Foundation; either version 2 of the License, or |
8 | | * (at your option) any later version. |
9 | | * |
10 | | * This program is distributed in the hope that it will be useful, |
11 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
12 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
13 | | * GNU General Public License for more details. |
14 | | * |
15 | | * You should have received a copy of the GNU General Public License |
16 | | * along with this program; if not, a copy is available at |
17 | | * https://www.R-project.org/Licenses/ |
18 | | */ |
19 | | |
20 | | #ifdef HAVE_CONFIG_H |
21 | | # include <config.h> |
22 | | #endif |
23 | | |
24 | | #define R_USE_SIGNALS 1 |
25 | | #include <Defn.h> |
26 | | #include <Internal.h> |
27 | | #include <Rconnections.h> |
28 | | |
29 | | #include <tre/tre.h> |
30 | | |
31 | | static SEXP allocMatrixNA(SEXPTYPE, int, int); |
32 | | static void transferVector(SEXP s, SEXP t); |
33 | | |
34 | | /* Build a CHARSXP marked as UTF-8 from the NUL-terminated string 's'. |
35 | | DCF files are required to be UTF-8, so 's' is interpreted as UTF-8 |
36 | | regardless of its actual encoding; any invalid byte sequences are |
37 | | repaired by escaping them as "<xx>", as |
38 | | iconv(from = "UTF-8", to = "UTF-8", sub = "byte") does (which the R |
39 | | code paths in read.dcf()/write.dcf() also use). The common case of |
40 | | already-valid input is handled without conversion. |
41 | | |
42 | | The escaping is done with utf8Valid()'s rules rather than by iconv(): |
43 | | glibc and libiconv pass 4-byte sequences above U+10FFFF and the |
44 | | withdrawn 5- and 6-byte forms through, which utf8Valid() rejects, so |
45 | | an iconv() round trip could leave the result invalid and |
46 | | do_readDCF() would then re-encode the whole field value for every |
47 | | further continuation line. The same rules repair each input line |
48 | | before field names are matched, so a repaired name matches its |
49 | | later occurrences. */ |
50 | | static char *escapeInvalidUTF8(const char *s) |
51 | 11.9k | { |
52 | 11.9k | char *out = R_alloc(4 * strlen(s) + 1, sizeof(char)); /* all escaped */ |
53 | 11.9k | char *q = out; |
54 | 1.67M | for (const char *p = s; *p; ) { |
55 | 1.66M | if ((unsigned char) *p < 0x80) { |
56 | 391k | *q++ = *p++; |
57 | 391k | continue; |
58 | 391k | } |
59 | 1.27M | int len = utf8ValidClen(p); |
60 | 1.27M | if (len > 0) { |
61 | 21.0k | memcpy(q, p, len); |
62 | 21.0k | q += len; |
63 | 21.0k | p += len; |
64 | 1.25M | } else { |
65 | 1.25M | snprintf(q, 5, "<%02x>", (unsigned char) *p); |
66 | 1.25M | q += 4; |
67 | 1.25M | p++; |
68 | 1.25M | } |
69 | 1.27M | } |
70 | 11.9k | *q = '\0'; |
71 | 11.9k | return out; |
72 | 11.9k | } |
73 | | |
74 | | static SEXP mkCharUTF8sub(const char *s) |
75 | 275k | { |
76 | 275k | if (utf8Valid(s)) |
77 | 275k | return mkCharCE(s, CE_UTF8); |
78 | | |
79 | 0 | const void *vmax = vmaxget(); |
80 | 0 | SEXP ans = mkCharCE(escapeInvalidUTF8(s), CE_UTF8); |
81 | 0 | vmaxset(vmax); |
82 | 0 | return ans; |
83 | 275k | } |
84 | | |
85 | | /* What do_readDCF() has to release however it exits: the error() calls |
86 | | in its read loop would otherwise leak the line buffer and the compiled |
87 | | regexps, and leave open a connection it opened itself. */ |
88 | | typedef struct dcf_info { |
89 | | Rconnection con; |
90 | | bool wasopen; |
91 | | char **buf; /* the line buffer, which realloc() may move */ |
92 | | regex_t *regex[5]; |
93 | | int nregex; /* how many of regex[] have been compiled */ |
94 | | } dcf_info; |
95 | | |
96 | | static void dcf_cleanup(void *data) |
97 | 827 | { |
98 | 827 | dcf_info *pdi = (dcf_info *) data; |
99 | | |
100 | 4.96k | for(int i = 0; i < pdi->nregex; i++) |
101 | 4.13k | tre_regfree(pdi->regex[i]); |
102 | 827 | pdi->nregex = 0; |
103 | 827 | free(*pdi->buf); |
104 | 827 | *pdi->buf = NULL; |
105 | 827 | if(!pdi->wasopen && pdi->con->isopen) |
106 | 0 | pdi->con->close(pdi->con); |
107 | 827 | } |
108 | | |
109 | | static bool field_is_foldable_p(const char *, SEXP); |
110 | | |
111 | | /* Use R_alloc as this might get interrupted */ |
112 | | static char *Rconn_getline2(Rconnection con, char *buf, int bufsize) |
113 | 280k | { |
114 | 280k | int c, nbuf = 0; |
115 | 3.14M | while((c = Rconn_fgetc(con)) != R_EOF) { |
116 | 3.14M | if(nbuf+1 >= bufsize) { // allow for terminator below |
117 | 53 | bufsize *= 2; |
118 | 53 | char *buf2 = R_alloc(bufsize, sizeof(char)); |
119 | 53 | memcpy(buf2, buf, nbuf); |
120 | 53 | buf = buf2; |
121 | 53 | } |
122 | 3.14M | if(c != '\n'){ |
123 | 2.86M | buf[nbuf++] = (char) c; |
124 | 2.86M | } else { |
125 | 280k | buf[nbuf++] = '\0'; |
126 | 280k | break; |
127 | 280k | } |
128 | 3.14M | } |
129 | 280k | if (!nbuf) |
130 | 431 | return NULL; |
131 | | /* Make sure it is null-terminated even if file did not end with |
132 | | * newline. |
133 | | */ |
134 | 280k | if(buf[nbuf-1]) buf[nbuf] = '\0'; |
135 | 280k | return buf; |
136 | 280k | } |
137 | | |
138 | | attribute_hidden SEXP do_readDCF(SEXP call, SEXP op, SEXP args, SEXP env) |
139 | 827 | { |
140 | 827 | int nwhat, nwhatcap, nret, nc, nr, m, k, lastm, need, i, n_eblanklines = 0; |
141 | 827 | bool blank_skip, field_skip = false; |
142 | 827 | int whatlen, dynwhat, buflen = 8096; // was 100, but that re-alloced often |
143 | 827 | char *line, *buf = NULL; |
144 | 827 | regex_t blankline, contline, trailblank, regline, eblankline; |
145 | 827 | regmatch_t regmatch[1]; |
146 | 827 | SEXP file, what, what2, retval, retval2, dims, dimnames; |
147 | 827 | Rconnection con = NULL; |
148 | 827 | bool wasopen, is_eblankline; |
149 | 827 | RCNTXT cntxt; |
150 | 827 | dcf_info di = { |
151 | 827 | .buf = &buf, |
152 | 827 | .regex = { &blankline, &trailblank, &contline, ®line, &eblankline }, |
153 | 827 | .nregex = 0 |
154 | 827 | }; |
155 | | |
156 | 827 | SEXP fold_excludes; |
157 | 827 | bool field_fold = true, has_fold_excludes; |
158 | 827 | const char *field_name; |
159 | 827 | int offset = 0; /* -Wall */ |
160 | | |
161 | 827 | checkArity(op, args); |
162 | | |
163 | 827 | file = CAR(args); |
164 | 827 | con = getConnection(asInteger(file)); |
165 | 827 | wasopen = con->isopen; |
166 | 827 | if(!wasopen && !con->open(con)) |
167 | 0 | error(_("cannot open the connection")); |
168 | | /* Set up a context which will close the connection if we opened it, |
169 | | and free the line buffer and regexps, on error */ |
170 | 827 | di.con = con; |
171 | 827 | di.wasopen = wasopen; |
172 | 827 | begincontext(&cntxt, CTXT_CCODE, R_NilValue, R_BaseEnv, R_BaseEnv, |
173 | 827 | R_NilValue, R_NilValue); |
174 | 827 | cntxt.cend = &dcf_cleanup; |
175 | 827 | cntxt.cenddata = &di; |
176 | 827 | if(!con->canread) error(_("cannot read from this connection")); |
177 | | |
178 | 827 | args = CDR(args); |
179 | 827 | PROTECT(what = coerceVector(CAR(args), STRSXP)); /* argument fields */ |
180 | 827 | nwhat = LENGTH(what); |
181 | 827 | dynwhat = (nwhat == 0); |
182 | | |
183 | 827 | args = CDR(args); |
184 | 827 | PROTECT(fold_excludes = coerceVector(CAR(args), STRSXP)); |
185 | 827 | has_fold_excludes = (LENGTH(fold_excludes) > 0); |
186 | | |
187 | 827 | buf = (char *) malloc(buflen); |
188 | 827 | if(!buf) error(_("could not allocate memory for 'read.dcf'")); |
189 | 827 | nret = 20; |
190 | | /* it is easier if we first have a record per column. The rows are |
191 | | the fields: an exact count when 'fields' was given, otherwise a |
192 | | capacity that doubles as fields are discovered, with nwhat of its |
193 | | rows (and entries of 'what') in use. */ |
194 | 827 | nwhatcap = nwhat; |
195 | 827 | PROTECT(retval = allocMatrixNA(STRSXP, nwhatcap, nret)); |
196 | | |
197 | | /* These used to use [:blank:] and [:space:] but those are locale-dependent |
198 | | and :blank: can match \xa0 as part of a UTF-8 character |
199 | | (and is nbspace on Windows). */ |
200 | 827 | tre_regcompb(&blankline, "^[ \t]*$", REG_NOSUB & REG_EXTENDED); |
201 | 827 | tre_regcompb(&trailblank, "[ \t]+$", REG_EXTENDED); |
202 | 827 | tre_regcompb(&contline, "^[ \t]+", REG_EXTENDED); |
203 | 827 | tre_regcompb(®line, "^[^:]+:[ \t]*", REG_EXTENDED); |
204 | 827 | tre_regcompb(&eblankline, "^[ \f\n\r\t\v]+\\.[ \f\n\r\t\v]*$", REG_EXTENDED); |
205 | 827 | di.nregex = 5; |
206 | | |
207 | 827 | k = 0; |
208 | 827 | lastm = -1; /* index of the field currently being recorded */ |
209 | 827 | blank_skip = true; |
210 | 827 | void *vmax = vmaxget(); |
211 | 827 | char buf0[MAXELTSIZE]; |
212 | 281k | while((line = Rconn_getline2(con, buf0, MAXELTSIZE))) { |
213 | | /* Repair invalid UTF-8 in the whole line before matching, as the |
214 | | all = TRUE R code does: a new field name is recorded repaired, so |
215 | | matching its later occurrences against the raw bytes would fail |
216 | | and add a duplicate column for each. The repair must use the |
217 | | rules mkCharUTF8sub() records the name with: iconv() passes |
218 | | 5- and 6-byte forms through, which left such a line unrepaired |
219 | | and the name unmatched. The repaired copy comes from R_alloc() |
220 | | and may be shorter than 21 bytes. */ |
221 | 280k | if(!utf8Valid(line)) |
222 | 11.9k | line = escapeInvalidUTF8(line); |
223 | 280k | if(strlen(line) == 0 || |
224 | 272k | tre_regexecb(&blankline, line, 0, 0, 0) == 0) { |
225 | | /* A blank line. The first one after a record ends a new |
226 | | * record, subsequent ones are skipped */ |
227 | 8.18k | if(!blank_skip) { |
228 | 6.78k | k++; |
229 | 6.78k | if(k > nret - 1){ |
230 | 67 | nret *= 2; |
231 | 67 | PROTECT(retval2 = allocMatrixNA(STRSXP, nwhatcap, nret)); |
232 | 67 | transferVector(retval2, retval); |
233 | 67 | retval = retval2; |
234 | 67 | UNPROTECT(2); /* retval, retval2 */ |
235 | 67 | PROTECT(retval); |
236 | 67 | } |
237 | 6.78k | blank_skip = true; |
238 | 6.78k | lastm = -1; |
239 | 6.78k | field_skip = false; |
240 | 6.78k | field_fold = true; |
241 | 6.78k | n_eblanklines = 0; |
242 | 6.78k | } |
243 | 272k | } else if(line[0] == '#') { |
244 | | /* Ignore comment lines */ |
245 | 271k | } else { |
246 | 271k | blank_skip = false; |
247 | 271k | if(tre_regexecb(&contline, line, 1, regmatch, 0) == 0) { |
248 | | /* A continuation line: wrong if at the beginning of a |
249 | | record. */ |
250 | 251k | if((lastm == -1) && !field_skip) { |
251 | 19 | if(strlen(line) > 20) |
252 | 4 | line[20] = '\0'; |
253 | 19 | error(_("Found continuation line starting '%s ...' at begin of record."), |
254 | 19 | line); |
255 | 19 | } |
256 | 251k | if(lastm >= 0) { |
257 | 251k | need = (int) strlen(CHAR(STRING_ELT(retval, |
258 | 251k | lastm + nwhatcap * k))) + 2; |
259 | 251k | if(tre_regexecb(&eblankline, line, 0, NULL, 0) == 0) { |
260 | 461 | is_eblankline = true; |
261 | 461 | if(field_fold) { |
262 | 461 | n_eblanklines++; |
263 | 461 | continue; |
264 | 461 | } |
265 | 251k | } else { |
266 | 251k | is_eblankline = false; |
267 | 251k | if(field_fold) { |
268 | 251k | offset = regmatch[0].rm_eo; |
269 | | /* Also remove trailing whitespace. */ |
270 | 251k | if((tre_regexecb(&trailblank, line, 1, |
271 | 251k | regmatch, 0) == 0)) |
272 | 390 | line[regmatch[0].rm_so] = '\0'; |
273 | 251k | } else { |
274 | 0 | offset = 0; |
275 | 0 | } |
276 | 251k | need += (int) strlen(line + offset) + n_eblanklines; |
277 | 251k | } |
278 | 251k | if(buflen < need) { |
279 | | /* on failure, dcf_cleanup() frees the old buffer */ |
280 | 222k | char *tmp = (char *) realloc(buf, need); |
281 | 222k | if(!tmp) |
282 | 0 | error(_("could not allocate memory for 'read.dcf'")); |
283 | 222k | buf = tmp; |
284 | 222k | buflen = need; |
285 | 222k | } |
286 | 251k | strcpy(buf, CHAR(STRING_ELT(retval, lastm + nwhatcap * k))); |
287 | 251k | if(strlen(buf) || !field_fold) |
288 | 250k | strcat(buf, "\n"); |
289 | 251k | if(!is_eblankline) { |
290 | 251k | if(n_eblanklines > 0) { |
291 | 529 | for(i = 0; i < n_eblanklines; i++) { |
292 | 435 | strcat(buf, "\n"); |
293 | 435 | } |
294 | 94 | n_eblanklines = 0; |
295 | 94 | } |
296 | 251k | strcat(buf, line + offset); |
297 | 251k | } |
298 | 251k | SET_STRING_ELT(retval, lastm + nwhatcap * k, mkCharUTF8sub(buf)); |
299 | 251k | } |
300 | 251k | } else { |
301 | 20.1k | if(tre_regexecb(®line, line, 1, regmatch, 0) == 0) { |
302 | 297k | for(m = 0; m < nwhat; m++){ |
303 | 293k | whatlen = (int) strlen(CHAR(STRING_ELT(what, m))); |
304 | 293k | if(strlen(line) > whatlen && |
305 | 172k | line[whatlen] == ':' && |
306 | 34.2k | strncmp(CHAR(STRING_ELT(what, m)), |
307 | 34.2k | line, whatlen) == 0) { |
308 | | /* An already known field we are recording. */ |
309 | 15.5k | lastm = m; |
310 | 15.5k | field_skip = false; |
311 | 15.5k | field_name = CHAR(STRING_ELT(what, lastm)); |
312 | 15.5k | if(has_fold_excludes) { |
313 | 0 | field_fold = |
314 | 0 | field_is_foldable_p(field_name, |
315 | 0 | fold_excludes); |
316 | 0 | } |
317 | 15.5k | offset = regmatch[0].rm_eo; |
318 | 15.5k | if(field_fold) { |
319 | | /* Also remove trailing whitespace. */ |
320 | 15.5k | if((tre_regexecb(&trailblank, line, 1, |
321 | 15.5k | regmatch, 0) == 0)) |
322 | 404 | line[regmatch[0].rm_so] = '\0'; |
323 | 15.5k | } |
324 | 15.5k | SET_STRING_ELT(retval, m + nwhatcap * k, |
325 | 15.5k | mkCharUTF8sub(line + offset)); |
326 | 15.5k | break; |
327 | 277k | } else { |
328 | | /* This is a field, but not one prespecified */ |
329 | 277k | lastm = -1; |
330 | 277k | field_skip = true; |
331 | 277k | } |
332 | 293k | } |
333 | 19.7k | if(dynwhat && (lastm == -1)) { |
334 | | /* A previously unseen field and we are |
335 | | * recording all fields */ |
336 | 4.21k | field_skip = false; |
337 | 4.21k | if(nwhat == nwhatcap) { |
338 | | /* Double the row capacity. Growing by one |
339 | | row per field copied the whole matrix for |
340 | | every new field, quadratic in their number: |
341 | | 7000 fields in one record took 7 s. */ |
342 | 605 | int oldcap = nwhatcap; |
343 | 605 | nwhatcap = oldcap ? 2 * oldcap : 16; |
344 | 605 | PROTECT(what2 = allocVector(STRSXP, nwhatcap)); |
345 | 605 | PROTECT(retval2 = allocMatrixNA(STRSXP, nwhatcap, nret)); |
346 | 3.46k | for(nr = 0; nr < nwhat; nr++) { |
347 | 2.86k | SET_STRING_ELT(what2, nr, STRING_ELT(what, nr)); |
348 | | /* only the columns up to this record hold data */ |
349 | 79.9k | for(nc = 0; nc <= k; nc++) |
350 | 77.0k | SET_STRING_ELT(retval2, nr + nc * nwhatcap, |
351 | 77.0k | STRING_ELT(retval, nr + nc * oldcap)); |
352 | 2.86k | } |
353 | 605 | retval = retval2; |
354 | 605 | what = what2; |
355 | 605 | UNPROTECT(5); /* what, fold_excludes, retval, what2, retval2 */ |
356 | 605 | PROTECT(what); |
357 | 605 | PROTECT(fold_excludes); |
358 | 605 | PROTECT(retval); |
359 | 605 | } |
360 | | /* Make sure enough space was used */ |
361 | 4.21k | need = (int) (Rf_strchr(line, ':') - line + 1); |
362 | 4.21k | if(buflen < need){ |
363 | 16 | char *tmp = (char *) realloc(buf, need); |
364 | 16 | if(!tmp) |
365 | 0 | error(_("could not allocate memory for 'read.dcf'")); |
366 | 16 | buf = tmp; |
367 | 16 | buflen = need; |
368 | 16 | } |
369 | 4.21k | strncpy(buf, line, Rf_strchr(line, ':') - line); |
370 | 4.21k | buf[Rf_strchr(line, ':') - line] = '\0'; |
371 | 4.21k | SET_STRING_ELT(what, nwhat, mkCharUTF8sub(buf)); |
372 | 4.21k | nwhat++; |
373 | | /* lastm uses C indexing, hence nwhat - 1 */ |
374 | 4.21k | lastm = nwhat - 1; |
375 | 4.21k | field_name = CHAR(STRING_ELT(what, lastm)); |
376 | 4.21k | if(has_fold_excludes) { |
377 | 0 | field_fold = |
378 | 0 | field_is_foldable_p(field_name, |
379 | 0 | fold_excludes); |
380 | 0 | } |
381 | 4.21k | offset = regmatch[0].rm_eo; |
382 | 4.21k | if(field_fold) { |
383 | | /* Also remove trailing whitespace. */ |
384 | 4.21k | if((tre_regexecb(&trailblank, line, 1, |
385 | 4.21k | regmatch, 0) == 0)) |
386 | 298 | line[regmatch[0].rm_so] = '\0'; |
387 | 4.21k | } |
388 | 4.21k | SET_STRING_ELT(retval, lastm + nwhatcap * k, |
389 | 4.21k | mkCharUTF8sub(line + offset)); |
390 | 4.21k | } |
391 | 19.7k | } else { |
392 | | /* Must be a regular line with no tag ... */ |
393 | 377 | if(strlen(line) > 20) |
394 | 78 | line[20] = '\0'; |
395 | 377 | error(_("Line starting '%s ...' is malformed!"), line); |
396 | 377 | } |
397 | 20.1k | } |
398 | 271k | } |
399 | 280k | } |
400 | 827 | vmaxset(vmax); |
401 | 827 | endcontext(&cntxt); |
402 | 827 | dcf_cleanup(&di); |
403 | | |
404 | 827 | if(!blank_skip) k++; |
405 | | |
406 | | /* and now transpose the whole matrix, dropping the spare rows */ |
407 | 827 | PROTECT(retval2 = allocMatrixNA(STRSXP, k, nwhat)); |
408 | 4.43k | for(nr = 0; nr < nwhat; nr++) |
409 | 58.2k | for(nc = 0; nc < k; nc++) |
410 | 54.6k | SET_STRING_ELT(retval2, nc + nr * k, |
411 | 54.6k | STRING_ELT(retval, nr + nc * nwhatcap)); |
412 | 827 | if(LENGTH(what) > nwhat) { |
413 | 391 | what = lengthgets(what, nwhat); |
414 | 391 | UNPROTECT(4); /* what, fold_excludes, retval, retval2 */ |
415 | 391 | PROTECT(what); |
416 | 391 | PROTECT(fold_excludes); |
417 | 391 | PROTECT(retval); |
418 | 391 | PROTECT(retval2); |
419 | 391 | } |
420 | | |
421 | 827 | PROTECT(dimnames = allocVector(VECSXP, 2)); |
422 | 827 | PROTECT(dims = allocVector(INTSXP, 2)); |
423 | 827 | INTEGER(dims)[0] = k; |
424 | 827 | INTEGER(dims)[1] = nwhat; |
425 | 827 | SET_VECTOR_ELT(dimnames, 1, what); |
426 | 827 | setAttrib(retval2, R_DimSymbol, dims); |
427 | 827 | setAttrib(retval2, R_DimNamesSymbol, dimnames); |
428 | 827 | UNPROTECT(6); /* what, fold_excludes, retval, retval2, dimnames, dims */ |
429 | 827 | return(retval2); |
430 | 827 | } |
431 | | |
432 | | |
433 | | static SEXP allocMatrixNA(SEXPTYPE mode, int nrow, int ncol) |
434 | 1.93k | { |
435 | 1.93k | int k; |
436 | 1.93k | SEXP retval; |
437 | | |
438 | 1.93k | PROTECT(retval = allocMatrix(mode, nrow, ncol)); |
439 | 1.05M | for(k = 0; k < LENGTH(retval); k++) |
440 | 1.05M | SET_STRING_ELT(retval, k, NA_STRING); |
441 | 1.93k | UNPROTECT(1); |
442 | 1.93k | return(retval); |
443 | 1.93k | } |
444 | | |
445 | | /* This one is needed because the normal copy operations will do |
446 | | recycling */ |
447 | | |
448 | | static void transferVector(SEXP s, SEXP t) |
449 | 67 | { |
450 | 271k | for (int i = 0; i < LENGTH(t); i++) |
451 | 271k | SET_STRING_ELT(s, i, STRING_ELT(t, i)); |
452 | 67 | } |
453 | | |
454 | | static bool field_is_foldable_p(const char *field, SEXP excludes) |
455 | 0 | { |
456 | 0 | int i, n = LENGTH(excludes); |
457 | 0 | for(i = 0; i < n; i++) { |
458 | 0 | if(strcmp(field, CHAR(STRING_ELT(excludes, i))) == 0) |
459 | 0 | return false; |
460 | 0 | } |
461 | 0 | return true; |
462 | 0 | } |