Coverage Report

Created: 2026-11-13 07:26

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/r-source/src/main/dcf.c
Line
Count
Source
1
/*
2
 *  R : A Computer Language for Statistical Data Analysis
3
 *  Copyright (C) 2001-2026   The R Core Team.
4
 *
5
 *  This program is free software; you can redistribute it and/or modify
6
 *  it under the terms of the GNU General Public License as published by
7
 *  the Free Software Foundation; either version 2 of the License, or
8
 *  (at your option) any later version.
9
 *
10
 *  This program is distributed in the hope that it will be useful,
11
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
12
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13
 *  GNU General Public License for more details.
14
 *
15
 *  You should have received a copy of the GNU General Public License
16
 *  along with this program; if not, a copy is available at
17
 *  https://www.R-project.org/Licenses/
18
 */
19
20
#ifdef HAVE_CONFIG_H
21
# include <config.h>
22
#endif
23
24
#define R_USE_SIGNALS 1
25
#include <Defn.h>
26
#include <Internal.h>
27
#include <Rconnections.h>
28
29
#include <tre/tre.h>
30
31
static SEXP allocMatrixNA(SEXPTYPE, int, int);
32
static void transferVector(SEXP s, SEXP t);
33
34
/* Build a CHARSXP marked as UTF-8 from the NUL-terminated string 's'.
35
   DCF files are required to be UTF-8, so 's' is interpreted as UTF-8
36
   regardless of its actual encoding; any invalid byte sequences are
37
   repaired by escaping them as "<xx>", as
38
   iconv(from = "UTF-8", to = "UTF-8", sub = "byte") does (which the R
39
   code paths in read.dcf()/write.dcf() also use).  The common case of
40
   already-valid input is handled without conversion.
41
42
   The escaping is done with utf8Valid()'s rules rather than by iconv():
43
   glibc and libiconv pass 4-byte sequences above U+10FFFF and the
44
   withdrawn 5- and 6-byte forms through, which utf8Valid() rejects, so
45
   an iconv() round trip could leave the result invalid and
46
   do_readDCF() would then re-encode the whole field value for every
47
   further continuation line.  The same rules repair each input line
48
   before field names are matched, so a repaired name matches its
49
   later occurrences. */
50
static char *escapeInvalidUTF8(const char *s)
51
11.9k
{
52
11.9k
    char *out = R_alloc(4 * strlen(s) + 1, sizeof(char)); /* all escaped */
53
11.9k
    char *q = out;
54
1.67M
    for (const char *p = s; *p; ) {
55
1.66M
  if ((unsigned char) *p < 0x80) {
56
391k
      *q++ = *p++;
57
391k
      continue;
58
391k
  }
59
1.27M
  int len = utf8ValidClen(p);
60
1.27M
  if (len > 0) {
61
21.0k
      memcpy(q, p, len);
62
21.0k
      q += len;
63
21.0k
      p += len;
64
1.25M
  } else {
65
1.25M
      snprintf(q, 5, "<%02x>", (unsigned char) *p);
66
1.25M
      q += 4;
67
1.25M
      p++;
68
1.25M
  }
69
1.27M
    }
70
11.9k
    *q = '\0';
71
11.9k
    return out;
72
11.9k
}
73
74
static SEXP mkCharUTF8sub(const char *s)
75
275k
{
76
275k
    if (utf8Valid(s))
77
275k
  return mkCharCE(s, CE_UTF8);
78
79
0
    const void *vmax = vmaxget();
80
0
    SEXP ans = mkCharCE(escapeInvalidUTF8(s), CE_UTF8);
81
0
    vmaxset(vmax);
82
0
    return ans;
83
275k
}
84
85
/* What do_readDCF() has to release however it exits: the error() calls
86
   in its read loop would otherwise leak the line buffer and the compiled
87
   regexps, and leave open a connection it opened itself. */
88
typedef struct dcf_info {
89
    Rconnection con;
90
    bool wasopen;
91
    char **buf;     /* the line buffer, which realloc() may move */
92
    regex_t *regex[5];
93
    int nregex;     /* how many of regex[] have been compiled */
94
} dcf_info;
95
96
static void dcf_cleanup(void *data)
97
827
{
98
827
    dcf_info *pdi = (dcf_info *) data;
99
100
4.96k
    for(int i = 0; i < pdi->nregex; i++)
101
4.13k
  tre_regfree(pdi->regex[i]);
102
827
    pdi->nregex = 0;
103
827
    free(*pdi->buf);
104
827
    *pdi->buf = NULL;
105
827
    if(!pdi->wasopen && pdi->con->isopen)
106
0
  pdi->con->close(pdi->con);
107
827
}
108
109
static bool field_is_foldable_p(const char *, SEXP);
110
111
/* Use R_alloc as this might get interrupted */
112
static char *Rconn_getline2(Rconnection con, char *buf, int bufsize)
113
280k
{
114
280k
    int c, nbuf = 0;
115
3.14M
    while((c = Rconn_fgetc(con)) != R_EOF) {
116
3.14M
  if(nbuf+1 >= bufsize) { // allow for terminator below
117
53
      bufsize *= 2;
118
53
      char *buf2 = R_alloc(bufsize, sizeof(char));
119
53
      memcpy(buf2, buf, nbuf);
120
53
      buf = buf2;
121
53
  }
122
3.14M
  if(c != '\n'){
123
2.86M
      buf[nbuf++] = (char) c;
124
2.86M
  } else {
125
280k
      buf[nbuf++] = '\0';
126
280k
      break;
127
280k
  }
128
3.14M
    }
129
280k
    if (!nbuf)
130
431
      return NULL;
131
    /* Make sure it is null-terminated even if file did not end with
132
     *  newline.
133
     */
134
280k
    if(buf[nbuf-1]) buf[nbuf] = '\0';
135
280k
    return buf;
136
280k
}
137
138
attribute_hidden SEXP do_readDCF(SEXP call, SEXP op, SEXP args, SEXP env)
139
827
{
140
827
    int nwhat, nwhatcap, nret, nc, nr, m, k, lastm, need, i, n_eblanklines = 0;
141
827
    bool blank_skip, field_skip = false;
142
827
    int whatlen, dynwhat, buflen = 8096; // was 100, but that re-alloced often
143
827
    char *line, *buf = NULL;
144
827
    regex_t blankline, contline, trailblank, regline, eblankline;
145
827
    regmatch_t regmatch[1];
146
827
    SEXP file, what, what2, retval, retval2, dims, dimnames;
147
827
    Rconnection con = NULL;
148
827
    bool wasopen, is_eblankline;
149
827
    RCNTXT cntxt;
150
827
    dcf_info di = {
151
827
  .buf = &buf,
152
827
  .regex = { &blankline, &trailblank, &contline, &regline, &eblankline },
153
827
  .nregex = 0
154
827
    };
155
156
827
    SEXP fold_excludes;
157
827
    bool field_fold = true, has_fold_excludes;
158
827
    const char *field_name;
159
827
    int offset = 0; /* -Wall */
160
161
827
    checkArity(op, args);
162
163
827
    file = CAR(args);
164
827
    con = getConnection(asInteger(file));
165
827
    wasopen = con->isopen;
166
827
    if(!wasopen && !con->open(con))
167
0
  error(_("cannot open the connection"));
168
    /* Set up a context which will close the connection if we opened it,
169
       and free the line buffer and regexps, on error */
170
827
    di.con = con;
171
827
    di.wasopen = wasopen;
172
827
    begincontext(&cntxt, CTXT_CCODE, R_NilValue, R_BaseEnv, R_BaseEnv,
173
827
     R_NilValue, R_NilValue);
174
827
    cntxt.cend = &dcf_cleanup;
175
827
    cntxt.cenddata = &di;
176
827
    if(!con->canread) error(_("cannot read from this connection"));
177
178
827
    args = CDR(args);
179
827
    PROTECT(what = coerceVector(CAR(args), STRSXP)); /* argument fields */
180
827
    nwhat = LENGTH(what);
181
827
    dynwhat = (nwhat == 0);
182
183
827
    args = CDR(args);
184
827
    PROTECT(fold_excludes = coerceVector(CAR(args), STRSXP));
185
827
    has_fold_excludes = (LENGTH(fold_excludes) > 0);
186
187
827
    buf = (char *) malloc(buflen);
188
827
    if(!buf) error(_("could not allocate memory for 'read.dcf'"));
189
827
    nret = 20;
190
    /* it is easier if we first have a record per column.  The rows are
191
       the fields: an exact count when 'fields' was given, otherwise a
192
       capacity that doubles as fields are discovered, with nwhat of its
193
       rows (and entries of 'what') in use. */
194
827
    nwhatcap = nwhat;
195
827
    PROTECT(retval = allocMatrixNA(STRSXP, nwhatcap, nret));
196
197
    /* These used to use [:blank:] and [:space:] but those are locale-dependent
198
       and :blank: can match \xa0 as part of a UTF-8 character
199
       (and is nbspace on Windows). */
200
827
    tre_regcompb(&blankline, "^[ \t]*$", REG_NOSUB & REG_EXTENDED);
201
827
    tre_regcompb(&trailblank, "[ \t]+$", REG_EXTENDED);
202
827
    tre_regcompb(&contline, "^[ \t]+", REG_EXTENDED);
203
827
    tre_regcompb(&regline, "^[^:]+:[ \t]*", REG_EXTENDED);
204
827
    tre_regcompb(&eblankline, "^[ \f\n\r\t\v]+\\.[ \f\n\r\t\v]*$", REG_EXTENDED);
205
827
    di.nregex = 5;
206
207
827
    k = 0;
208
827
    lastm = -1; /* index of the field currently being recorded */
209
827
    blank_skip = true;
210
827
    void *vmax = vmaxget();
211
827
    char buf0[MAXELTSIZE];
212
281k
    while((line = Rconn_getline2(con, buf0, MAXELTSIZE))) {
213
  /* Repair invalid UTF-8 in the whole line before matching, as the
214
     all = TRUE R code does: a new field name is recorded repaired, so
215
     matching its later occurrences against the raw bytes would fail
216
     and add a duplicate column for each.  The repair must use the
217
     rules mkCharUTF8sub() records the name with: iconv() passes
218
     5- and 6-byte forms through, which left such a line unrepaired
219
     and the name unmatched.  The repaired copy comes from R_alloc()
220
     and may be shorter than 21 bytes. */
221
280k
  if(!utf8Valid(line))
222
11.9k
      line = escapeInvalidUTF8(line);
223
280k
  if(strlen(line) == 0 ||
224
272k
     tre_regexecb(&blankline, line, 0, 0, 0) == 0) {
225
      /* A blank line.  The first one after a record ends a new
226
       * record, subsequent ones are skipped */
227
8.18k
      if(!blank_skip) {
228
6.78k
    k++;
229
6.78k
    if(k > nret - 1){
230
67
        nret *= 2;
231
67
        PROTECT(retval2 = allocMatrixNA(STRSXP, nwhatcap, nret));
232
67
        transferVector(retval2, retval);
233
67
        retval = retval2;
234
67
        UNPROTECT(2); /* retval, retval2 */
235
67
        PROTECT(retval);
236
67
    }
237
6.78k
    blank_skip = true;
238
6.78k
    lastm = -1;
239
6.78k
    field_skip = false;
240
6.78k
    field_fold = true;
241
6.78k
    n_eblanklines = 0;
242
6.78k
      }
243
272k
  } else if(line[0] == '#') {
244
      /* Ignore comment lines */
245
271k
  } else {
246
271k
      blank_skip = false;
247
271k
      if(tre_regexecb(&contline, line, 1, regmatch, 0) == 0) {
248
    /* A continuation line: wrong if at the beginning of a
249
       record. */
250
251k
    if((lastm == -1) && !field_skip) {
251
19
        if(strlen(line) > 20)
252
4
      line[20] = '\0';
253
19
        error(_("Found continuation line starting '%s ...' at begin of record."),
254
19
        line);
255
19
    }
256
251k
    if(lastm >= 0) {
257
251k
        need = (int) strlen(CHAR(STRING_ELT(retval,
258
251k
              lastm + nwhatcap * k))) + 2;
259
251k
        if(tre_regexecb(&eblankline, line, 0, NULL, 0) == 0) {
260
461
      is_eblankline = true;
261
461
      if(field_fold) {
262
461
          n_eblanklines++;
263
461
          continue;
264
461
      }
265
251k
        } else {
266
251k
      is_eblankline = false;
267
251k
      if(field_fold) {
268
251k
          offset = regmatch[0].rm_eo;
269
          /* Also remove trailing whitespace. */
270
251k
          if((tre_regexecb(&trailblank, line, 1,
271
251k
               regmatch, 0) == 0))
272
390
        line[regmatch[0].rm_so] = '\0';
273
251k
      } else {
274
0
          offset = 0;
275
0
      }
276
251k
      need += (int) strlen(line + offset) + n_eblanklines;
277
251k
        }
278
251k
        if(buflen < need) {
279
      /* on failure, dcf_cleanup() frees the old buffer */
280
222k
      char *tmp = (char *) realloc(buf, need);
281
222k
      if(!tmp)
282
0
          error(_("could not allocate memory for 'read.dcf'"));
283
222k
      buf = tmp;
284
222k
      buflen = need;
285
222k
        }
286
251k
        strcpy(buf, CHAR(STRING_ELT(retval, lastm + nwhatcap * k)));
287
251k
        if(strlen(buf) || !field_fold)
288
250k
      strcat(buf, "\n");
289
251k
        if(!is_eblankline) {
290
251k
      if(n_eblanklines > 0) {
291
529
          for(i = 0; i < n_eblanklines; i++) {
292
435
        strcat(buf, "\n");
293
435
          }
294
94
          n_eblanklines = 0;
295
94
      }
296
251k
      strcat(buf, line + offset);
297
251k
        }
298
251k
        SET_STRING_ELT(retval, lastm + nwhatcap * k, mkCharUTF8sub(buf));
299
251k
    }
300
251k
      } else {
301
20.1k
    if(tre_regexecb(&regline, line, 1, regmatch, 0) == 0) {
302
297k
        for(m = 0; m < nwhat; m++){
303
293k
      whatlen = (int) strlen(CHAR(STRING_ELT(what, m)));
304
293k
      if(strlen(line) > whatlen &&
305
172k
         line[whatlen] == ':' &&
306
34.2k
         strncmp(CHAR(STRING_ELT(what, m)),
307
34.2k
           line, whatlen) == 0) {
308
          /* An already known field we are recording. */
309
15.5k
          lastm = m;
310
15.5k
          field_skip = false;
311
15.5k
          field_name = CHAR(STRING_ELT(what, lastm));
312
15.5k
          if(has_fold_excludes) {
313
0
        field_fold =
314
0
            field_is_foldable_p(field_name,
315
0
              fold_excludes);
316
0
          }
317
15.5k
          offset = regmatch[0].rm_eo;
318
15.5k
          if(field_fold) {
319
        /* Also remove trailing whitespace. */
320
15.5k
        if((tre_regexecb(&trailblank, line, 1,
321
15.5k
             regmatch, 0) == 0))
322
404
            line[regmatch[0].rm_so] = '\0';
323
15.5k
          }
324
15.5k
          SET_STRING_ELT(retval, m + nwhatcap * k,
325
15.5k
             mkCharUTF8sub(line + offset));
326
15.5k
          break;
327
277k
      } else {
328
          /* This is a field, but not one prespecified */
329
277k
          lastm = -1;
330
277k
          field_skip = true;
331
277k
      }
332
293k
        }
333
19.7k
        if(dynwhat && (lastm == -1)) {
334
      /* A previously unseen field and we are
335
       * recording all fields */
336
4.21k
      field_skip = false;
337
4.21k
      if(nwhat == nwhatcap) {
338
          /* Double the row capacity.  Growing by one
339
             row per field copied the whole matrix for
340
             every new field, quadratic in their number:
341
             7000 fields in one record took 7 s. */
342
605
          int oldcap = nwhatcap;
343
605
          nwhatcap = oldcap ? 2 * oldcap : 16;
344
605
          PROTECT(what2 = allocVector(STRSXP, nwhatcap));
345
605
          PROTECT(retval2 = allocMatrixNA(STRSXP, nwhatcap, nret));
346
3.46k
          for(nr = 0; nr < nwhat; nr++) {
347
2.86k
        SET_STRING_ELT(what2, nr, STRING_ELT(what, nr));
348
        /* only the columns up to this record hold data */
349
79.9k
        for(nc = 0; nc <= k; nc++)
350
77.0k
            SET_STRING_ELT(retval2, nr + nc * nwhatcap,
351
77.0k
               STRING_ELT(retval, nr + nc * oldcap));
352
2.86k
          }
353
605
          retval = retval2;
354
605
          what = what2;
355
605
          UNPROTECT(5); /* what, fold_excludes, retval, what2, retval2 */
356
605
          PROTECT(what);
357
605
          PROTECT(fold_excludes);
358
605
          PROTECT(retval);
359
605
      }
360
      /* Make sure enough space was used */
361
4.21k
      need = (int) (Rf_strchr(line, ':') - line + 1);
362
4.21k
      if(buflen < need){
363
16
          char *tmp = (char *) realloc(buf, need);
364
16
          if(!tmp)
365
0
        error(_("could not allocate memory for 'read.dcf'"));
366
16
          buf = tmp;
367
16
          buflen = need;
368
16
      }
369
4.21k
      strncpy(buf, line, Rf_strchr(line, ':') - line);
370
4.21k
      buf[Rf_strchr(line, ':') - line] = '\0';
371
4.21k
      SET_STRING_ELT(what, nwhat, mkCharUTF8sub(buf));
372
4.21k
      nwhat++;
373
      /* lastm uses C indexing, hence nwhat - 1 */
374
4.21k
      lastm = nwhat - 1;
375
4.21k
      field_name = CHAR(STRING_ELT(what, lastm));
376
4.21k
      if(has_fold_excludes) {
377
0
          field_fold =
378
0
        field_is_foldable_p(field_name,
379
0
                fold_excludes);
380
0
      }
381
4.21k
      offset = regmatch[0].rm_eo;
382
4.21k
      if(field_fold) {
383
          /* Also remove trailing whitespace. */
384
4.21k
          if((tre_regexecb(&trailblank, line, 1,
385
4.21k
               regmatch, 0) == 0))
386
298
        line[regmatch[0].rm_so] = '\0';
387
4.21k
      }
388
4.21k
      SET_STRING_ELT(retval, lastm + nwhatcap * k,
389
4.21k
               mkCharUTF8sub(line + offset));
390
4.21k
        }
391
19.7k
    } else {
392
        /* Must be a regular line with no tag ... */
393
377
        if(strlen(line) > 20)
394
78
      line[20] = '\0';
395
377
        error(_("Line starting '%s ...' is malformed!"), line);
396
377
    }
397
20.1k
      }
398
271k
  }
399
280k
    }
400
827
    vmaxset(vmax);
401
827
    endcontext(&cntxt);
402
827
    dcf_cleanup(&di);
403
404
827
    if(!blank_skip) k++;
405
406
    /* and now transpose the whole matrix, dropping the spare rows */
407
827
    PROTECT(retval2 = allocMatrixNA(STRSXP, k, nwhat));
408
4.43k
    for(nr = 0; nr < nwhat; nr++)
409
58.2k
  for(nc = 0; nc < k; nc++)
410
54.6k
      SET_STRING_ELT(retval2, nc + nr * k,
411
54.6k
         STRING_ELT(retval, nr + nc * nwhatcap));
412
827
    if(LENGTH(what) > nwhat) {
413
391
  what = lengthgets(what, nwhat);
414
391
  UNPROTECT(4); /* what, fold_excludes, retval, retval2 */
415
391
  PROTECT(what);
416
391
  PROTECT(fold_excludes);
417
391
  PROTECT(retval);
418
391
  PROTECT(retval2);
419
391
    }
420
421
827
    PROTECT(dimnames = allocVector(VECSXP, 2));
422
827
    PROTECT(dims = allocVector(INTSXP, 2));
423
827
    INTEGER(dims)[0] = k;
424
827
    INTEGER(dims)[1] = nwhat;
425
827
    SET_VECTOR_ELT(dimnames, 1, what);
426
827
    setAttrib(retval2, R_DimSymbol, dims);
427
827
    setAttrib(retval2, R_DimNamesSymbol, dimnames);
428
827
    UNPROTECT(6); /* what, fold_excludes, retval, retval2, dimnames, dims */
429
827
    return(retval2);
430
827
}
431
432
433
static SEXP allocMatrixNA(SEXPTYPE mode, int nrow, int ncol)
434
1.93k
{
435
1.93k
    int k;
436
1.93k
    SEXP retval;
437
438
1.93k
    PROTECT(retval = allocMatrix(mode, nrow, ncol));
439
1.05M
    for(k = 0; k < LENGTH(retval); k++)
440
1.05M
  SET_STRING_ELT(retval, k, NA_STRING);
441
1.93k
    UNPROTECT(1);
442
1.93k
    return(retval);
443
1.93k
}
444
445
/* This one is needed because the normal copy operations will do
446
   recycling */
447
448
static void transferVector(SEXP s, SEXP t)
449
67
{
450
271k
    for (int i = 0; i < LENGTH(t); i++)
451
271k
  SET_STRING_ELT(s, i, STRING_ELT(t, i));
452
67
}
453
454
static bool field_is_foldable_p(const char *field, SEXP excludes)
455
0
{
456
0
    int i, n = LENGTH(excludes);
457
0
    for(i = 0; i < n; i++) {
458
0
  if(strcmp(field, CHAR(STRING_ELT(excludes, i))) == 0)
459
0
      return false;
460
0
    }
461
0
    return true;
462
0
}