Coverage Report

Created: 2026-11-13 07:26

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/r-source/src/main/character.c
Line
Count
Source
1
/*
2
 *  R : A Computer Language for Statistical Data Analysis
3
 *  Copyright (C) 1997--2026  The R Core Team
4
 *  Copyright (C) 1995, 1996  Robert Gentleman and Ross Ihaka
5
 *
6
 *  This program is free software; you can redistribute it and/or modify
7
 *  it under the terms of the GNU General Pulic License as published by
8
 *  the Free Software Foundation; either version 2 of the License, or
9
 *  (at your option) any later version.
10
 *
11
 *  This program is distributed in the hope that it will be useful,
12
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
 *  GNU General Public License for more details.
15
 *
16
 *  You should have received a copy of the GNU General Public License
17
 *  along with this program; if not, a copy is available at
18
 *  https://www.R-project.org/Licenses/
19
 */
20
21
/* The character functions in this file are
22
23
nzchar nchar substr substr<- abbreviate tolower toupper chartr strtrim
24
25
and the utility
26
27
make.names
28
29
The regex functions
30
31
strsplit grep [g]sub [g]regexpr agrep
32
33
here prior to 2.10.0 are now in grep.c and agrep.c
34
35
make.unique, duplicated, unique, match, pmatch, charmatch are in unique.c
36
iconv is in sysutils.c
37
38
Character strings in R are at most 2^31-1 bytes, so we use int not size_t.
39
40
Support for UTF-8-encoded strings in non-UTF-8 locales
41
======================================================
42
43
Comparison is done directly unless you happen to be comparing the same
44
string in different encodings.
45
46
nzchar and nchar(, "bytes") are independent of the encoding
47
nchar(, "char") nchar(, "width") handle UTF-8 and Latin-1 directly
48
substr substr<-  handle UTF-8 and Latin-1 directly
49
tolower toupper chartr  translate UTF-8 and Latin-1 to wchar (which needs
50
  Unicode wide characters), rest to current charset
51
abbreviate translates non-ASCII inputs to UTF-8 then wchar_t*.
52
strtrim translates to the native encoding
53
make.names translates to the native encoding, works in wchar_t in a MBCS.
54
55
All the string matching functions handle UTF-8 directly, otherwise
56
translate (latin1 to UTF-8, otherwise to native).
57
58
Support for "bytes" marked encoding
59
===================================
60
61
nzchar and nchar(, "bytes") are independent of the encoding.
62
63
nchar(, "char") nchar(, "width") give NA (if allowed) or error.
64
substr substr<-  work in bytes
65
66
abbreviate chartr make.names strtrim tolower toupper give error.
67
68
*/
69
70
#ifdef HAVE_CONFIG_H
71
# include <config.h>
72
#endif
73
74
/* Used to indicate that we can safely convert marked UTF-8 strings
75
   to wchar_t* -- not currently used.
76
*/
77
#if defined(Win32) || defined(__STDC_ISO_10646__) || defined(__APPLE__) || defined(__FreeBSD__) || defined(__sun)
78
# define TO_WCS_OK 1
79
#else
80
/*
81
  Maybe warn if utf8towcs is used, but no known platforms.
82
 */
83
#endif
84
85
#include <Defn.h>
86
#include <Internal.h>
87
#include <errno.h>
88
#include <R_ext/RS.h>  // for R_Calloc/R_Free
89
#include <R_ext/Itermacros.h>
90
#include <rlocale.h>   // overrides iswxxxx on some platforms.
91
92
/* We use a shared buffer here to avoid reallocing small buffers, and
93
   keep a standard-size (MAXELTSIZE = 8192) buffer allocated shared
94
   between the various functions.
95
96
   If we want to make this thread-safe, we would need to initialize an
97
   instance non-statically in each using function, but this would add
98
   to the overhead.
99
 */
100
101
#include "RBufferUtils.h"
102
static R_StringBuffer cbuff = {NULL, 0, MAXELTSIZE};
103
104
static int int_max = INT_MAX;
105
106
/* Functions to perform analogues of the standard C string library. */
107
/* Most are vectorized */
108
109
/* primitive, nzchar(x, keepNA = FALSE) where the second argument is optional.
110
   Encoding of x is immaterial.
111
*/
112
attribute_hidden SEXP do_nzchar(SEXP call, SEXP op, SEXP args, SEXP env)
113
383k
{
114
383k
    int nargs = length(args);
115
116
    // checkArity(op, args);  .Primitive() and may have 1 or 2 args
117
383k
    if (nargs < 1 || nargs > 2)
118
0
  errorcall(call,
119
0
      ngettext("%d argument passed to '%s' which requires %d to %d",
120
0
         "%d arguments passed to '%s' which requires %d to %d",
121
0
         (unsigned long) nargs),
122
0
      nargs, PRIMNAME(op), 1, 2);
123
383k
    check1arg(args, call, "x");
124
125
383k
    if (isFactor(CAR(args)))
126
0
  error(_("'%s' requires a character vector"), "nzchar()");
127
383k
    SEXP x = PROTECT(coerceVector(CAR(args), STRSXP));
128
383k
    if (!isString(x))
129
0
  error(_("'%s' requires a character vector"), "nzchar()");
130
131
383k
    int keepNA = FALSE; // the default
132
383k
    if(nargs > 1) {
133
0
  keepNA = asLogical(CADR(args));
134
0
  if (keepNA == NA_LOGICAL) keepNA = FALSE;
135
0
    }
136
383k
    R_xlen_t i, len = XLENGTH(x);
137
383k
    SEXP ans = PROTECT(allocVector(LGLSXP, len));
138
383k
    if (keepNA)
139
0
  for (i = 0; i < len; i++) {
140
0
      SEXP sxi = STRING_ELT(x, i);
141
0
      LOGICAL(ans)[i] = (sxi == NA_STRING) ? NA_LOGICAL : LENGTH(sxi) > 0;
142
0
  }
143
383k
    else
144
936k
  for (i = 0; i < len; i++)
145
552k
      LOGICAL(ans)[i] = LENGTH(STRING_ELT(x, i)) > 0;
146
383k
    UNPROTECT(2);
147
383k
    return ans;
148
383k
}
149
150
/* R strings are limited to 2^31 - 1 bytes on all platforms */
151
152
/* when msg_name is not NULL,
153
     error handling is via error() with a message including msg_name
154
     semi-internal buffer cbuff is freed if over default size
155
156
   when msg_name is NULL, (for use where performance matters)
157
     error handling is via negative return value (other than NA_INTEGER):
158
       -1 ... invalid multi-byte string
159
       -2 ... the quantity is not computable (bytes encoding)
160
     semi-internal buffer cbuff is never freed, should be freed by caller
161
*/
162
// in Defn.h
163
attribute_hidden
164
int R_nchar(SEXP string, nchar_type type_,
165
      Rboolean allowNA, Rboolean keepNA, const char* msg_name)
166
982k
{
167
982k
    if (string == NA_STRING)
168
0
  return keepNA ? NA_INTEGER : 2;
169
    // else :
170
982k
    switch(type_) {
171
809
    case Bytes:
172
809
  return LENGTH(string);
173
0
  break;
174
980k
    case Chars:
175
980k
  if (IS_UTF8(string)) {
176
21
      const char *p = CHAR(string);
177
21
      if (!utf8Valid(p)) {
178
5
    if (!allowNA) {
179
0
        if (msg_name)
180
0
      error(_("invalid multibyte string, %s"), msg_name);
181
0
        else
182
0
      return -1;
183
0
    }
184
5
    return NA_INTEGER;
185
16
      } else {
186
16
    int nc = 0;
187
400
    for( ; *p; p += utf8clen(*p)) nc++;
188
16
    return nc;
189
16
      }
190
979k
  } else if (IS_LATIN1(string)) {
191
      // just count bytes
192
6
      return (int) strlen(CHAR(string));
193
979k
  } else if (IS_BYTES(string)) {
194
0
      if (!allowNA) /* could do chars 0 */ {
195
0
    if (msg_name)
196
0
        error(_("number of characters is not computable in \"bytes\" encoding, %s"),
197
0
              msg_name);
198
0
    else
199
0
        return -2;
200
0
      }
201
0
      return NA_INTEGER;
202
979k
  } else if (mbcslocale) {
203
0
      int nc = (int) mbstowcs(NULL, translateChar(string), 0);
204
0
      if (!allowNA && nc < 0) {
205
0
    if (msg_name)
206
0
        error(_("invalid multibyte string, %s"), msg_name);
207
0
    else
208
0
        return -1;
209
0
      }
210
0
      return (nc >= 0 ? nc : NA_INTEGER);
211
0
  } else
212
979k
      return ((int) strlen(translateChar(string)));
213
0
  break;
214
1.82k
    case Width:
215
1.82k
  if (IS_UTF8(string)) {
216
86
      const char *p = CHAR(string);
217
86
      if (!utf8Valid(p)) {
218
8
    if (!allowNA) {
219
0
        if (msg_name)
220
0
      error(_("invalid multibyte string, %s"), msg_name);
221
0
        else
222
0
      return -1;
223
0
    }
224
8
    return NA_INTEGER;
225
78
      } else {
226
78
    int nc = 0;
227
38.7k
    for( ; *p; p += utf8clen(*p)) {
228
38.7k
        wchar_t wc1;
229
38.7k
        utf8toucs(&wc1, p);
230
38.7k
        R_wchar_t ucs;
231
38.7k
        if (IS_HIGH_SURROGATE(wc1))
232
0
          ucs = utf8toucs32(wc1, p);
233
38.7k
        else
234
38.7k
          ucs = wc1;
235
38.7k
#ifdef USE_RI18N_WIDTH
236
38.7k
        nc += Ri18n_wcwidth(ucs);
237
#else
238
        {
239
      int this = wcwidth(ucs);
240
      if (this >= 0) nc += this;
241
        }
242
#endif
243
38.7k
    }
244
78
    return nc;
245
78
      }
246
1.73k
  } else if (IS_BYTES(string)) {
247
0
      if (!allowNA) { /* could do width 0 */
248
0
    if (msg_name)
249
0
        error(_("width is not computable for %s in \"bytes\" encoding"),
250
0
              msg_name);
251
0
    else
252
0
        return -2;
253
0
      }
254
0
      return NA_INTEGER;
255
1.73k
  } else if (IS_LATIN1(string)) {
256
      // just count bytes as they are all width-1 chars
257
      // FIXME, well not control chars but there is ambiguity for most of 0x80-9F
258
0
      return (int) strlen(CHAR(string));
259
1.73k
  } else if (mbcslocale) {
260
0
      const char *xi = translateChar(string);
261
0
      int nc = (int) mbstowcs(NULL, xi, 0);
262
0
      if (nc >= 0) {
263
0
    const void *vmax = vmaxget();
264
    /* working in wchar_t restricts this to the BMP on
265
       Windows, but maybe that is all current native
266
       charsets cover. */
267
0
    wchar_t *wc = (wchar_t *)
268
0
        R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
269
0
    mbstowcs(wc, xi, nc + 1);
270
    // FIXME: width could conceivably exceed MAX_INT.
271
0
#ifdef USE_RI18N_WIDTH
272
0
    int nci18n = Ri18n_wcswidth(wc, 2147483647);
273
#else
274
    // We do not use this unless R_wchar_t is wchar_t
275
    // This could be -1 if there are non-printable chars,
276
    // then this is ignored
277
    int nci18n = wcswidth(wc, 2147483647);
278
#endif
279
0
    if (msg_name)
280
0
        R_FreeStringBufferL(&cbuff);
281
0
    vmaxset(vmax);
282
0
    return (nci18n < 0) ? nc : nci18n;
283
0
      } else if (!allowNA) {
284
0
    if (msg_name)
285
0
        error(_("invalid multibyte string, %s"), msg_name);
286
0
    else
287
0
        return -1;
288
0
      }
289
0
      else
290
0
    return NA_INTEGER;
291
0
  } else
292
      // See Latin-1 comment.
293
1.73k
      return (int) strlen(translateChar(string));
294
295
982k
    } // switch
296
0
    return NA_INTEGER; // -Wall
297
982k
} // R_nchar()
298
299
attribute_hidden SEXP do_nchar(SEXP call, SEXP op, SEXP args, SEXP env)
300
411k
{
301
411k
    SEXP d, s, x, stype, ans;
302
411k
    int nargs = length(args);
303
304
#ifdef R_version_3_4_or_so
305
    checkArity(op, args);
306
#else
307
    // will work also for code byte-compiled *before* 'keepNA' was introduced
308
411k
    if (nargs < 3 || nargs > 4)
309
0
  error(ngettext("%d argument passed to '%s' which requires %d to %d",
310
0
           "%d arguments passed to '%s' which requires %d to %d",
311
0
         (unsigned long) nargs),
312
0
        nargs, PRIMNAME(op), 3, 4);
313
411k
#endif
314
    /* DispatchOrEval internal generic: nchar */
315
411k
    if (DispatchOrEval(call, op, "nchar", args, env, &ans, 0, 1))
316
0
      return(ans);
317
411k
    if (isFactor(CAR(args)))
318
0
  error(_("'%s' requires a character vector"), "nchar()");
319
411k
    PROTECT(x = coerceVector(CAR(args), STRSXP));
320
411k
    if (!isString(x))
321
0
  error(_("'%s' requires a character vector"), "nchar()");
322
411k
    R_xlen_t len = XLENGTH(x);
323
411k
    stype = CADR(args);
324
411k
    if (!isString(stype) || LENGTH(stype) != 1)
325
0
  error(_("invalid '%s' argument"), "type");
326
411k
    const char *type = CHAR(STRING_ELT(stype, 0)); /* always ASCII */
327
411k
    size_t ntype = strlen(type);
328
411k
    if (ntype == 0) error(_("invalid '%s' argument"), "type");
329
411k
    nchar_type type_;
330
411k
    if (strncmp(type, "bytes", ntype) == 0)   type_ = Bytes;
331
411k
    else if (strncmp(type, "chars", ntype) == 0) type_ = Chars;
332
1.82k
    else if (strncmp(type, "width", ntype) == 0) type_ = Width;
333
0
    else error(_("invalid '%s' argument"), "type");
334
411k
    int allowNA = asLogical(CADDR(args));
335
411k
    if (allowNA == NA_LOGICAL) allowNA = 0;
336
411k
    int keepNA;
337
411k
    if(nargs >= 4) {
338
411k
  keepNA = asLogical(CADDDR(args));
339
411k
  if (keepNA == NA_LOGICAL) // default
340
411k
      keepNA = (type_ == Width) ? FALSE : TRUE;
341
411k
    } else  keepNA = (type_ == Width) ? FALSE : TRUE;
342
411k
    PROTECT(s = allocVector(INTSXP, len));
343
411k
    int *s_ = INTEGER(s);
344
1.39M
    for (R_xlen_t i = 0; i < len; i++) {
345
982k
  SEXP sxi = STRING_ELT(x, i);
346
  // NA_LOGICAL has now been excluded
347
982k
  int res = R_nchar(sxi, type_,
348
982k
        (Rboolean) allowNA, (Rboolean) keepNA, NULL);
349
982k
  switch(res) {
350
0
  case -1:
351
0
      error(_("invalid multibyte string, element %lld"), (long long)i+1);
352
0
  case -2:
353
0
      if (type_ == Chars)
354
0
    error(_("number of characters is not computable in \"bytes\" encoding, element %lld"),
355
0
          (long long)i+1);
356
0
      else /* type_ == Width */
357
0
    error(_("width is not computable in \"bytes\" encoding, element %lld"),
358
0
          (long long)i+1);
359
982k
  default:
360
982k
      s_[i] = res;
361
982k
      break;
362
982k
  }
363
982k
    }
364
411k
    R_FreeStringBufferL(&cbuff);
365
411k
    if ((d = getAttrib(x, R_NamesSymbol)) != R_NilValue)
366
0
  setAttrib(s, R_NamesSymbol, d);
367
411k
    if ((d = getAttrib(x, R_DimSymbol)) != R_NilValue)
368
0
  setAttrib(s, R_DimSymbol, d);
369
411k
    if ((d = getAttrib(x, R_DimNamesSymbol)) != R_NilValue)
370
0
  setAttrib(s, R_DimNamesSymbol, d);
371
411k
    UNPROTECT(2);
372
411k
    return s;
373
411k
}
374
375
/* Assumes sa < so; sa, so are 1-based indices in character units to str,
376
   len is length of str in bytes, excluding the terminator.
377
378
   Returns pointer to result string in rfrom, of length rlen (in bytes,
379
   excluding the terminator - the string is not terminated).
380
381
   *rfrom may be invalid pointer when rlen is zero.
382
*/
383
static void substr(const char *str, int len, int ienc, int sa, int so,
384
                   R_xlen_t idx, int isascii, const char **rfrom,
385
             int *rlen, int assumevalid)
386
248k
{
387
248k
    int i;
388
248k
    const char *end = str + len;
389
390
248k
    if (ienc == CE_UTF8) {
391
3.92k
  if (!assumevalid && !utf8Valid(str)) {
392
88
      char msg[40];
393
88
      snprintf(msg, 40, "element %lld", (long long)idx+1);
394
88
      error(_("invalid multibyte string, %s"), msg);
395
88
  }
396
3.95k
  for (i = 0; i < sa - 1 && str < end; i++)
397
23
      str += utf8clen(*str);
398
3.92k
  *rfrom = str;
399
119k
  for(; i < so && str < end; i++)
400
115k
      str += utf8clen(*str);
401
3.92k
  *rlen = (int) (str - *rfrom);
402
245k
    } else if (!isascii && ienc != CE_LATIN1 && ienc != CE_BYTES
403
125
               && mbcslocale) {
404
0
  mbstate_t mb_st;
405
0
  mbs_init(&mb_st);
406
0
  for (i = 0; i < sa - 1 && str < end; i++)
407
      /* throws error on invalid multi-byte string */
408
0
      str += Mbrtowc(NULL, str, R_MB_CUR_MAX, &mb_st);
409
0
  *rfrom = str;
410
0
  for (; i < so && str < end; i++)
411
      /* throws error on invalid multi-byte string */
412
0
      str += (int) Mbrtowc(NULL, str, R_MB_CUR_MAX, &mb_st);
413
0
  *rlen = (int) (str - *rfrom);
414
245k
    } else {
415
245k
  if (so - 1 < len) {
416
81.1k
      *rfrom = str + sa - 1;
417
81.1k
      *rlen = so - sa + 1;
418
163k
  } else if (sa - 1 < len) {
419
163k
      *rfrom = str + sa - 1;
420
163k
      *rlen = len - (sa - 1);
421
163k
  } else {
422
90
      *rfrom = NULL;
423
90
      *rlen = 0;
424
90
  }
425
245k
    }
426
248k
}
427
428
attribute_hidden SEXP
429
do_substr(SEXP call, SEXP op, SEXP args, SEXP env)
430
161k
{
431
161k
    checkArity(op, args);
432
161k
    SEXP x = CAR(args);
433
161k
    if (!isString(x))
434
0
  error(_("extracting substrings from a non-character object"));
435
161k
    R_xlen_t len = XLENGTH(x);
436
161k
    SEXP s = PROTECT(allocVector(STRSXP, len));
437
438
161k
    if (len > 0) {
439
161k
  SEXP sa = CADR(args), // start
440
161k
      so = CADDR(args); // stop
441
161k
  int
442
161k
      k = LENGTH(sa),
443
161k
      l = LENGTH(so);
444
161k
  if (!isInteger(sa) || k == 0 ||
445
161k
      (so != R_NilValue && (!isInteger(so) || l == 0)))
446
0
      error(_("invalid substring arguments"));
447
448
161k
  int *starts = INTEGER(sa), *stops = NULL;
449
161k
  if (so == R_NilValue) {
450
79.9k
      stops = &int_max;
451
79.9k
      l = 1;
452
81.8k
  } else { // as.integer(.) in R
453
81.8k
      stops = INTEGER(so);
454
81.8k
  }
455
456
161k
  SEXP lastel = NULL;
457
410k
  for (R_xlen_t i = 0; i < len; i++) {
458
459
248k
      int start = starts[i % k],
460
248k
    stop  = stops [i % l];
461
248k
      SEXP el = STRING_ELT(x,i);
462
248k
      if (el == NA_STRING || start == NA_INTEGER || stop == NA_INTEGER) {
463
3
    SET_STRING_ELT(s, i, NA_STRING);
464
3
    continue;
465
3
      }
466
248k
      cetype_t ienc = getCharCE(el);
467
248k
      const char *ss = CHAR(el);
468
248k
      int slen = LENGTH(el);
469
248k
      if (start < 1) start = 1;
470
248k
      if (start > stop) {
471
0
    SET_STRING_ELT(s, i, R_BlankString);
472
248k
      } else {
473
248k
    const char *rfrom;
474
248k
    int rlen;
475
    /* Skip checking UTF-8 validity if the string is the same
476
       R object as previously. This improves performance of
477
       substring() used on a single string but many substrings
478
       to be extracted from it */
479
248k
    substr(ss, slen, ienc, start, stop, i,
480
248k
           IS_ASCII(el), &rfrom, &rlen, el == lastel);
481
248k
    SET_STRING_ELT(s, i, mkCharLenCE(rfrom, rlen, ienc));
482
248k
      }
483
248k
      lastel = el;
484
248k
  }
485
161k
    }
486
161k
    SHALLOW_DUPLICATE_ATTRIB(s, x);
487
    /* This copied the class, if any */
488
161k
    UNPROTECT(1);
489
161k
    return s;
490
161k
}
491
492
// .Internal( startsWith(x, prefix) )  and
493
// .Internal( endsWith  (x, suffix) )
494
attribute_hidden SEXP
495
do_startsWith(SEXP call, SEXP op, SEXP args, SEXP env)
496
5.68k
{
497
5.68k
    checkArity(op, args);
498
499
5.68k
    SEXP x = CAR(args), Xfix = CADR(args); // 'prefix' or 'suffix'
500
5.68k
    if (!isString(x) || !isString(Xfix))
501
0
  error(_("non-character object(s)"));
502
5.68k
    R_xlen_t
503
5.68k
  n1 = XLENGTH(x),
504
5.68k
  n2 = XLENGTH(Xfix),
505
5.68k
  n = (n1 > 0 && n2 > 0) ? ((n1 >= n2) ? n1 : n2) : 0;
506
5.68k
    if (n == 0) return allocVector(LGLSXP, 0);
507
5.46k
    SEXP ans = PROTECT(allocVector(LGLSXP, n));
508
509
5.46k
    typedef const char * cp;
510
5.46k
    if (n2 == 1) { // optimize the most common case
511
5.46k
  SEXP el = STRING_ELT(Xfix, 0);
512
5.46k
  if (el == NA_STRING) {
513
0
      for (R_xlen_t i = 0; i < n1; i++)
514
0
    LOGICAL(ans)[i] = NA_LOGICAL;
515
5.46k
  } else {
516
      // ASCII matching will do for ASCII Xfix except in non-UTF-8 MBCS
517
5.46k
      Rboolean need_translate = TRUE;
518
5.46k
      if (IS_ASCII(el) && (utf8locale || !mbcslocale))
519
5.46k
    need_translate = FALSE;
520
5.46k
      cp y0 = need_translate ? translateCharUTF8(el) : CHAR(el);
521
5.46k
      int ylen = (int) strlen(y0);
522
4.71M
      for (R_xlen_t i = 0; i < n1; i++) {
523
4.71M
    SEXP el = STRING_ELT(x, i);
524
4.71M
    if (el == NA_STRING) {
525
0
        LOGICAL(ans)[i] = NA_LOGICAL;
526
4.71M
    } else {
527
4.71M
        cp x0 = need_translate ? translateCharUTF8(el) : CHAR(el);
528
4.71M
        if(PRIMVAL(op) == 0) { // startsWith
529
4.71M
      LOGICAL(ans)[i] = strncmp(x0, y0, ylen) == 0;
530
4.71M
        } else { // endsWith
531
0
      int off = (int)strlen(x0) - ylen;
532
0
      if (off < 0)
533
0
          LOGICAL(ans)[i] = 0;
534
0
      else {
535
0
          LOGICAL(ans)[i] = memcmp(x0 + off, y0, ylen) == 0;
536
0
      }
537
0
        }
538
4.71M
    }
539
4.71M
      }
540
5.46k
  }
541
5.46k
    } else { // n2 > 1
542
  // convert both inputs to UTF-8
543
0
  cp *x0 = (cp *) R_alloc(n1, sizeof(char *));
544
0
  cp *y0 = (cp *) R_alloc(n2, sizeof(char *));
545
  // and record lengths, -1 for NA
546
0
  int *x1 = (int *) R_alloc(n1, sizeof(int));
547
0
  int *y1 = (int *) R_alloc(n2, sizeof(int));
548
0
  for (R_xlen_t i = 0; i < n1; i++) {
549
0
      SEXP el = STRING_ELT(x, i);
550
0
      if (el == NA_STRING)
551
0
    x1[i] = -1;
552
0
      else {
553
0
    x0[i] = translateCharUTF8(el);
554
0
    x1[i] = (int) strlen(x0[i]);
555
0
      }
556
0
  }
557
0
  for (R_xlen_t i = 0; i < n2; i++) {
558
0
      SEXP el = STRING_ELT(Xfix, i);
559
0
      if (el == NA_STRING)
560
0
    y1[i] = -1;
561
0
      else {
562
0
    y0[i] = translateCharUTF8(el);
563
0
    y1[i] = (int) strlen(y0[i]);
564
0
      }
565
0
  }
566
0
  R_xlen_t i, i1, i2;
567
0
  if(PRIMVAL(op) == 0) { // 0 = startsWith, 1 = endsWith
568
0
      MOD_ITERATE2(n, n1, n2, i, i1, i2, {
569
0
        if (x1[i1] < 0 || y1[i2] < 0)
570
0
      LOGICAL(ans)[i] = NA_LOGICAL;
571
0
        else if (x1[i1] < y1[i2])
572
0
      LOGICAL(ans)[i] = 0;
573
0
        else // memcmp should be faster than strncmp
574
0
      LOGICAL(ans)[i] =
575
0
          memcmp(x0[i1], y0[i2], y1[i2]) == 0;
576
0
    });
577
0
  } else { // endsWith
578
0
      MOD_ITERATE2(n, n1, n2, i, i1, i2, {
579
0
        if (x1[i1] < 0 || y1[i2] < 0)
580
0
      LOGICAL(ans)[i] = NA_LOGICAL;
581
0
        else {
582
0
      int off = x1[i1] - y1[i2];
583
0
      if (off < 0)
584
0
          LOGICAL(ans)[i] = 0;
585
0
      else {
586
0
          LOGICAL(ans)[i] =
587
0
        memcmp(x0[i1] + off, y0[i2], y1[i2]) == 0;
588
0
      }
589
0
        }
590
0
    });
591
0
  }
592
0
    }
593
5.46k
    UNPROTECT(1);
594
5.46k
    return ans;
595
5.68k
}
596
597
598
static void
599
substrset(char *buf, const char *const str, cetype_t ienc, int sa, int so,
600
          R_xlen_t xidx, R_xlen_t vidx)
601
13.3k
{
602
    /* Replace the substring buf[sa:so] by str[] */
603
13.3k
    int i, in = 0, out = 0;
604
605
13.3k
    if (ienc == CE_UTF8) {
606
152
  if (!utf8Valid(buf)) {
607
131
      char msg[40];
608
131
      snprintf(msg, 40, "element %lld", (long long)xidx+1);
609
131
      error(_("invalid multibyte string, %s"), msg);
610
131
  }
611
152
  if (!utf8Valid(str)) {
612
1
      char msg[40];
613
1
      snprintf(msg, 40, "value element %lld", (long long)vidx+1);
614
1
      error(_("invalid multibyte string, %s"), msg);
615
1
  }
616
172
  for (i = 1; i < sa; i++) buf += utf8clen(*buf);
617
228
  for (i = sa; i <= so && buf[out] && str[in]; i++) {
618
76
      in +=  utf8clen(str[in]);
619
76
      out += utf8clen(buf[out]);
620
76
  }
621
152
  if (in != out) memmove(buf+in, buf+out, strlen(buf+out)+1);
622
152
  memcpy(buf, str, in);
623
13.2k
    } else if (ienc == CE_LATIN1 || ienc == CE_BYTES) {
624
0
  in = (int) strlen(str);
625
0
  out = so - sa + 1;
626
0
  memcpy(buf + sa - 1, str, (in < out) ? in : out);
627
13.2k
    } else {
628
  /* This cannot work for stateful encodings */
629
13.2k
  if (mbcslocale) {
630
0
      mbstate_t mb_st_in;
631
0
      mbs_init(&mb_st_in);
632
0
      for (i = 1; i < sa; i++)
633
0
    buf += Mbrtowc(NULL, buf, R_MB_CUR_MAX, &mb_st_in);
634
      /* now work out how many bytes to replace by how many */
635
0
      mbstate_t mb_st_out;
636
0
      mbs_init(&mb_st_out);
637
0
      for (i = sa; i <= so && buf[out] && str[in]; i++) {
638
0
    in  += (int) Mbrtowc(NULL, str+in,  R_MB_CUR_MAX, &mb_st_in);
639
0
    out += (int) Mbrtowc(NULL, buf+out, R_MB_CUR_MAX, &mb_st_out);
640
0
      }
641
0
      if (in != out) memmove(buf+in, buf+out, strlen(buf+out)+1);
642
0
      memcpy(buf, str, in);
643
13.2k
  } else {
644
13.2k
      in = (int) strlen(str);
645
13.2k
      out = so - sa + 1;
646
13.2k
      memcpy(buf + sa - 1, str, (in < out) ? in : out);
647
13.2k
  }
648
13.2k
    }
649
13.3k
}
650
651
attribute_hidden SEXP
652
do_substrgets(SEXP call, SEXP op, SEXP args, SEXP env)
653
520
{
654
520
    checkArity(op, args);
655
520
    SEXP x = CAR(args);
656
520
    if (!isString(x))
657
0
  error(_("replacing substrings in a non-character object"));
658
520
    R_xlen_t len = XLENGTH(x);
659
520
    SEXP s = PROTECT(allocVector(STRSXP, len));
660
661
520
    if (len > 0) {
662
520
  SEXP sa = CADR(args), // start
663
520
      so = CADDR(args); // stop
664
520
  int
665
520
      k = LENGTH(sa),
666
520
      l = LENGTH(so);
667
520
  if (!isInteger(sa) || k == 0 ||
668
520
      (so != R_NilValue && (!isInteger(so) || l == 0)))
669
0
      error(_("invalid substring arguments"));
670
671
520
  int *starts = INTEGER(sa), *stops = NULL;
672
520
  if (so == R_NilValue) {
673
0
      stops = &int_max;
674
0
      l = 1;
675
520
  } else {
676
520
      stops = INTEGER(so);
677
520
  }
678
679
520
  SEXP value = CADDDR(args);
680
520
  int v = LENGTH(value);
681
520
  if (!isString(value) || v == 0)
682
0
      error(_("invalid value"));
683
684
520
  void* vmax = vmaxget();
685
13.9k
  for (R_xlen_t i = 0; i < len; i++) {
686
687
13.3k
      int start = starts[i % k],
688
13.3k
    stop  = stops [i % l];
689
13.3k
      SEXP el = STRING_ELT(x, i);
690
13.3k
      SEXP v_el = STRING_ELT(value, i % v);
691
13.3k
      if (el == NA_STRING || v_el == NA_STRING ||
692
13.3k
    start == NA_INTEGER || stop == NA_INTEGER) {
693
0
    SET_STRING_ELT(s, i, NA_STRING);
694
0
    continue;
695
0
      }
696
697
13.3k
      cetype_t ienc = getCharCE(el);
698
13.3k
      const char* ss = CHAR(el);
699
13.3k
      int slen = (int) strlen(ss);
700
13.3k
      if (start < 1) start = 1;
701
13.3k
      if (stop > (int) slen) stop = (int) slen; /* SBCS optimization */
702
13.3k
      if (start > stop) {
703
    /* just copy element across */
704
4
    SET_STRING_ELT(s, i, STRING_ELT(x, i));
705
13.3k
      } else {
706
13.3k
    int ienc2 = ienc;
707
13.3k
    const char* v_ss = CHAR(v_el);
708
    /* is the value in the same encoding?
709
       FIXME: could re-encode to UTF-8 rather than to native.
710
    */
711
13.3k
    cetype_t venc = getCharCE(v_el);
712
13.3k
    if (venc != ienc && !IS_ASCII(v_el)) {
713
14
        ss = translateChar(el);
714
14
        slen = (int) strlen(ss);
715
14
        v_ss = translateChar(v_el);
716
14
        ienc2 = CE_NATIVE;
717
14
    }
718
    /* might expand under MBCS */
719
13.3k
    char* buf = R_AllocStringBuffer(slen+strlen(v_ss), &cbuff);
720
13.3k
    strcpy(buf, ss);
721
13.3k
    substrset(buf, v_ss, ienc2, start, stop, i, i % v);
722
13.3k
    SET_STRING_ELT(s, i, mkCharCE(buf, ienc2));
723
13.3k
      }
724
13.3k
  }
725
520
  vmaxset(vmax);
726
727
520
  R_FreeStringBufferL(&cbuff);
728
520
    }
729
520
    SHALLOW_DUPLICATE_ATTRIB(s, x);
730
    /* This copied the class, if any */
731
520
    UNPROTECT(1);
732
520
    return s;
733
520
}
734
735
/* Abbreviate
736
   long names in the S-designated fashion:
737
   1) spaces
738
   2) lower case vowels
739
   3) lower case consonants
740
   4) upper case letters
741
   5) special characters.
742
743
   Letters are dropped from the end of words
744
   and at least one letter is retained from each word.
745
746
   If unique abbreviations are not produced letters are added until the
747
   results are unique (duplicated names are removed prior to entry).
748
   names, minlength, use.classes, dot
749
*/
750
751
752
776k
#define FIRSTCHAR(i) (isspace((int)s[i-1]))
753
194k
#define LASTCHAR(i) (!isspace((int)s[i-1]) && (!s[i+1] || isspace((int)s[i+1])))
754
1.94M
#define LC_VOWEL(i) (s[i] == 'a' || s[i] == 'e' || s[i] == 'i' || \
755
971k
       s[i] == 'o' || s[i] == 'u')
756
1.30k
#define UPPER (int)(strlen(s) - 1)
757
758
/* memmove does allow overlapping src and dest */
759
static void mystrcpy(char *dest, const char *src)
760
486k
{
761
486k
    memmove(dest, src, strlen(src)+1);
762
486k
}
763
764
static SEXP stripchars(const char * const inchar, int minlen, int usecl)
765
301
{
766
301
    int i, j, nspace = 0;
767
301
    char *s = cbuff.data;
768
769
    /* The R wrapper removed leading and trailing spces */
770
301
    mystrcpy(s, inchar);
771
301
    if (strlen(s) < minlen) goto donesc;
772
773
    /* The for() loops never touch the first character */
774
775
    /*  record spaces for removal later (as they act as word boundaries) */
776
491k
    for (i = UPPER, j = 1; i > 0; i--) {
777
491k
  if (isspace((int)s[i])) {
778
64.6k
      if (j) s[i] = '\0'; // trailing space
779
64.1k
      else nspace++;
780
426k
  } else j = 0;
781
491k
  if (strlen(s) - nspace <= minlen)
782
35
      goto donesc;
783
491k
    }
784
785
266
    if(usecl) {
786
  /* remove l/case vowels,
787
     which are not at the beginning of a word but are at the end */
788
491k
  for (i = UPPER; i > 0; i--) {
789
490k
      if (LC_VOWEL(i) && LASTCHAR(i))
790
9.75k
    mystrcpy(s + i, s + i + 1);
791
490k
      if (strlen(s) - nspace <= minlen)
792
19
    goto donesc;
793
490k
  }
794
795
  /* remove those not at the beginning of a word */
796
481k
  for (i = UPPER; i > 0; i--) {
797
480k
      if (LC_VOWEL(i) && !FIRSTCHAR(i))
798
169k
    mystrcpy(s + i, s + i + 1);
799
480k
      if (strlen(s) - nspace <= minlen)
800
59
    goto donesc;
801
480k
  }
802
803
  /* Now do the same for remaining l/case chars */
804
311k
  for (i = UPPER; i > 0; i--) {
805
311k
      if (islower((int)s[i]) && LASTCHAR(i))
806
1.92k
    mystrcpy(s + i, s + i + 1);
807
311k
      if (strlen(s) - nspace <= minlen)
808
23
    goto donesc;
809
311k
  }
810
811
309k
  for (i = UPPER; i > 0; i--) {
812
309k
      if (islower((int)s[i]) && !FIRSTCHAR(i))
813
12.0k
    mystrcpy(s + i, s + i + 1);
814
309k
      if (strlen(s) - nspace <= minlen)
815
29
    goto donesc;
816
309k
  }
817
165
    }
818
819
    /* all else has failed so we use brute force */
820
821
296k
    for (i = UPPER; i > 0; i--) {
822
296k
  if (!FIRSTCHAR(i) && !isspace((int)s[i]))
823
229k
      mystrcpy(s + i, s + i + 1);
824
296k
  if (strlen(s) - nspace <= minlen)
825
86
      goto donesc;
826
296k
    }
827
828
301
donesc:
829
301
    {  // remove internal spaces as required
830
301
  int upper = (int) strlen(s);
831
301
  if (upper > minlen)
832
68.3k
      for (i = upper - 1; i > 0; i--)
833
68.2k
    if (isspace((int)s[i]))
834
64.1k
        mystrcpy(s + i, s + i + 1);
835
301
    }
836
837
301
    return mkChar(s);
838
136
}
839
840
2.99M
#define FIRSTCHARW(i) (iswspace((int)wc[i-1]))
841
490k
#define LASTCHARW(i) (!iswspace((int)wc[i-1]) && (!wc[i+1] || iswspace((int)wc[i+1])))
842
2.65k
#define WUP (int)(wcslen(wc) - 1)
843
844
// lower-case vowels in English plus accented versions
845
static int vowels[] = {
846
    0x61, 0x65, 0x69, 0x6f, 0x75,
847
    0xe0, 0xe1, 0xe2, 0xe3, 0xe4, 0xe5,
848
    0xe8, 0xe9, 0xea, 0xeb, 0xec, 0xed, 0xee, 0xef,
849
    0xf2, 0xf3, 0xf4, 0xf5, 0xf6, 0xf8, 0xf9, 0xfa, 0xfb, 0xfc, 0xfd,
850
    0x101, 0x103, 0x105, 0x113, 0x115, 0x117, 0x119, 0x11b,
851
    0x129, 0x12b, 0x12d, 0x12f, 0x131, 0x14d, 0x14f, 0x151,
852
    0x169, 0x16b, 0x16d, 0x16f, 0x171, 0x173
853
};
854
855
static Rboolean iswvowel(wchar_t w)
856
3.48M
{
857
3.48M
    int v = (int) w, n = sizeof(vowels)/sizeof(int);
858
3.48M
    Rboolean found = FALSE;
859
159M
    for(int i = 0; i < n; i++)
860
156M
  if(v == vowels[i]) {found = TRUE; break;}
861
862
3.48M
    return found;
863
3.48M
}
864
865
static void mywcscpy(wchar_t *dest, const wchar_t *src)
866
1.72M
{
867
1.72M
    memmove(dest, src, sizeof(wchar_t) * (wcslen(src)+1));
868
1.72M
}
869
870
static SEXP wstripchars(const wchar_t * const inchar, int minlen, int usecl)
871
443
{
872
443
    int i, j, nspace = 0;
873
443
    wchar_t *wc = (wchar_t *)cbuff.data;
874
875
443
    mywcscpy(wc, inchar);
876
443
    if (wcslen(wc) < minlen) goto donewsc;
877
878
1.74M
    for (i = WUP, j = 1; i > 0; i--) {
879
1.74M
  if (iswspace((int)wc[i])) {
880
102k
      if (j) wc[i] = '\0' ; else nspace++;
881
1.64M
  } else j = 0;
882
1.74M
  if (wcslen(wc) - nspace <= minlen)
883
0
      goto donewsc;
884
1.74M
    }
885
886
443
    if(usecl) {
887
1.74M
  for (i = WUP; i > 0; i--) {
888
1.74M
      if (iswvowel(wc[i]) && LASTCHARW(i))
889
6.08k
    mywcscpy(wc + i, wc + i + 1);
890
1.74M
      if (wcslen(wc) - nspace <= minlen)
891
0
    goto donewsc;
892
1.74M
  }
893
894
1.73M
  for (i = WUP; i > 0; i--) {
895
1.73M
      if (iswvowel(wc[i]) && !FIRSTCHARW(i))
896
238k
    mywcscpy(wc + i, wc + i + 1);
897
1.73M
      if (wcslen(wc) - nspace <= minlen)
898
0
    goto donewsc;
899
1.73M
  }
900
901
1.49M
  for (i = WUP; i > 0; i--) {
902
1.49M
      if (iswlower((wint_t)wc[i]) && LASTCHARW(i))
903
6.42k
    mywcscpy(wc + i, wc + i + 1);
904
1.49M
      if (wcslen(wc) - nspace <= minlen)
905
0
    goto donewsc;
906
1.49M
  }
907
908
1.49M
  for (i = WUP; i > 0; i--) {
909
1.49M
      if (iswlower((wint_t)wc[i]) && !FIRSTCHARW(i))
910
234k
    mywcscpy(wc + i, wc + i + 1);
911
1.49M
      if (wcslen(wc) - nspace <= minlen)
912
1
    goto donewsc;
913
1.49M
  }
914
443
    }
915
916
1.25M
    for (i = WUP; i > 0; i--) {
917
1.25M
  if (!FIRSTCHARW(i) && !iswspace((int)wc[i]))
918
1.13M
      mywcscpy(wc + i, wc + i + 1);
919
1.25M
  if (wcslen(wc) - nspace <= minlen)
920
368
      goto donewsc;
921
1.25M
    }
922
923
443
donewsc:
924
925
443
    {
926
443
  int upper = (int) wcslen(wc);
927
443
  if (upper > minlen)
928
121k
      for (i = upper - 1; i > 0; i--)
929
121k
    if (iswspace((int)wc[i])) mywcscpy(wc + i, wc + i + 1);
930
443
    }
931
932
443
    size_t nb = wcstoutf8(NULL, wc, (size_t)INT_MAX + 2);
933
443
    char *cbuf = CallocCharBuf(nb);
934
443
    wcstoutf8(cbuf, wc, nb);
935
443
    SEXP ans = mkCharCE(cbuf, CE_UTF8);
936
443
    R_Free(cbuf);
937
443
    return ans;
938
442
}
939
940
941
attribute_hidden SEXP do_abbrev(SEXP call, SEXP op, SEXP args, SEXP env)
942
812
{
943
812
    checkArity(op,args);
944
812
    SEXP x = CAR(args);
945
946
812
    if (!isString(x))
947
0
  error(_("the first argument must be a character vector"));
948
812
    int minlen = asInteger(CADR(args));
949
812
    if (minlen == NA_INTEGER)
950
0
  error(_("invalid '%s' argument"), "minlength");
951
812
    int usecl = asLogical(CADDR(args));
952
812
    if (usecl == NA_INTEGER)
953
0
  error(_("invalid '%s' argument"), "use.classes");
954
955
812
    R_xlen_t len = XLENGTH(x);
956
812
    SEXP ans = PROTECT(allocVector(STRSXP, len));
957
812
    const void *vmax = vmaxget();
958
812
    Rboolean warn = FALSE;
959
1.62k
    for (R_xlen_t i = 0 ; i < len ; i++) {
960
812
  SEXP el = STRING_ELT(x, i);
961
812
  if (el  == NA_STRING)
962
0
      SET_STRING_ELT(ans, i, NA_STRING);
963
812
  else {
964
812
      const char *s = CHAR(el);
965
812
      if (IS_ASCII(el)) {
966
367
    if(strlen(s) > minlen) {
967
301
        R_AllocStringBuffer(strlen(s)+1, &cbuff);
968
301
        SET_STRING_ELT(ans, i, stripchars(s, minlen, usecl));
969
301
    } else SET_STRING_ELT(ans, i, el);
970
445
      } else {
971
445
    s = translateCharUTF8(el);
972
445
    int nc = (int) utf8towcs(NULL, s, 0);
973
445
    if (nc > minlen) {
974
443
        warn = TRUE;
975
443
        const wchar_t *wc = wtransChar(el); // to WCS-2 on Windows
976
443
        nc = (int) wcslen(wc);
977
443
        R_AllocStringBuffer(sizeof(wchar_t)*(nc+1), &cbuff);
978
443
        SET_STRING_ELT(ans, i, wstripchars(wc, minlen, usecl));
979
443
    } else SET_STRING_ELT(ans, i, el);
980
445
      }
981
812
  }
982
812
  vmaxset(vmax); // this throws away the result of wtransChar
983
812
    }
984
812
    if (usecl && warn) warning(_("abbreviate used with non-ASCII chars"));
985
812
    SHALLOW_DUPLICATE_ATTRIB(ans, x);
986
    /* This copied the class, if any */
987
812
    R_FreeStringBufferL(&cbuff);
988
812
    UNPROTECT(1);
989
812
    return ans;
990
812
}
991
992
attribute_hidden SEXP do_makenames(SEXP call, SEXP op, SEXP args, SEXP env)
993
2.12k
{
994
2.12k
    SEXP arg, ans;
995
2.12k
    R_xlen_t i, n;
996
2.12k
    int l, allow_;
997
2.12k
    char *p, *tmp = NULL, *cbuf;
998
2.12k
    const char *This;
999
2.12k
    Rboolean need_prefix;
1000
2.12k
    const void *vmax;
1001
1002
2.12k
    checkArity(op ,args);
1003
2.12k
    arg = CAR(args);
1004
2.12k
    if (!isString(arg))
1005
0
  error(_("non-character names"));
1006
2.12k
    n = XLENGTH(arg);
1007
2.12k
    allow_ = asLogical(CADR(args));
1008
2.12k
    if (allow_ == NA_LOGICAL)
1009
0
  error(_("invalid '%s' value"), "allow_");
1010
2.12k
    PROTECT(ans = allocVector(STRSXP, n));
1011
2.12k
    vmax = vmaxget();
1012
7.73k
    for (i = 0 ; i < n ; i++) {
1013
5.60k
  This = translateChar(STRING_ELT(arg, i));
1014
5.60k
  l = (int) strlen(This);
1015
  /* need to prefix names not beginning with alpha or ., as
1016
     well as . followed by a number */
1017
5.60k
  need_prefix = FALSE;
1018
5.60k
  if (mbcslocale && This[0]) {
1019
0
      int nc = l, used;
1020
0
      wchar_t wc;
1021
0
      mbstate_t mb_st;
1022
0
      const char *pp = This;
1023
0
      mbs_init(&mb_st);
1024
0
      used = (int) Mbrtowc(&wc, pp, R_MB_CUR_MAX, &mb_st);
1025
0
      pp += used; nc -= used;
1026
0
      if (wc == L'.') {
1027
0
    if (nc > 0) {
1028
0
        Mbrtowc(&wc, pp, R_MB_CUR_MAX, &mb_st);
1029
0
        if (iswdigit(wc))  need_prefix = TRUE;
1030
0
    }
1031
0
      } else if (!iswalpha(wc)) need_prefix = TRUE;
1032
5.60k
  } else {
1033
5.60k
      if (This[0] == '.') {
1034
54
    if (l >= 1 && isdigit(0xff & (int) This[1])) need_prefix = TRUE;
1035
5.55k
      } else if (!isalpha(0xff & (int) This[0])) need_prefix = TRUE;
1036
5.60k
  }
1037
  /* Use R_alloc() rather than R_Calloc(): the invalid multibyte
1038
     string error below would otherwise leak the buffer.
1039
     vmaxset() at the end of each iteration releases it. */
1040
5.60k
  if (need_prefix) {
1041
135
      tmp = R_alloc(l+2, sizeof(char));
1042
135
      strcpy(tmp, "X");
1043
135
      strcat(tmp, translateChar(STRING_ELT(arg, i)));
1044
5.47k
  } else {
1045
5.47k
      tmp = R_alloc(l+1, sizeof(char));
1046
5.47k
      strcpy(tmp, translateChar(STRING_ELT(arg, i)));
1047
5.47k
  }
1048
5.60k
  if (mbcslocale) {
1049
      /* This cannot lengthen the string, so safe to overwrite it. */
1050
0
      int nc = (int) mbstowcs(NULL, tmp, 0);
1051
0
      if (nc >= 0) {
1052
0
    wchar_t *wstr = R_Calloc(nc+1, wchar_t);
1053
0
    mbstowcs(wstr, tmp, nc+1);
1054
0
    for (wchar_t * wc = wstr; *wc; wc++) {
1055
0
        if (*wc == L'.' || (allow_ && *wc == L'_'))
1056
0
      /* leave alone */;
1057
0
        else if (!iswalnum((int)*wc)) *wc = L'.';
1058
0
    }
1059
0
    wcstombs(tmp, wstr, strlen(tmp)+1);
1060
0
    R_Free(wstr);
1061
0
      } else error(_("invalid multibyte string %lld"), (long long)i+1);
1062
5.60k
  } else {
1063
336k
      for (p = tmp; *p; p++) {
1064
330k
    if (*p == '.' || (allow_ && *p == '_')) /* leave alone */;
1065
328k
    else if (!isalnum(0xff & (int)*p)) *p = '.';
1066
    /* else leave alone */
1067
330k
      }
1068
5.60k
  }
1069
5.60k
  SET_STRING_ELT(ans, i, mkChar(tmp));
1070
  /* do we have a reserved word?  If so the name is invalid */
1071
5.60k
  if (!isValidName(tmp)) {
1072
5
      cbuf = R_alloc(strlen(tmp) + 2, sizeof(char));
1073
5
      strcpy(cbuf, tmp);
1074
5
      strcat(cbuf, ".");
1075
5
      SET_STRING_ELT(ans, i, mkChar(cbuf));
1076
5
  }
1077
5.60k
  vmaxset(vmax);
1078
5.60k
    }
1079
2.12k
    UNPROTECT(1);
1080
2.12k
    return ans;
1081
2.12k
}
1082
1083
1084
attribute_hidden SEXP do_tolower(SEXP call, SEXP op, SEXP args, SEXP env)
1085
1.36k
{
1086
1.36k
    SEXP x, y;
1087
1.36k
    R_xlen_t i, n;
1088
1.36k
    int ul;
1089
1.36k
    char *p;
1090
1.36k
    SEXP el;
1091
1.36k
    cetype_t ienc;
1092
1.36k
    Rboolean use_UTF8 = FALSE;
1093
1.36k
    const void *vmax;
1094
1095
1.36k
    checkArity(op, args);
1096
1.36k
    ul = PRIMVAL(op); /* 0 = tolower, 1 = toupper */
1097
1098
1.36k
    x = CAR(args);
1099
    /* coercion is done in wrapper */
1100
1.36k
    if (!isString(x)) error(_("non-character argument"));
1101
1.36k
    n = XLENGTH(x);
1102
1.36k
    PROTECT(y = allocVector(STRSXP, n));
1103
2.72k
    for (i = 0; i < n; i++) {
1104
1.36k
  SEXP xi = STRING_ELT(x, i);
1105
1.36k
  if (IS_UTF8(xi) ||
1106
774
      (!latin1locale && IS_LATIN1(xi))) use_UTF8 = TRUE;
1107
1.36k
    }
1108
1.36k
    if (mbcslocale || use_UTF8 == TRUE) {
1109
675
  int nb, nc, j;
1110
675
#ifndef USE_RI18N_CASE
1111
675
  wctrans_t tr = wctrans(ul ? "toupper" : "tolower");
1112
675
#endif
1113
675
  wchar_t * wc;
1114
675
  char * cbuf;
1115
1116
675
  vmax = vmaxget();
1117
  /* the translated string need not be the same length in bytes */
1118
1.35k
  for (i = 0; i < n; i++) {
1119
675
      el = STRING_ELT(x, i);
1120
675
      if (el == NA_STRING) SET_STRING_ELT(y, i, NA_STRING);
1121
675
      else {
1122
    /* FIXME: in Windows UTF-8 locales, use UTF-8 branch */
1123
675
    const char *xi;
1124
675
    ienc = getCharCE(el);
1125
675
    if (use_UTF8 && ienc == CE_UTF8) {
1126
588
        xi = CHAR(el);
1127
        // could overcount if there are conjugate pairs
1128
588
        nc = (int) utf8towcs(NULL, xi, 0);
1129
588
    } else if (use_UTF8 && ienc == CE_LATIN1) {
1130
87
        xi = translateCharUTF8(el); // in case it is really in CP1252
1131
87
        nc = (int) utf8towcs(NULL, xi, 0);
1132
87
        ienc = CE_UTF8;
1133
87
    } else {
1134
0
        xi = translateChar(el);
1135
0
        nc = (int) mbstowcs(NULL, xi, 0);
1136
0
        ienc = CE_NATIVE;
1137
0
    }
1138
675
    if (nc >= 0) {
1139
515
        if (ienc == CE_UTF8) {
1140
#ifdef USE_RI18N_CASE
1141
      R_wchar_t *wcr = (R_wchar_t *)
1142
          R_AllocStringBuffer((nc+1)*sizeof(R_wchar_t), &cbuff);
1143
      utf8towcs4(wcr, xi, nc + 1);
1144
      if (ul)
1145
          for (j = 0; j < nc; j++)
1146
        wcr[j] = Ri18n_towupper(wcr[j]);
1147
      else
1148
          for (j = 0; j < nc; j++)
1149
        wcr[j] = Ri18n_towlower(wcr[j]);
1150
      nb = (int) wcs4toutf8(NULL, wcr, INT_MAX);
1151
      cbuf = CallocCharBuf(nb);
1152
      wcs4toutf8(cbuf, wcr, nb);
1153
      SET_STRING_ELT(y, i, mkCharCE(cbuf, CE_UTF8));
1154
#else
1155
515
      wc = (wchar_t *)
1156
515
          R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1157
515
      utf8towcs(wc, xi, nc + 1);
1158
3.22M
      for (j = 0; j < nc; j++) wc[j] = towctrans(wc[j], tr);
1159
515
      nb = (int) wcstoutf8(NULL, wc, INT_MAX);
1160
515
      cbuf = CallocCharBuf(nb);
1161
515
      wcstoutf8(cbuf, wc, nb);
1162
515
      SET_STRING_ELT(y, i, mkCharCE(cbuf, CE_UTF8));
1163
515
#endif
1164
515
        } else {
1165
0
      wc = (wchar_t *)
1166
0
          R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1167
0
      mbstowcs(wc, xi, nc + 1);
1168
#ifdef USE_RI18N_CASE
1169
      if (ul)
1170
          for (j = 0; j < nc; j++)
1171
        wc[j] = Ri18n_towupper(wc[j]);
1172
      else
1173
          for (j = 0; j < nc; j++)
1174
        wc[j] = Ri18n_towlower(wc[j]);
1175
#else
1176
      /* This cannot cope with surrogate pairs,
1177
         if mbstowcs can make them. */
1178
0
      for (j = 0; j < nc; j++) wc[j] = towctrans(wc[j], tr);
1179
0
#endif
1180
0
      nb = (int) wcstombs(NULL, wc, 0);
1181
0
      cbuf = CallocCharBuf(nb);
1182
0
      wcstombs(cbuf, wc, nb + 1);
1183
0
      SET_STRING_ELT(y, i, markKnown(cbuf, el));
1184
0
        }
1185
515
        R_Free(cbuf);
1186
515
    } else {
1187
160
        error(_("invalid multibyte string %lld"), (long long)i+1);
1188
160
    }
1189
675
      }
1190
675
      vmaxset(vmax);
1191
675
  }
1192
675
  R_FreeStringBufferL(&cbuff);
1193
687
    } else {
1194
687
  char *xi;
1195
687
  vmax = vmaxget();
1196
1.37k
  for (i = 0; i < n; i++) {
1197
687
      if (STRING_ELT(x, i) == NA_STRING)
1198
0
    SET_STRING_ELT(y, i, NA_STRING);
1199
687
      else {
1200
687
    xi = CallocCharBuf(strlen(CHAR(STRING_ELT(x, i))));
1201
687
    strcpy(xi, translateChar(STRING_ELT(x, i)));
1202
462k
    for (p = xi; *p != '\0'; p++)
1203
462k
        *p = (char) (ul ? toupper(*p) : tolower(*p));
1204
687
    SET_STRING_ELT(y, i, markKnown(xi, STRING_ELT(x, i)));
1205
687
    R_Free(xi);
1206
687
      }
1207
687
      vmaxset(vmax);
1208
687
  }
1209
687
    }
1210
1.36k
    SHALLOW_DUPLICATE_ATTRIB(y, x);
1211
    /* This copied the class, if any */
1212
1.36k
    UNPROTECT(1);
1213
1.36k
    return(y);
1214
1.36k
}
1215
1216
1217
/* These assume one wchar_t per char so will not work with surrogate pairs */
1218
typedef enum { WTR_INIT, WTR_CHAR, WTR_RANGE } wtr_type;
1219
struct wtr_spec {
1220
    wtr_type type;
1221
    struct wtr_spec *next;
1222
    union {
1223
  wchar_t c;
1224
  struct {
1225
      wchar_t first;
1226
      wchar_t last;
1227
  } r;
1228
    } u;
1229
};
1230
1231
static void
1232
1.50k
wtr_build_spec(const wchar_t *s, struct wtr_spec *trs) {
1233
1.50k
    int i, len = (int) wcslen(s);
1234
1.50k
    struct wtr_spec *This, *_new;
1235
1236
1.50k
    This = trs;
1237
1.48M
    for (i = 0; i < len - 2; ) {
1238
1.48M
  _new = (struct wtr_spec *) R_alloc(1, sizeof(struct wtr_spec));
1239
1.48M
  _new->next = NULL;
1240
1.48M
  if (s[i + 1] == L'-') {
1241
962
      _new->type = WTR_RANGE;
1242
962
      if (s[i] > s[i + 2])
1243
25
    error(_("decreasing range specification ('%lc-%lc')"),
1244
25
          (wint_t)s[i], (wint_t)s[i + 2]);
1245
962
      _new->u.r.first = s[i];
1246
962
      _new->u.r.last = s[i + 2];
1247
962
      i = i + 3;
1248
1.48M
  } else {
1249
1.48M
      _new->type = WTR_CHAR;
1250
1.48M
      _new->u.c = s[i];
1251
1.48M
      i++;
1252
1.48M
  }
1253
1.48M
  This = This->next = _new;
1254
1.48M
    }
1255
4.27k
    for ( ; i < len; i++) {
1256
2.77k
  _new = (struct wtr_spec *) R_alloc(1, sizeof(struct wtr_spec));
1257
2.77k
  _new->next = NULL;
1258
2.77k
  _new->type = WTR_CHAR;
1259
2.77k
  _new->u.c = s[i];
1260
2.77k
  This = This->next = _new;
1261
2.77k
    }
1262
1.50k
}
1263
1264
static wchar_t
1265
38.4M
wtr_get_next_char_from_spec(struct wtr_spec **p) {
1266
38.4M
    wchar_t c;
1267
38.4M
    struct wtr_spec *This;
1268
1269
38.4M
    This = *p;
1270
38.4M
    if (!This)
1271
1.45k
  return('\0');
1272
38.4M
    switch(This->type) {
1273
  /* Note: this code does not deal with the WTR_INIT case. */
1274
1.48M
    case WTR_CHAR:
1275
1.48M
  c = This->u.c;
1276
1.48M
  *p = This->next;
1277
1.48M
  break;
1278
36.9M
    case WTR_RANGE:
1279
36.9M
  c = This->u.r.first;
1280
36.9M
  if (c == This->u.r.last) {
1281
893
      *p = This->next;
1282
36.9M
  } else {
1283
36.9M
      (This->u.r.first)++;
1284
36.9M
  }
1285
36.9M
  break;
1286
0
    default:
1287
0
  c = L'\0';
1288
0
  break;
1289
38.4M
    }
1290
38.4M
    return(c);
1291
38.4M
}
1292
1293
typedef enum { TR_INIT, TR_CHAR, TR_RANGE } tr_spec_type;
1294
struct tr_spec {
1295
    tr_spec_type type;
1296
    struct tr_spec *next;
1297
    union {
1298
  unsigned char c;
1299
  struct {
1300
      unsigned char first;
1301
      unsigned char last;
1302
  } r;
1303
    } u;
1304
};
1305
1306
static void
1307
500
tr_build_spec(const char *s, struct tr_spec *trs) {
1308
500
    int i, len = (int) strlen(s);
1309
500
    struct tr_spec *This, *_new;
1310
1311
500
    This = trs;
1312
11.6k
    for (i = 0; i < len - 2; ) {
1313
11.1k
  _new = (struct tr_spec *) R_alloc(1, sizeof(struct tr_spec));
1314
11.1k
  _new->next = NULL;
1315
11.1k
  if (s[i + 1] == '-') {
1316
249
      _new->type = TR_RANGE;
1317
249
      if (s[i] > s[i + 2])
1318
94
    error(_("decreasing range specification ('%c-%c')"),
1319
94
          s[i], s[i + 2]);
1320
249
      _new->u.r.first = s[i];
1321
249
      _new->u.r.last = s[i + 2];
1322
249
      i = i + 3;
1323
10.9k
  } else {
1324
10.9k
      _new->type = TR_CHAR;
1325
10.9k
      _new->u.c = s[i];
1326
10.9k
      i++;
1327
10.9k
  }
1328
11.1k
  This = This->next = _new;
1329
11.1k
    }
1330
1.15k
    for ( ; i < len; i++) {
1331
651
  _new = (struct tr_spec *) R_alloc(1, sizeof(struct tr_spec));
1332
651
  _new->next = NULL;
1333
651
  _new->type = TR_CHAR;
1334
651
  _new->u.c = s[i];
1335
651
  This = This->next = _new;
1336
651
    }
1337
500
}
1338
1339
static unsigned char
1340
14.9k
tr_get_next_char_from_spec(struct tr_spec **p) {
1341
14.9k
    unsigned char c;
1342
14.9k
    struct tr_spec *This;
1343
1344
14.9k
    This = *p;
1345
14.9k
    if (!This)
1346
366
  return('\0');
1347
14.6k
    switch(This->type) {
1348
  /* Note: this code does not deal with the TR_INIT case. */
1349
9.93k
    case TR_CHAR:
1350
9.93k
  c = This->u.c;
1351
9.93k
  *p = This->next;
1352
9.93k
  break;
1353
4.66k
    case TR_RANGE:
1354
4.66k
  c = This->u.r.first;
1355
4.66k
  if (c == This->u.r.last) {
1356
109
      *p = This->next;
1357
4.55k
  } else {
1358
4.55k
      (This->u.r.first)++;
1359
4.55k
  }
1360
4.66k
  break;
1361
0
    default:
1362
0
  c = '\0';
1363
0
  break;
1364
14.6k
    }
1365
14.6k
    return(c);
1366
14.6k
}
1367
1368
/* idx is the entry's position in the spec: qsort() is not stable, and
1369
   when a character is given more than once the last mapping wins. */
1370
typedef struct { wchar_t c_old, c_new; int idx; } xtable_t;
1371
1372
static int xtable_comp(const void *a, const void *b)
1373
130M
{
1374
130M
    const xtable_t *x = a, *y = b;
1375
130M
    if (x->c_old != y->c_old)
1376
122M
  return x->c_old < y->c_old ? -1 : 1;
1377
7.92M
    return x->idx < y->idx ? -1 : (x->idx > y->idx);
1378
130M
}
1379
1380
static R_INLINE int xtable_key_comp(const void *a, const void *b)
1381
6.99M
{
1382
6.99M
    return *((wchar_t *)a) - ((xtable_t *)b)->c_old;
1383
6.99M
}
1384
1385
2.43M
#define BSEARCH(_rc,_key,_base,_nmemb,_TYPE,_comp)             \
1386
2.43M
{                                                              \
1387
2.43M
    size_t l, u, idx;                                          \
1388
2.43M
    _TYPE *p;                                                  \
1389
2.43M
    int comp;                                                  \
1390
2.43M
    l = 0;                                                     \
1391
2.43M
    u = _nmemb;                                                \
1392
2.43M
    _rc = NULL;                                                \
1393
7.56M
    while (l < u)                                              \
1394
6.99M
    {                                                          \
1395
6.99M
  idx = (l + u) / 2;                                     \
1396
6.99M
  p =  (_base) + idx;                                    \
1397
6.99M
  comp = (*_comp)(_key, p);                              \
1398
6.99M
  if (comp < 0)                                          \
1399
6.99M
      u = idx;                                           \
1400
6.99M
  else if (comp > 0)                                     \
1401
2.63M
      l = idx + 1;                                       \
1402
2.63M
  else{                                                  \
1403
1.86M
    _rc = p;                                             \
1404
1.86M
    break;                                               \
1405
1.86M
  }                                                      \
1406
6.99M
    }                                                          \
1407
2.43M
}
1408
1409
attribute_hidden SEXP do_chartr(SEXP call, SEXP op, SEXP args, SEXP env)
1410
951
{
1411
951
    SEXP old, _new, x, y;
1412
951
    R_xlen_t i, n;
1413
951
    char *cbuf;
1414
951
    SEXP el;
1415
951
    cetype_t ienc;
1416
951
    Rboolean use_WC = FALSE;
1417
951
    const void *vmax;
1418
1419
951
    checkArity(op, args);
1420
951
    old = CAR(args); args = CDR(args);
1421
951
    _new = CAR(args); args = CDR(args);
1422
951
    x = CAR(args);
1423
951
    n = XLENGTH(x);
1424
951
    if (!isString(old) || LENGTH(old) < 1 || STRING_ELT(old, 0) == NA_STRING)
1425
0
  error(_("invalid '%s' argument"), "old");
1426
951
    if (LENGTH(old) > 1)
1427
0
  warning(_("argument '%s' has length > 1 and only the first element will be used"), "old");
1428
951
    if (!isString(_new) || LENGTH(_new) < 1 || STRING_ELT(_new, 0) == NA_STRING)
1429
0
  error(_("invalid '%s' argument"), "new");
1430
951
    if (LENGTH(_new) > 1)
1431
0
  warning(_("argument '%s' has length > 1 and only the first element will be used"), "new");
1432
951
    if (!isString(x)) error("invalid '%s' argument", "x");
1433
1434
    /* If we have marked strings we want to do this in Unicode as some
1435
     * of them might be mis-represented by translateChar.  But
1436
     * utf8towcs may not be reliable unless TO_WCS_OK is defined.
1437
     */
1438
1.90k
    for (i = 0; i < n; i++) {
1439
951
  SEXP xi = STRING_ELT(x, i);
1440
951
  if (IS_UTF8(xi) || (!latin1locale && IS_LATIN1(xi))) use_WC = TRUE;
1441
951
    }
1442
1443
951
    if (IS_UTF8(STRING_ELT(old, 0)) ||
1444
679
  (!latin1locale && IS_LATIN1(STRING_ELT(old, 0)))) use_WC = TRUE;
1445
951
    if (IS_UTF8(STRING_ELT(_new, 0)) ||
1446
682
  (!latin1locale && IS_LATIN1(STRING_ELT(_new, 0)))) use_WC = TRUE;
1447
1448
951
    if (mbcslocale || use_WC == TRUE) {
1449
656
  int j, nb, nc;
1450
656
  xtable_t *xtable, *tbl;
1451
656
  R_xlen_t xtable_cnt;
1452
656
  struct wtr_spec *trs_cnt, **trs_cnt_ptr;
1453
656
  wchar_t c_old, c_new, *wc;
1454
656
  const char *xi, *s;
1455
656
  struct wtr_spec *trs_old, **trs_old_ptr;
1456
656
  struct wtr_spec *trs_new, **trs_new_ptr;
1457
1458
  /* The spec lists come from R_alloc(), so the errors raised while
1459
     they are built (a decreasing range, 'old' longer than 'new',
1460
     an invalid string) do not leak them. */
1461
656
  trs_old = (struct wtr_spec *) R_alloc(1, sizeof(struct wtr_spec));
1462
656
  trs_old->type = WTR_INIT;
1463
656
  trs_old->next = NULL;
1464
656
  trs_new = (struct wtr_spec *) R_alloc(1, sizeof(struct wtr_spec));
1465
656
  trs_new->type = WTR_INIT;
1466
656
  trs_new->next = NULL;
1467
  /* Build the old and new wtr_spec lists. */
1468
656
  if (use_WC && IS_UTF8(STRING_ELT(old, 0))) {
1469
272
      s = CHAR(STRING_ELT(old, 0));
1470
      /* utf8towcs() still decodes the withdrawn 5- and 6-byte forms,
1471
         so without this check a range can span 2^31 code points. */
1472
272
      if (!utf8Valid(s)) error(_("invalid UTF-8 string 'old'"));
1473
272
      nc = (int) utf8towcs(NULL, s, 0);
1474
272
      if (nc < 0) error(_("invalid UTF-8 string 'old'"));
1475
272
      wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1476
272
      utf8towcs(wc, s, nc + 1);
1477
384
  } else if (use_WC && IS_LATIN1(STRING_ELT(old, 0))) {
1478
0
      s = translateCharUTF8(STRING_ELT(old, 0));
1479
0
      nc = (int) utf8towcs(NULL, s, 0);
1480
0
      if (nc < 0) error(_("invalid UTF-8 string 'old'")); // but must be valid
1481
0
      wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1482
0
      utf8towcs(wc, s, nc + 1);
1483
384
  } else {
1484
384
      s = translateChar(STRING_ELT(old, 0));
1485
384
      nc = (int) mbstowcs(NULL, s, 0);
1486
384
      if (nc < 0) error(_("invalid multibyte string 'old'"));
1487
384
      wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1488
384
      mbstowcs(wc, s, nc + 1);
1489
384
  }
1490
656
  wtr_build_spec(wc, trs_old);
1491
656
  trs_cnt = (struct wtr_spec *) R_alloc(1, sizeof(struct wtr_spec));
1492
656
  trs_cnt->type = WTR_INIT;
1493
656
  trs_cnt->next = NULL;
1494
656
  wtr_build_spec(wc, trs_cnt); /* use count only */
1495
1496
656
  if (use_WC && IS_UTF8(STRING_ELT(_new, 0))) {
1497
108
      s = CHAR(STRING_ELT(_new, 0));
1498
108
      if (!utf8Valid(s)) error(_("invalid UTF-8 string 'new'"));
1499
108
      nc = (int) utf8towcs(NULL, s, 0);
1500
108
      if (nc < 0) error(_("invalid UTF-8 string 'new'"));
1501
108
      wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1502
108
      utf8towcs(wc, s, nc + 1);
1503
548
  } else if (use_WC && IS_LATIN1(STRING_ELT(_new, 0))) {
1504
0
      s = translateCharUTF8(STRING_ELT(_new, 0));
1505
0
      nc = (int) utf8towcs(NULL, s, 0);
1506
0
      if (nc < 0) error(_("invalid UTF-8 string 'new'"));
1507
0
      wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1508
0
      utf8towcs(wc, s, nc + 1);
1509
548
  } else {
1510
548
      s = translateChar(STRING_ELT(_new, 0));
1511
548
      nc = (int) mbstowcs(NULL, s, 0);
1512
548
      if (nc < 0) error(_("invalid multibyte string 'new'"));
1513
548
      wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t), &cbuff);
1514
548
      mbstowcs(wc, s, nc + 1);
1515
548
  }
1516
656
  wtr_build_spec(wc, trs_new);
1517
1518
  /* Initialize the pointers for walking through the old and new
1519
     wtr_spec lists and retrieving the next chars from the lists.
1520
  */
1521
1522
656
  trs_cnt_ptr = (struct wtr_spec **) R_alloc(1, sizeof(struct wtr_spec *));
1523
656
  *trs_cnt_ptr = trs_cnt->next;
1524
12.8M
  for (xtable_cnt = 0 ; wtr_get_next_char_from_spec(trs_cnt_ptr);
1525
12.8M
        xtable_cnt++) ;
1526
  /* the table is indexed with int below; counted in int, 2^32
1527
     entries wrapped to 0 and the table was written past its end */
1528
656
  if (xtable_cnt > INT_MAX)
1529
0
      error(_("'old' expands to too many characters"));
1530
656
  xtable = (xtable_t *) R_alloc(xtable_cnt+1, sizeof(xtable_t));
1531
1532
656
  trs_old_ptr = (struct wtr_spec **) R_alloc(1, sizeof(struct wtr_spec *));
1533
656
  *trs_old_ptr = trs_old->next;
1534
656
  trs_new_ptr = (struct wtr_spec **) R_alloc(1, sizeof(struct wtr_spec *));
1535
656
  *trs_new_ptr = trs_new->next;
1536
12.8M
  for (i = 0; ; i++) {
1537
12.8M
      c_old = wtr_get_next_char_from_spec(trs_old_ptr);
1538
12.8M
      c_new = wtr_get_next_char_from_spec(trs_new_ptr);
1539
12.8M
      if (c_old == '\0')
1540
489
    break;
1541
12.8M
      else if (c_new == '\0')
1542
2
    error(_("'old' is longer than 'new'"));
1543
12.8M
      else {
1544
12.8M
    xtable[i].c_old = c_old;
1545
12.8M
    xtable[i].c_new = c_new;
1546
12.8M
    xtable[i].idx = (int) i;
1547
12.8M
      }
1548
12.8M
  }
1549
1550
  /* Sort by c_old, keeping only the last entry for each: the
1551
     insertion sort used before was quadratic in the table size,
1552
     and a spec with two large ranges took tens of seconds. */
1553
656
  qsort(xtable, xtable_cnt, sizeof(xtable_t), xtable_comp);
1554
12.8M
  for (j = 0, i = 0; i < xtable_cnt; i++)
1555
12.8M
      if (i + 1 == xtable_cnt || xtable[i].c_old != xtable[i + 1].c_old)
1556
8.83M
    xtable[j++] = xtable[i];
1557
656
  xtable_cnt = j;
1558
1559
656
  PROTECT(y = allocVector(STRSXP, n));
1560
656
  vmax = vmaxget();
1561
1.14k
  for (i = 0; i < n; i++) {
1562
489
      el = STRING_ELT(x,i);
1563
489
      if (el == NA_STRING)
1564
0
    SET_STRING_ELT(y, i, NA_STRING);
1565
489
      else {
1566
489
    ienc = getCharCE(el);
1567
489
    if (use_WC && ienc == CE_UTF8) {
1568
425
        xi = CHAR(el);
1569
425
        nc = (int) utf8towcs(NULL, xi, 0);
1570
425
    } else {
1571
64
        xi = translateChar(el);
1572
64
        nc = (int) mbstowcs(NULL, xi, 0);
1573
64
        ienc = CE_NATIVE;
1574
64
    }
1575
489
    if (nc < 0)
1576
0
        error(_("invalid input multibyte string %lld"),
1577
0
              (long long)i+1);
1578
489
    wc = (wchar_t *) R_AllocStringBuffer((nc+1)*sizeof(wchar_t),
1579
489
                 &cbuff);
1580
489
    if (ienc == CE_UTF8) utf8towcs(wc, xi, nc + 1);
1581
220
    else mbstowcs(wc, xi, nc + 1);
1582
2.43M
    for (j = 0; j < nc; j++){
1583
2.43M
        BSEARCH(tbl,&wc[j], xtable, xtable_cnt,
1584
2.43M
          xtable_t, xtable_key_comp);
1585
2.43M
        if (tbl) wc[j] = tbl->c_new;
1586
2.43M
    }
1587
489
    if (ienc == CE_UTF8) {
1588
269
        nb = (int) wcstoutf8(NULL, wc, INT_MAX);
1589
269
        cbuf = CallocCharBuf(nb);
1590
269
        wcstoutf8(cbuf, wc, nb);
1591
269
        SET_STRING_ELT(y, i, mkCharCE(cbuf, CE_UTF8));
1592
269
    } else {
1593
220
        size_t nbytes = wcstombs(NULL, wc, 0);
1594
220
        if (nbytes == (size_t)-1)
1595
0
      error(_("translated string is not representable in the current locale"));
1596
220
        if (nbytes > INT_MAX)
1597
0
      error(_("R character strings are limited to 2^31-1 bytes"));
1598
220
        cbuf = CallocCharBuf(nbytes);
1599
220
        wcstombs(cbuf, wc, nbytes + 1);
1600
220
        SET_STRING_ELT(y, i, markKnown(cbuf, el));
1601
220
    }
1602
489
    R_Free(cbuf);
1603
489
      }
1604
489
      vmaxset(vmax);
1605
489
  }
1606
656
  R_FreeStringBufferL(&cbuff);
1607
656
    } else {
1608
295
  unsigned char xtable[UCHAR_MAX + 1], *p, c_old, c_new;
1609
295
  struct tr_spec *trs_old, **trs_old_ptr;
1610
295
  struct tr_spec *trs_new, **trs_new_ptr;
1611
1612
75.8k
  for (unsigned int ii = 0; ii <= UCHAR_MAX; ii++)
1613
75.5k
      xtable[ii] = (unsigned char) ii;
1614
1615
  /* R_alloc(), as above. */
1616
295
  trs_old = (struct tr_spec *) R_alloc(1, sizeof(struct tr_spec));
1617
295
  trs_old->type = TR_INIT;
1618
295
  trs_old->next = NULL;
1619
295
  trs_new = (struct tr_spec *) R_alloc(1, sizeof(struct tr_spec));
1620
295
  trs_new->type = TR_INIT;
1621
295
  trs_new->next = NULL;
1622
  /* Build the old and new tr_spec lists. */
1623
295
  tr_build_spec(translateChar(STRING_ELT(old, 0)), trs_old);
1624
295
  tr_build_spec(translateChar(STRING_ELT(_new, 0)), trs_new);
1625
  /* Initialize the pointers for walking through the old and new
1626
     tr_spec lists and retrieving the next chars from the lists.
1627
  */
1628
295
  trs_old_ptr = (struct tr_spec **) R_alloc(1, sizeof(struct tr_spec *));
1629
295
  *trs_old_ptr = trs_old->next;
1630
295
  trs_new_ptr = (struct tr_spec **) R_alloc(1, sizeof(struct tr_spec *));
1631
295
  *trs_new_ptr = trs_new->next;
1632
7.48k
  for (;;) {
1633
7.48k
      c_old = tr_get_next_char_from_spec(trs_old_ptr);
1634
7.48k
      c_new = tr_get_next_char_from_spec(trs_new_ptr);
1635
7.48k
      if (c_old == '\0')
1636
197
    break;
1637
7.28k
      else if (c_new == '\0')
1638
4
    error(_("'old' is longer than 'new'"));
1639
7.28k
      else
1640
7.28k
    xtable[c_old] = c_new;
1641
7.48k
  }
1642
295
  n = LENGTH(x);
1643
295
  PROTECT(y = allocVector(STRSXP, n));
1644
295
  vmax = vmaxget();
1645
492
  for (i = 0; i < n; i++) {
1646
197
      if (STRING_ELT(x,i) == NA_STRING)
1647
0
    SET_STRING_ELT(y, i, NA_STRING);
1648
197
      else {
1649
197
    const char *xi = translateChar(STRING_ELT(x, i));
1650
197
    cbuf = CallocCharBuf(strlen(xi));
1651
197
    strcpy(cbuf, xi);
1652
57.1k
    for (p = (unsigned char *) cbuf; *p != '\0'; p++)
1653
56.9k
        *p = xtable[*p];
1654
197
    SET_STRING_ELT(y, i, markKnown(cbuf, STRING_ELT(x, i)));
1655
197
    R_Free(cbuf);
1656
197
      }
1657
197
  }
1658
295
  vmaxset(vmax);
1659
295
    }
1660
1661
951
    SHALLOW_DUPLICATE_ATTRIB(y, x);
1662
    /* This copied the class, if any */
1663
951
    UNPROTECT(1);
1664
951
    return(y);
1665
951
}
1666
1667
attribute_hidden SEXP do_strtrim(SEXP call, SEXP op, SEXP args, SEXP env)
1668
1.49k
{
1669
1.49k
    SEXP s, x, width;
1670
1.49k
    R_xlen_t i, len;
1671
1.49k
    int nw, w, nc;
1672
1.49k
    const char *This;
1673
1.49k
    char *buf;
1674
1.49k
    const char *p; char *q;
1675
1.49k
    int w0, wsum, k, nb;
1676
1.49k
    mbstate_t mb_st;
1677
1.49k
    const void *vmax;
1678
1679
1.49k
    checkArity(op, args);
1680
    /* as.character happens at R level now */
1681
1.49k
    if (!isString(x = CAR(args)))
1682
0
  error(_("strtrim() requires a character vector"));
1683
1.49k
    len = XLENGTH(x);
1684
1.49k
    PROTECT(s = allocVector(STRSXP, len));
1685
1.49k
    if(len > 0) {
1686
1.49k
  PROTECT(width = coerceVector(CADR(args), INTSXP));
1687
1.49k
  nw = LENGTH(width);
1688
1.49k
  if (!nw || (nw < len && len % nw))
1689
0
      error(_("invalid '%s' argument"), "width");
1690
2.98k
  for (i = 0; i < nw; i++)
1691
1.49k
      if (INTEGER(width)[i] == NA_INTEGER ||
1692
1.49k
    INTEGER(width)[i] < 0)
1693
0
    error(_("invalid '%s' argument"), "width");
1694
1.49k
  vmax = vmaxget();
1695
2.98k
  for (i = 0; i < len; i++) {
1696
1.49k
      if (STRING_ELT(x, i) == NA_STRING) {
1697
0
    SET_STRING_ELT(s, i, STRING_ELT(x, i));
1698
0
    continue;
1699
0
      }
1700
1.49k
      w = INTEGER(width)[i % nw];
1701
      // FIXME: this could do a better job with UTF-8 or Latin-1 input
1702
1.49k
      This = translateChar(STRING_ELT(x, i));
1703
1.49k
      nc = (int) strlen(This);
1704
1.49k
      buf = R_AllocStringBuffer(nc, &cbuff);
1705
1.49k
      wsum = 0;
1706
1.49k
      mbs_init(&mb_st);
1707
96.8k
      for (p = This, w0 = 0, q = buf; *p ;) {
1708
96.4k
    wchar_t wc;
1709
96.4k
    nb =  (int) Mbrtowc(&wc, p, R_MB_CUR_MAX, &mb_st);
1710
96.4k
#ifdef USE_RI18N_WIDTH
1711
96.4k
    w0 = Ri18n_wcwidth((R_wchar_t) wc);
1712
#else
1713
    w0 = wcwidth(wc);
1714
#endif
1715
96.4k
    if (w0 < 0) { p += nb; continue; } /* skip non-printable chars */
1716
96.4k
    wsum += w0;
1717
96.4k
    if (wsum <= w) {
1718
190k
        for (k = 0; k < nb; k++) *q++ = *p++;
1719
95.3k
    } else break;
1720
96.4k
      }
1721
1.49k
      *q = '\0';
1722
1.49k
      SET_STRING_ELT(s, i, markKnown(buf, STRING_ELT(x, i)));
1723
1.49k
      vmaxset(vmax);
1724
1.49k
  }
1725
1.49k
  R_FreeStringBufferL(&cbuff);
1726
1.49k
  UNPROTECT(1);
1727
1.49k
    }
1728
1.49k
    SHALLOW_DUPLICATE_ATTRIB(s, x);
1729
    /* This copied the class, if any */
1730
1.49k
    UNPROTECT(1);
1731
1.49k
    return s;
1732
1.49k
}
1733
1734
static int strtoi(SEXP s, int base)
1735
243
{
1736
243
    if(s == NA_STRING || CHAR(s)[0] == '\0') return(NA_INTEGER);
1737
1738
    /* strtol might return extreme values on error */
1739
243
    errno = 0;
1740
242
    char *endp;
1741
242
    long int res = strtol(CHAR(s), &endp, base); /* ASCII */
1742
242
    return (errno || *endp != '\0' ||
1743
235
      res > INT_MAX || res < INT_MIN)
1744
242
  ? NA_INTEGER
1745
242
  : (int) res;
1746
243
}
1747
1748
attribute_hidden SEXP do_strtoi(SEXP call, SEXP op, SEXP args, SEXP env)
1749
243
{
1750
243
    SEXP ans, x, b;
1751
243
    R_xlen_t i, n;
1752
243
    int base;
1753
1754
243
    checkArity(op, args);
1755
1756
243
    x = CAR(args); args = CDR(args);
1757
243
    b = CAR(args);
1758
1759
243
    if(!isInteger(b) || (LENGTH(b) < 1))
1760
0
  error(_("invalid '%s' argument"), "base");
1761
243
    base = INTEGER(b)[0];
1762
243
    if((base != 0) && ((base < 2) || (base > 36)))
1763
0
  error(_("invalid '%s' argument"), "base");
1764
1765
243
    PROTECT(ans = allocVector(INTSXP, n = LENGTH(x)));
1766
486
    for(i = 0; i < n; i++)
1767
243
  INTEGER(ans)[i] = strtoi(STRING_ELT(x, i), base);
1768
243
    UNPROTECT(1);
1769
1770
243
    return ans;
1771
243
}
1772
1773
/* creates a new STRSXP which is a suffix of string, starting
1774
   with given index; the result is returned unprotected  */
1775
1776
170k
attribute_hidden SEXP stringSuffix(SEXP string, int fromIndex) {
1777
1778
170k
    int origLen = LENGTH(string);
1779
170k
    int newLen = origLen - fromIndex;
1780
1781
170k
    SEXP res = PROTECT(allocVector(STRSXP, newLen));
1782
170k
    int i;
1783
334k
    for(i = 0; i < newLen; i++) {
1784
164k
  SET_STRING_ELT(res, i, STRING_ELT(string, fromIndex++));
1785
164k
    }
1786
1787
170k
    UNPROTECT(1); /* res */
1788
170k
    return res;
1789
170k
}
1790
1791
attribute_hidden SEXP do_strrep(SEXP call, SEXP op, SEXP args, SEXP env)
1792
135
{
1793
135
    SEXP d, s, x, n, el;
1794
135
    R_xlen_t is, ix, in, ns, nx, nn;
1795
135
    const char *xi;
1796
135
    int j, ni, nc;
1797
135
    const char *cbuf;
1798
135
    char *buf;
1799
135
    const void *vmax;
1800
1801
135
    checkArity(op, args);
1802
1803
135
    x = CAR(args); args = CDR(args);
1804
135
    n = CAR(args);
1805
1806
135
    nx = XLENGTH(x);
1807
135
    nn = XLENGTH(n);
1808
135
    if((nx == 0) || (nn == 0))
1809
0
  return allocVector(STRSXP, 0);
1810
1811
135
    ns = (nx > nn) ? nx : nn;
1812
1813
135
    PROTECT(s = allocVector(STRSXP, ns));
1814
135
    vmax = vmaxget();
1815
135
    is = ix = in = 0;
1816
4.09k
    for(; is < ns; is++) {
1817
3.95k
  el = STRING_ELT(x, ix);
1818
3.95k
  ni = INTEGER(n)[in];
1819
3.95k
  if((el == NA_STRING) || (ni == NA_INTEGER)) {
1820
0
      SET_STRING_ELT(s, is, NA_STRING);
1821
3.95k
  } else {
1822
3.95k
      if(ni < 0)
1823
0
    error(_("invalid '%s' value"), "times");
1824
3.95k
      xi = CHAR(el);
1825
3.95k
      nc = (int) strlen(xi);
1826
1827
      /* check for feasible result length; use double to protect
1828
         against integer overflow */
1829
3.95k
      double len = ((double) nc) * ni;
1830
3.95k
      if (len > INT_MAX)
1831
0
    error("R character strings are limited to 2^31-1 bytes");
1832
1833
3.95k
      cbuf = buf = CallocCharBuf(nc * ni);
1834
22.2k
      for(j = 0; j < ni; j++) {
1835
18.2k
    strcpy(buf, xi);
1836
18.2k
    buf += nc;
1837
18.2k
      }
1838
3.95k
      SET_STRING_ELT(s, is, mkCharCE(cbuf, getCharCE(el)));
1839
3.95k
      R_Free(cbuf);
1840
3.95k
      vmaxset(vmax);
1841
3.95k
  }
1842
3.95k
  ix = (++ix == nx) ? 0 : ix;
1843
3.95k
  in = (++in == nn) ? 0 : in;
1844
3.95k
    }
1845
    /* Copy names if not recycled. */
1846
135
    if((ns == nx) &&
1847
36
       (d = getAttrib(x, R_NamesSymbol)) != R_NilValue)
1848
0
  setAttrib(s, R_NamesSymbol, d);
1849
135
    UNPROTECT(1);
1850
135
    return s;
1851
135
}