/src/httrack/src/htsencoding.c

Source
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 2013 Xavier Roche and other contributors

SPDX-License-Identifier: GPL-3.0-or-later

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.

Please visit our Website: http://www.httrack.com
*/

/* ------------------------------------------------------------ */
/* File: Encoding conversion functions                          */
/* Author: Xavier Roche                                         */
/* ------------------------------------------------------------ */

#include <stdint.h>

#include "htscharset.h"
#include "htsencoding.h"
#include "htssafe.h"

/* static int decode_entity(const uint64_t hash, const size_t len);
 */
#include "htsentities.h"

/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
 */
#define HASH_INIT 0xcbf29ce484222325ULL
#define HASH_PRIME 0x100000001b3ULL
#define HASH_ADD(HASH, C)                                                      \
  do {                                                                         \
    (HASH) ^= (unsigned char) (C);                                             \
    (HASH) *= HASH_PRIME;                                                      \
  } while (0)

int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
  return hts_unescapeEntitiesWithCharsetSpecial(src, dest, max, charset, 0);
}

int hts_unescapeEntitiesWithCharsetSpecial(const char *src, char *dest,
                                           const size_t max,
                                           const char *charset,
                                           const int flags) {
  size_t i, j, ampStart, ampStartDest;
  int uc;
  int hex;
  uint64_t hash;

  assertf(max != 0);
  for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
      hash = HASH_INIT;
       src[i] != '\0'; i++) {
    /* start of entity */
    if (src[i] == '&') {
      ampStart = i;
      ampStartDest = j;
      hash = HASH_INIT;
      uc = -1;
    }
    /* inside a potential entity */
    else if (ampStart != (size_t) -1) {
      /* &#..; entity */
      if (ampStart + 1 == i && src[ampStart + 1] == '#') {
        uc = 0;
        hex = 0;
      }
      /* &#x..; entity */
      else if (ampStart + 2 == i && src[ampStart + 1] == '#'
               && src[ampStart + 2] == 'x') {
        hex = 1;
      }
      /* end of entity */
      else if (src[i] == ';') {
        size_t len;
        
        /* decode entity */
        if (uc == -1) {
          /* &foo; */
          uc = decode_entity(hash, /*&src[ampStart + 1],*/
                             i - ampStart - 1);
          /* FIXME: TEMPORARY HACK FROM PREVIOUS VERSION TO BE INVESTIGATED */
          if (uc == 160) {
            uc = 32;
          }
        }
        
        /* end */
        ampStart = (size_t) -1;
        
        /* success ? */
        if (uc > 0) {
          const size_t maxOut = max - ampStartDest;
          /* write at position */
          if (charset != NULL && hts_isCharsetUTF8(charset)) {
            len = hts_writeUTF8(uc, &dest[ampStartDest], maxOut);
          } else {
            size_t ulen;
            char buffer[32];
            len = 0;
            if ( ( ulen = hts_writeUTF8(uc, buffer, sizeof(buffer)) ) != 0) {
              const hts_boolean urlQuery =
                  (flags & UNESCAPE_ENTITIES_URL_QUERY) != 0;
              char *s;
              buffer[ulen] = '\0';
              /* Strict for a query only: a substituted '?' must not pass for
                 the code point the document wrote. */
              if (urlQuery) {
                s = hts_convertStringFromUTF8Strict(buffer, strlen(buffer),
                                                    charset);
              } else {
                s = hts_convertStringFromUTF8(buffer, strlen(buffer), charset);
              }
              if (s != NULL) {
                const size_t sLen = strlen(s);
                if (sLen < maxOut) {
                  /* Do not copy \0. */
                  memcpy(&dest[ampStartDest], s, sLen);
                  len = sLen;
                }
                freet(s);
              } else if (urlQuery) {
                /* URL Standard: an unrepresentable code point is written
                   %26%23<decimal>%3B rather than left as source text. */
                char esc[32];
                const int escLen =
                    snprintf(esc, sizeof(esc), "%%26%%23%d%%3B", uc);

                if (escLen > 0 && (size_t) escLen < maxOut) {
                  memcpy(&dest[ampStartDest], esc, (size_t) escLen);
                  len = (size_t) escLen;
                }
              }
            }
          }
          if (len > 0) {
            /* new dest position */
            j = ampStartDest + len;
            /* do not copy ; */
            continue;
          }
        }
      }
      /* numerical entity */
      else if (uc != -1) {
        /* decimal */
        if (!hex) {
          if (src[i] >= '0' && src[i] <= '9') {
            const int h = src[i] - '0';
            /* Guard before multiplying: a codepoint past the Unicode max
               (0x10FFFF) is invalid anyway, so stop rather than overflow uc. */
            if (uc > (0x10FFFF - h) / 10) {
              ampStart = (size_t) -1;
            } else {
              uc = uc * 10 + h;
            }
          } else {
            /* abandon */
            ampStart = (size_t) -1;
          }
        }
        /* hex */
        else {
          const int h = hts_ehexh(src[i]);
          if (h != -1) {
            if (uc > (0x10FFFF - h) / 16) {
              ampStart = (size_t) -1;
            } else {
              uc = uc * 16 + h;
            }
          } else {
            /* abandon */
            ampStart = (size_t) -1;
          }
        }
      }
      /* alphanumerical entity */
      else {
        /* alphanum, capped at the longest name
         * '&CounterClockwiseContourIntegral;' (31) */
        if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') ||
                                   (src[i] >= 'A' && src[i] <= 'Z') ||
                                   (src[i] >= 'a' && src[i] <= 'z'))) {
          /* compute hash */
          HASH_ADD(hash, (unsigned char) src[i]);
        } else {
          /* abandon */
          ampStart = (size_t) -1;
        }
      }
    }

    /* reserve one byte for the trailing NUL written after the loop */
    if (j + 1 >= max) {
      /* overflow */
      return -1;
    }
    if (src != dest || i != j) {
      dest[j] = src[i];
    }
    j++;
  }
  dest[j] = '\0';

  return 0;
}

int hts_unescapeEntities(const char *src, char *dest, const size_t max) {
  return hts_unescapeEntitiesWithCharset(src, dest, max, "UTF-8");
}

int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
                           const int flags) {
  size_t i, j, lastI, lastJ, k, utfBufferJ, utfBufferSize;
  int seenQuery = 0;
  char utfBuffer[32];

  assertf(src != dest);
  assertf(max != 0);

  for(i = 0, j = 0, k = 0, utfBufferJ = 0, utfBufferSize = 0,
      lastI = (size_t) -1, lastJ = (size_t) -1
      ; src[i] != '\0' ; i++) {
    char c = src[i];
    unsigned char cUtf = (unsigned char) c;

    /* Replacement for ' ' */
    if (c == '+' && seenQuery) {
      c = cUtf = ' ';
      k = 0;  /* cancel any sequence */
    }
    /* Escape sequence start */
    else if (c == '%') {
      /* last known position of % written on destination
         copy blindly c, we'll rollback later */
      lastI = i;
      lastJ = j;
    }
    /* End of sequence seen */
    else if (i >= 2 && i == lastI + 2) {
      const int a1 = hts_ehexh(src[lastI + 1]);
      const int a2 = hts_ehexh(src[lastI + 2]);
      if (a1 != -1 && a2 != -1) {
        const char ec = a1*16 + a2;  /* new character */
        cUtf = (unsigned char) ec;

        /* Shortcut for ASCII (do not unescape non-printable) */
        if (
            (cUtf < 0x80 && cUtf >= 32)
            && ( flags & UNESCAPE_URL_NO_ASCII ) == 0
            ) {
          /* Rollback new write position and character */
          j = lastJ;
          c = ec;
        }
      } else {
        k = 0;  /* cancel any sequence */
      }
    }
    /* ASCII (and not in %xx) */
    else if (cUtf < 0x80 && i != lastI + 1) {
      k = 0;  /* cancel any sequence */
      if (c == '?' && !seenQuery) {
        seenQuery = 1;
      }
    }
    
    /* UTF-8 sequence in progress (either a raw or a %xx character) */
    if (cUtf >= 0x80) {
      /* Leading UTF ? */
      if (HTS_IS_LEADING_UTF8(cUtf)) {
        k = 0;  /* cancel any sequence */
      }

      /* Copy */
      if (k < sizeof(utfBuffer)) {
        /* First character */
        if (k == 0) {
          /* New destination-centric offset of utf-8 buffer beginning */
          if (lastI != (size_t) -1 && i == lastI + 2) {  /* just read a %xx */
            utfBufferJ = lastJ;  /* position of % */
          } else {
            utfBufferJ = j;      /* current position otherwise */
          }

          /* Sequence length */
          utfBufferSize = hts_getUTF8SequenceLength(cUtf);
        }

        /* Copy */
        utfBuffer[k++] = cUtf;

        /* Flush UTF-8 buffer when completed. */
        if (k == utfBufferSize) {
          const size_t nRead = hts_readUTF8(utfBuffer, utfBufferSize, NULL);

          /* Reset UTF-8 buffer in all cases. */
          k = 0;

          /* Was the character read successfully ? */
          if (nRead == utfBufferSize) {
            /* the 'continue' below skips the NUL-reserve guard: re-check */
            if (utfBufferJ + utfBufferSize >= max) {
              return -1;
            }

            /* Rollback write position to sequence start write position */
            j = utfBufferJ;

            /* Copy full character sequence */
            memcpy(&dest[j], utfBuffer, utfBufferSize);
            j += utfBufferSize;

            /* Skip current character */
            continue;
          }
        }
      }
    }

    /* reserve one byte for the trailing NUL written after the loop */
    if (j + 1 >= max) {
      return -1;
    }

    /* Copy current */
    dest[j++] = c;
  }
  dest[j] = '\0';

  return 0;
}

int hts_unescapeUrl(const char *src, char *dest, const size_t max) {
  return hts_unescapeUrlSpecial(src, dest, max, 0);
}

Coverage Report

Created: 2026-09-01 06:54

Line	Count	Source
1		/* ------------------------------------------------------------ */
2		/*
3		HTTrack Website Copier, Offline Browser for Windows and Unix
4		Copyright (C) 2013 Xavier Roche and other contributors
5
6		SPDX-License-Identifier: GPL-3.0-or-later
7
8		This program is free software: you can redistribute it and/or modify
9		it under the terms of the GNU General Public License as published by
10		the Free Software Foundation, either version 3 of the License, or
11		(at your option) any later version.
12
13		This program is distributed in the hope that it will be useful,
14		but WITHOUT ANY WARRANTY; without even the implied warranty of
15		MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16		GNU General Public License for more details.
17
18		You should have received a copy of the GNU General Public License
19		along with this program. If not, see <http://www.gnu.org/licenses/>.
20
21		Ethical use: we kindly ask that you NOT use this software to harvest email
22		addresses or to collect any other private information about people. Doing so
23		would dishonor our work and waste the many hours we have spent on it.
24
25		Please visit our Website: http://www.httrack.com
26		*/
27
28		/* ------------------------------------------------------------ */
29		/* File: Encoding conversion functions */
30		/* Author: Xavier Roche */
31		/* ------------------------------------------------------------ */
32
33		#include <stdint.h>
34
35		#include "htscharset.h"
36		#include "htsencoding.h"
37		#include "htssafe.h"
38
39		/* static int decode_entity(const uint64_t hash, const size_t len);
40		*/
41		#include "htsentities.h"
42
43		/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
44		*/
45	527k	#define HASH_INIT 0xcbf29ce484222325ULL
46	2.34M	#define HASH_PRIME 0x100000001b3ULL
47		#define HASH_ADD(HASH, C) \
48	2.34M	do { \
49	2.34M	(HASH) ^= (unsigned char) (C); \
50	2.34M	(HASH) *= HASH_PRIME; \
51	2.34M	} while (0)
52
53	71.5k	int hts_unescapeEntitiesWithCharset(const char src, char dest, const size_t max, const char *charset) {
54	71.5k	return hts_unescapeEntitiesWithCharsetSpecial(src, dest, max, charset, 0);
55	71.5k	}
56
57		int hts_unescapeEntitiesWithCharsetSpecial(const char src, char dest,
58		const size_t max,
59		const char *charset,
60	71.5k	const int flags) {
61	71.5k	size_t i, j, ampStart, ampStartDest;
62	71.5k	int uc;
63	71.5k	int hex;
64	71.5k	uint64_t hash;
65
66	71.5k	assertf(max != 0);
67	71.5k	for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
68	71.5k	hash = HASH_INIT;
69	3.54M	src[i] != '\0'; i++) {
70		/* start of entity */
71	3.47M	if (src[i] == '&') {
72	455k	ampStart = i;
73	455k	ampStartDest = j;
74	455k	hash = HASH_INIT;
75	455k	uc = -1;
76	455k	}
77		/* inside a potential entity */
78	3.02M	else if (ampStart != (size_t) -1) {
79		/* &#..; entity */
80	2.78M	if (ampStart + 1 == i && src[ampStart + 1] == '#') {
81	2.40k	uc = 0;
82	2.40k	hex = 0;
83	2.40k	}
84		/* &#x..; entity */
85	2.77M	else if (ampStart + 2 == i && src[ampStart + 1] == '#'
86	2.17k	&& src[ampStart + 2] == 'x') {
87	1.08k	hex = 1;
88	1.08k	}
89		/* end of entity */
90	2.77M	else if (src[i] == ';') {
91	416k	size_t len;
92
93		/* decode entity */
94	416k	if (uc == -1) {
95		/* &foo; */
96	416k	uc = decode_entity(hash, /&src[ampStart + 1],/
97	416k	i - ampStart - 1);
98		/* FIXME: TEMPORARY HACK FROM PREVIOUS VERSION TO BE INVESTIGATED */
99	416k	if (uc == 160) {
100	194	uc = 32;
101	194	}
102	416k	}
103
104		/* end */
105	416k	ampStart = (size_t) -1;
106
107		/* success ? */
108	416k	if (uc > 0) {
109	362k	const size_t maxOut = max - ampStartDest;
110		/* write at position */
111	362k	if (charset != NULL && hts_isCharsetUTF8(charset)) {
112	182k	len = hts_writeUTF8(uc, &dest[ampStartDest], maxOut);
113	182k	} else {
114	179k	size_t ulen;
115	179k	char buffer[32];
116	179k	len = 0;
117	179k	if ( ( ulen = hts_writeUTF8(uc, buffer, sizeof(buffer)) ) != 0) {
118	179k	const hts_boolean urlQuery =
119	179k	(flags & UNESCAPE_ENTITIES_URL_QUERY) != 0;
120	179k	char *s;
121	179k	buffer[ulen] = '\0';
122		/* Strict for a query only: a substituted '?' must not pass for
123		the code point the document wrote. */
124	179k	if (urlQuery) {
125	0	s = hts_convertStringFromUTF8Strict(buffer, strlen(buffer),
126	0	charset);
127	179k	} else {
128	179k	s = hts_convertStringFromUTF8(buffer, strlen(buffer), charset);
129	179k	}
130	179k	if (s != NULL) {
131	16.6k	const size_t sLen = strlen(s);
132	16.6k	if (sLen < maxOut) {
133		/* Do not copy \0. */
134	16.6k	memcpy(&dest[ampStartDest], s, sLen);
135	16.6k	len = sLen;
136	16.6k	}
137	16.6k	freet(s);
138	163k	} else if (urlQuery) {
139		/* URL Standard: an unrepresentable code point is written
140		%26%23<decimal>%3B rather than left as source text. */
141	0	char esc[32];
142	0	const int escLen =
143	0	snprintf(esc, sizeof(esc), "%%26%%23%d%%3B", uc);
144
145	0	if (escLen > 0 && (size_t) escLen < maxOut) {
146	0	memcpy(&dest[ampStartDest], esc, (size_t) escLen);
147	0	len = (size_t) escLen;
148	0	}
149	0	}
150	179k	}
151	179k	}
152	362k	if (len > 0) {
153		/* new dest position */
154	199k	j = ampStartDest + len;
155		/* do not copy ; */
156	199k	continue;
157	199k	}
158	362k	}
159	416k	}
160		/* numerical entity */
161	2.36M	else if (uc != -1) {
162		/* decimal */
163	5.49k	if (!hex) {
164	2.72k	if (src[i] >= '0' && src[i] <= '9') {
165	2.16k	const int h = src[i] - '0';
166		/* Guard before multiplying: a codepoint past the Unicode max
167		(0x10FFFF) is invalid anyway, so stop rather than overflow uc. */
168	2.16k	if (uc > (0x10FFFF - h) / 10) {
169	232	ampStart = (size_t) -1;
170	1.93k	} else {
171	1.93k	uc = uc * 10 + h;
172	1.93k	}
173	2.16k	} else {
174		/* abandon */
175	552	ampStart = (size_t) -1;
176	552	}
177	2.72k	}
178		/* hex */
179	2.77k	else {
180	2.77k	const int h = hts_ehexh(src[i]);
181	2.77k	if (h != -1) {
182	2.30k	if (uc > (0x10FFFF - h) / 16) {
183	232	ampStart = (size_t) -1;
184	2.07k	} else {
185	2.07k	uc = uc * 16 + h;
186	2.07k	}
187	2.30k	} else {
188		/* abandon */
189	472	ampStart = (size_t) -1;
190	472	}
191	2.77k	}
192	5.49k	}
193		/* alphanumerical entity */
194	2.35M	else {
195		/* alphanum, capped at the longest name
196		* '&CounterClockwiseContourIntegral;' (31) */
197	2.35M	if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') \|\|
198	2.32M	(src[i] >= 'A' && src[i] <= 'Z') \|\|
199	2.34M	(src[i] >= 'a' && src[i] <= 'z'))) {
200		/* compute hash */
201	2.34M	HASH_ADD(hash, (unsigned char) src[i]);
202	2.34M	} else {
203		/* abandon */
204	12.6k	ampStart = (size_t) -1;
205	12.6k	}
206	2.35M	}
207	2.78M	}
208
209		/* reserve one byte for the trailing NUL written after the loop */
210	3.28M	if (j + 1 >= max) {
211		/* overflow */
212	4.29k	return -1;
213	4.29k	}
214	3.27M	if (src != dest \|\| i != j) {
215	3.27M	dest[j] = src[i];
216	3.27M	}
217	3.27M	j++;
218	3.27M	}
219	67.2k	dest[j] = '\0';
220
221	67.2k	return 0;
222	71.5k	}
223
224	35.7k	int hts_unescapeEntities(const char src, char dest, const size_t max) {
225	35.7k	return hts_unescapeEntitiesWithCharset(src, dest, max, "UTF-8");
226	35.7k	}
227
228		int hts_unescapeUrlSpecial(const char src, char dest, const size_t max,
229	0	const int flags) {
230	0	size_t i, j, lastI, lastJ, k, utfBufferJ, utfBufferSize;
231	0	int seenQuery = 0;
232	0	char utfBuffer[32];
233
234	0	assertf(src != dest);
235	0	assertf(max != 0);
236
237	0	for(i = 0, j = 0, k = 0, utfBufferJ = 0, utfBufferSize = 0,
238	0	lastI = (size_t) -1, lastJ = (size_t) -1
239	0	; src[i] != '\0' ; i++) {
240	0	char c = src[i];
241	0	unsigned char cUtf = (unsigned char) c;
242
243		/* Replacement for ' ' */
244	0	if (c == '+' && seenQuery) {
245	0	c = cUtf = ' ';
246	0	k = 0; /* cancel any sequence */
247	0	}
248		/* Escape sequence start */
249	0	else if (c == '%') {
250		/* last known position of % written on destination
251		copy blindly c, we'll rollback later */
252	0	lastI = i;
253	0	lastJ = j;
254	0	}
255		/* End of sequence seen */
256	0	else if (i >= 2 && i == lastI + 2) {
257	0	const int a1 = hts_ehexh(src[lastI + 1]);
258	0	const int a2 = hts_ehexh(src[lastI + 2]);
259	0	if (a1 != -1 && a2 != -1) {
260	0	const char ec = a116 + a2; / new character */
261	0	cUtf = (unsigned char) ec;
262
263		/* Shortcut for ASCII (do not unescape non-printable) */
264	0	if (
265	0	(cUtf < 0x80 && cUtf >= 32)
266	0	&& ( flags & UNESCAPE_URL_NO_ASCII ) == 0
267	0	) {
268		/* Rollback new write position and character */
269	0	j = lastJ;
270	0	c = ec;
271	0	}
272	0	} else {
273	0	k = 0; /* cancel any sequence */
274	0	}
275	0	}
276		/* ASCII (and not in %xx) */
277	0	else if (cUtf < 0x80 && i != lastI + 1) {
278	0	k = 0; /* cancel any sequence */
279	0	if (c == '?' && !seenQuery) {
280	0	seenQuery = 1;
281	0	}
282	0	}
283
284		/* UTF-8 sequence in progress (either a raw or a %xx character) */
285	0	if (cUtf >= 0x80) {
286		/* Leading UTF ? */
287	0	if (HTS_IS_LEADING_UTF8(cUtf)) {
288	0	k = 0; /* cancel any sequence */
289	0	}
290
291		/* Copy */
292	0	if (k < sizeof(utfBuffer)) {
293		/* First character */
294	0	if (k == 0) {
295		/* New destination-centric offset of utf-8 buffer beginning */
296	0	if (lastI != (size_t) -1 && i == lastI + 2) { /* just read a %xx */
297	0	utfBufferJ = lastJ; /* position of % */
298	0	} else {
299	0	utfBufferJ = j; /* current position otherwise */
300	0	}
301
302		/* Sequence length */
303	0	utfBufferSize = hts_getUTF8SequenceLength(cUtf);
304	0	}
305
306		/* Copy */
307	0	utfBuffer[k++] = cUtf;
308
309		/* Flush UTF-8 buffer when completed. */
310	0	if (k == utfBufferSize) {
311	0	const size_t nRead = hts_readUTF8(utfBuffer, utfBufferSize, NULL);
312
313		/* Reset UTF-8 buffer in all cases. */
314	0	k = 0;
315
316		/* Was the character read successfully ? */
317	0	if (nRead == utfBufferSize) {
318		/* the 'continue' below skips the NUL-reserve guard: re-check */
319	0	if (utfBufferJ + utfBufferSize >= max) {
320	0	return -1;
321	0	}
322
323		/* Rollback write position to sequence start write position */
324	0	j = utfBufferJ;
325
326		/* Copy full character sequence */
327	0	memcpy(&dest[j], utfBuffer, utfBufferSize);
328	0	j += utfBufferSize;
329
330		/* Skip current character */
331	0	continue;
332	0	}
333	0	}
334	0	}
335	0	}
336
337		/* reserve one byte for the trailing NUL written after the loop */
338	0	if (j + 1 >= max) {
339	0	return -1;
340	0	}
341
342		/* Copy current */
343	0	dest[j++] = c;
344	0	}
345	0	dest[j] = '\0';
346
347	0	return 0;
348	0	}
349
350	0	int hts_unescapeUrl(const char src, char dest, const size_t max) {
351	0	return hts_unescapeUrlSpecial(src, dest, max, 0);
352	0	}