/src/w3m/libwc/utf8.c

Source

#ifdef USE_UNICODE

#include "wc.h"
#include "ucs.h"
#include "utf8.h"
#include "wtf.h"

wc_uint8 WC_UTF8_MAP[ 0x100 ] = {
   8, 8, 8, 8, 8, 8, 8, 8,  8, 8, 8, 8, 8, 8, 8, 8,
   8, 8, 8, 8, 8, 8, 8, 8,  8, 8, 8, 8, 8, 8, 8, 8,
   1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 1,
   1, 1, 1, 1, 1, 1, 1, 1,  1, 1, 1, 1, 1, 1, 1, 8,

   0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
   0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
   0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
   0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0, 0,
   2, 2, 2, 2, 2, 2, 2, 2,  2, 2, 2, 2, 2, 2, 2, 2,
   2, 2, 2, 2, 2, 2, 2, 2,  2, 2, 2, 2, 2, 2, 2, 2,
   3, 3, 3, 3, 3, 3, 3, 3,  3, 3, 3, 3, 3, 3, 3, 3,
   4, 4, 4, 4, 4, 4, 4, 4,  5, 5, 5, 5, 6, 6, 7, 7,
};

static wc_uchar utf8_buf[7];

size_t
wc_ucs_to_utf8(wc_uint32 ucs, wc_uchar *utf8)
{
    if (ucs < WC_C_UTF8_L2) {
  utf8[0] =   ucs;
  utf8[1] = 0;
  return 1;
    } else if (ucs < WC_C_UTF8_L3) {
  utf8[0] =  (ucs >> 6)          | 0xc0;
  utf8[1] =  (ucs        & 0x3f) | 0x80;
  utf8[2] = 0;
  return 2;
    } else if (ucs < WC_C_UTF8_L4) {
  utf8[0] =  (ucs >> 12)         | 0xe0;
  utf8[1] = ((ucs >> 6)  & 0x3f) | 0x80;
  utf8[2] =  (ucs        & 0x3f) | 0x80;
  utf8[3] = 0;
  return 3;
    } else if (ucs < WC_C_UTF8_L5) {
  utf8[0] =  (ucs >> 18)         | 0xf0;
  utf8[1] = ((ucs >> 12) & 0x3f) | 0x80;
  utf8[2] = ((ucs >> 6)  & 0x3f) | 0x80;
  utf8[3] =  (ucs        & 0x3f) | 0x80;
  utf8[4] = 0;
  return 4;
    } else if (ucs < WC_C_UTF8_L6) {
  utf8[0] =  (ucs >> 24)         | 0xf8;
  utf8[1] = ((ucs >> 18) & 0x3f) | 0x80;
  utf8[2] = ((ucs >> 12) & 0x3f) | 0x80;
  utf8[3] = ((ucs >> 6)  & 0x3f) | 0x80;
  utf8[4] =  (ucs        & 0x3f) | 0x80;
  utf8[5] = 0;
  return 5;
    } else if (ucs <= WC_C_UCS4_END) {
  utf8[0] =  (ucs >> 30)         | 0xfc;
  utf8[1] = ((ucs >> 24) & 0x3f) | 0x80;
  utf8[2] = ((ucs >> 18) & 0x3f) | 0x80;
  utf8[3] = ((ucs >> 12) & 0x3f) | 0x80;
  utf8[4] = ((ucs >> 6)  & 0x3f) | 0x80;
  utf8[5] =  (ucs        & 0x3f) | 0x80;
  utf8[6] = 0;
  return 6;
    } else {
  utf8[0] = 0;
  return 0;
    }
}

wc_uint32
wc_utf8_to_ucs(wc_uchar *utf8)
{
    wc_uint32 ucs;

    switch (WC_UTF8_MAP[utf8[0]]) {
    case 1:
  ucs =  (wc_uint32) utf8[0];
  if (ucs >= WC_C_UTF8_L2)
      break;
  return ucs;
    case 2:
  ucs = ((wc_uint32)(utf8[0] & 0x1f) << 6)
      |  (wc_uint32)(utf8[1] & 0x3f);
  if (ucs < WC_C_UTF8_L2)
      break;
  return ucs;
    case 3:
  ucs = ((wc_uint32)(utf8[0] & 0x0f) << 12)
      | ((wc_uint32)(utf8[1] & 0x3f) << 6)
      |  (wc_uint32)(utf8[2] & 0x3f);
  if (ucs < WC_C_UTF8_L3)
      break;
  return ucs;
    case 4:
  ucs = ((wc_uint32)(utf8[0] & 0x07) << 18)
      | ((wc_uint32)(utf8[1] & 0x3f) << 12)
      | ((wc_uint32)(utf8[2] & 0x3f) << 6)
      |  (wc_uint32)(utf8[3] & 0x3f);
  if (ucs < WC_C_UTF8_L4)
      break;
  return ucs;
    case 5:
  ucs = ((wc_uint32)(utf8[0] & 0x03) << 24)
      | ((wc_uint32)(utf8[1] & 0x3f) << 18)
      | ((wc_uint32)(utf8[2] & 0x3f) << 12)
      | ((wc_uint32)(utf8[3] & 0x3f) << 6)
      |  (wc_uint32)(utf8[4] & 0x3f);
  if (ucs < WC_C_UTF8_L5)
      break;
  return ucs;
    case 6:
  ucs = ((wc_uint32)(utf8[0] & 0x01) << 30)
      | ((wc_uint32)(utf8[1] & 0x3f) << 24)
      | ((wc_uint32)(utf8[2] & 0x3f) << 18)
      | ((wc_uint32)(utf8[3] & 0x3f) << 12)
      | ((wc_uint32)(utf8[4] & 0x3f) << 6)
      |  (wc_uint32)(utf8[5] & 0x3f);
  if (ucs < WC_C_UTF8_L6)
      break;
  return ucs;
    default:
  break;
    }
    return WC_C_UCS4_ERROR;
}

Str
wc_conv_from_utf8(Str is, wc_ces ces)
{
    Str os;
    wc_uchar *sp = (wc_uchar *)is->ptr;
    wc_uchar *ep = sp + is->length;
    wc_uchar *p;
    wc_uchar *q = NULL;
    int state = WC_UTF8_NOSTATE;
    size_t next = 0;
    wc_uint32 ucs;
    wc_status st;

    for (p = sp; p < ep && *p < 0x80; p++)
  ;
    if (p == ep)
  return is;
    os = Strnew_size(is->length + is->length / 3);
    if (p > sp)
  Strcat_charp_n(os, is->ptr, (int)(p - sp));

    st.tag = NULL;
    st.ntag = 0;
    for (; p < ep; p++) {
  switch (state) {
  case WC_UTF8_NOSTATE:
      next = WC_UTF8_MAP[*p];
      switch (next) {
      case 1:
    wtf_push_ucs(os, (wc_uint32)*p, &st);
    break;
      case 8:
    Strcat_char(os, (char)*p);
    break;
      case 0:
      case 7:
    wtf_push_unknown(os, p, 1);
    break;
      default:
    q = p;
    next--;
    state = WC_UTF8_NEXT;
    break;
      }
      break;
  case WC_UTF8_NEXT:
      if (WC_UTF8_MAP[*p]) {
    wtf_push_unknown(os, q, p - q + 1);
    state = WC_UTF8_NOSTATE;
    break;
      }
      if (--next)
    break;
      state = WC_UTF8_NOSTATE;
      ucs = wc_utf8_to_ucs(q);
      if (ucs == WC_C_UCS4_ERROR ||
    (ucs >= WC_C_UCS2_SURROGATE && ucs <= WC_C_UCS2_SURROGATE_END))
    wtf_push_unknown(os, q, p - q + 1);
      else if (ucs != WC_C_UCS2_BOM)
    wtf_push_ucs(os, ucs, &st);
      break;
  }
    }
    switch (state) {
    case WC_UTF8_NEXT:
  wtf_push_unknown(os, q, p - q);
  break;
    }
    return os;
}

static int
wc_push_tag_to_utf8(Str os, int ntag)
{
    char *p;

    if (ntag) {
  p = wc_ucs_get_tag(ntag);
  if (p == NULL)
      ntag = 0;
    }
    if (ntag) {
  wc_ucs_to_utf8(WC_C_LANGUAGE_TAG, utf8_buf);
  Strcat_charp(os, (char *)utf8_buf);
  for (; *p; p++) {
      wc_ucs_to_utf8(WC_C_LANGUAGE_TAG0 | *p, utf8_buf);
      Strcat_charp(os, (char *)utf8_buf);
  }
    } else {
  wc_ucs_to_utf8(WC_C_CANCEL_TAG, utf8_buf);
  Strcat_charp(os, (char *)utf8_buf);
    }
    return ntag;
}

void
wc_push_to_utf8(Str os, wc_wchar_t cc, wc_status *st)
{
  while (1) {
    switch (WC_CCS_SET(cc.ccs)) {
    case WC_CCS_US_ASCII:
  if (st->ntag)
      st->ntag = wc_push_tag_to_utf8(os, 0);
  Strcat_char(os, (char)(cc.code & 0x7f));
  return;
    case WC_CCS_UCS2:
    case WC_CCS_UCS4:
  if (st->ntag)
      st->ntag = wc_push_tag_to_utf8(os, 0);
  wc_ucs_to_utf8(cc.code, utf8_buf);
  Strcat_charp(os, (char *)utf8_buf);
  return;
    case WC_CCS_UCS_TAG:
  if (WcOption.use_language_tag && wc_ucs_tag_to_tag(cc.code) != st->ntag)
      st->ntag = wc_push_tag_to_utf8(os, wc_ucs_tag_to_tag(cc.code));
  wc_ucs_to_utf8(wc_ucs_tag_to_ucs(cc.code), utf8_buf);
  Strcat_charp(os, (char *)utf8_buf);
  return;
    case WC_CCS_ISO_8859_1:
  if (st->ntag)
      st->ntag = wc_push_tag_to_utf8(os, 0);
  wc_ucs_to_utf8((cc.code | 0x80), utf8_buf);
  Strcat_charp(os, (char *)utf8_buf);
  return;
    case WC_CCS_UNKNOWN_W:
  if (!WcOption.no_replace) {
      if (st->ntag)
          st->ntag = wc_push_tag_to_utf8(os, 0);
      Strcat_charp(os, WC_REPLACE_W);
  }
  return;
    case WC_CCS_UNKNOWN:
  if (!WcOption.no_replace) {
      if (st->ntag)
          st->ntag = wc_push_tag_to_utf8(os, 0);
      Strcat_charp(os, WC_REPLACE);
  }
  return;
    default:
  if (WcOption.ucs_conv &&
    (cc.code = wc_any_to_ucs(cc)) != WC_C_UCS4_ERROR)
      cc.ccs = WC_CCS_UCS2;
  else
      cc.ccs = WC_CCS_IS_WIDE(cc.ccs) ? WC_CCS_UNKNOWN_W : WC_CCS_UNKNOWN;
  continue;
    }
  }
}

void
wc_push_to_utf8_end(Str os, wc_status *st)
{
    if (st->ntag)
  st->ntag = wc_push_tag_to_utf8(os, 0);
    return;
}

Str
wc_char_conv_from_utf8(wc_uchar c, wc_status *st)
{
    static Str os;
    static wc_uchar buf[6];
    static size_t nbuf, next;
    wc_uint32 ucs;

    if (st->state == -1) {
  st->state = WC_UTF8_NOSTATE;
  os = Strnew_size(8);
  st->tag = NULL;
  st->ntag = 0;
  nbuf = 0;
    }

    switch (st->state) {
    case WC_UTF8_NOSTATE:
  switch (next = WC_UTF8_MAP[c]) {
  case 1:
      wtf_push_ucs(os, (wc_uint32)c, st);
      break;
  case 8:
      Strcat_char(os, (char)c);
      break;
  case 0:
  case 7:
      break;
  default:
      buf[nbuf++] = c;
      next--;
      st->state = WC_UTF8_NEXT;
      return NULL;
  }
  break;
    case WC_UTF8_NEXT:
  if (WC_UTF8_MAP[c])
      break;
  buf[nbuf++] = c;
  if (--next)
      return NULL;
  ucs = wc_utf8_to_ucs(buf);
  if (ucs == WC_C_UCS4_ERROR ||
      (ucs >= WC_C_UCS2_SURROGATE && ucs <= WC_C_UCS2_SURROGATE_END))
      break;
  if (ucs != WC_C_UCS2_BOM)
      wtf_push_ucs(os, ucs, st);
  break;
    }
    st->state = -1;
    return os;
}

#endif

Coverage Report

Created: 2025-10-14 06:38

Line	Count	Source
1
2		#ifdef USE_UNICODE
3
4		#include "wc.h"
5		#include "ucs.h"
6		#include "utf8.h"
7		#include "wtf.h"
8
9		wc_uint8 WC_UTF8_MAP[ 0x100 ] = {
10		8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
11		8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8,
12		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
13		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
14		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
15		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
16		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
17		1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 8,
18
19		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
20		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
21		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
22		0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
23		2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
24		2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
25		3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
26		4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 6, 6, 7, 7,
27		};
28
29		static wc_uchar utf8_buf[7];
30
31		size_t
32		wc_ucs_to_utf8(wc_uint32 ucs, wc_uchar *utf8)
33	49.9M	{
34	49.9M	if (ucs < WC_C_UTF8_L2) {
35	2.75M	utf8[0] = ucs;
36	2.75M	utf8[1] = 0;
37	2.75M	return 1;
38	47.2M	} else if (ucs < WC_C_UTF8_L3) {
39	74.8k	utf8[0] = (ucs >> 6) \| 0xc0;
40	74.8k	utf8[1] = (ucs & 0x3f) \| 0x80;
41	74.8k	utf8[2] = 0;
42	74.8k	return 2;
43	47.1M	} else if (ucs < WC_C_UTF8_L4) {
44	134k	utf8[0] = (ucs >> 12) \| 0xe0;
45	134k	utf8[1] = ((ucs >> 6) & 0x3f) \| 0x80;
46	134k	utf8[2] = (ucs & 0x3f) \| 0x80;
47	134k	utf8[3] = 0;
48	134k	return 3;
49	47.0M	} else if (ucs < WC_C_UTF8_L5) {
50	47.0M	utf8[0] = (ucs >> 18) \| 0xf0;
51	47.0M	utf8[1] = ((ucs >> 12) & 0x3f) \| 0x80;
52	47.0M	utf8[2] = ((ucs >> 6) & 0x3f) \| 0x80;
53	47.0M	utf8[3] = (ucs & 0x3f) \| 0x80;
54	47.0M	utf8[4] = 0;
55	47.0M	return 4;
56	47.0M	} else if (ucs < WC_C_UTF8_L6) {
57	337	utf8[0] = (ucs >> 24) \| 0xf8;
58	337	utf8[1] = ((ucs >> 18) & 0x3f) \| 0x80;
59	337	utf8[2] = ((ucs >> 12) & 0x3f) \| 0x80;
60	337	utf8[3] = ((ucs >> 6) & 0x3f) \| 0x80;
61	337	utf8[4] = (ucs & 0x3f) \| 0x80;
62	337	utf8[5] = 0;
63	337	return 5;
64	1.41k	} else if (ucs <= WC_C_UCS4_END) {
65	906	utf8[0] = (ucs >> 30) \| 0xfc;
66	906	utf8[1] = ((ucs >> 24) & 0x3f) \| 0x80;
67	906	utf8[2] = ((ucs >> 18) & 0x3f) \| 0x80;
68	906	utf8[3] = ((ucs >> 12) & 0x3f) \| 0x80;
69	906	utf8[4] = ((ucs >> 6) & 0x3f) \| 0x80;
70	906	utf8[5] = (ucs & 0x3f) \| 0x80;
71	906	utf8[6] = 0;
72	906	return 6;
73	906	} else {
74	506	utf8[0] = 0;
75	506	return 0;
76	506	}
77	49.9M	}
78
79		wc_uint32
80		wc_utf8_to_ucs(wc_uchar *utf8)
81	127k	{
82	127k	wc_uint32 ucs;
83
84	127k	switch (WC_UTF8_MAP[utf8[0]]) {
85	0	case 1:
86	0	ucs = (wc_uint32) utf8[0];
87	0	if (ucs >= WC_C_UTF8_L2)
88	0	break;
89	0	return ucs;
90	65.4k	case 2:
91	65.4k	ucs = ((wc_uint32)(utf8[0] & 0x1f) << 6)
92	65.4k	\| (wc_uint32)(utf8[1] & 0x3f);
93	65.4k	if (ucs < WC_C_UTF8_L2)
94	8.44k	break;
95	56.9k	return ucs;
96	11.0k	case 3:
97	11.0k	ucs = ((wc_uint32)(utf8[0] & 0x0f) << 12)
98	11.0k	\| ((wc_uint32)(utf8[1] & 0x3f) << 6)
99	11.0k	\| (wc_uint32)(utf8[2] & 0x3f);
100	11.0k	if (ucs < WC_C_UTF8_L3)
101	662	break;
102	10.3k	return ucs;
103	48.1k	case 4:
104	48.1k	ucs = ((wc_uint32)(utf8[0] & 0x07) << 18)
105	48.1k	\| ((wc_uint32)(utf8[1] & 0x3f) << 12)
106	48.1k	\| ((wc_uint32)(utf8[2] & 0x3f) << 6)
107	48.1k	\| (wc_uint32)(utf8[3] & 0x3f);
108	48.1k	if (ucs < WC_C_UTF8_L4)
109	206	break;
110	47.9k	return ucs;
111	1.07k	case 5:
112	1.07k	ucs = ((wc_uint32)(utf8[0] & 0x03) << 24)
113	1.07k	\| ((wc_uint32)(utf8[1] & 0x3f) << 18)
114	1.07k	\| ((wc_uint32)(utf8[2] & 0x3f) << 12)
115	1.07k	\| ((wc_uint32)(utf8[3] & 0x3f) << 6)
116	1.07k	\| (wc_uint32)(utf8[4] & 0x3f);
117	1.07k	if (ucs < WC_C_UTF8_L5)
118	208	break;
119	867	return ucs;
120	2.16k	case 6:
121	2.16k	ucs = ((wc_uint32)(utf8[0] & 0x01) << 30)
122	2.16k	\| ((wc_uint32)(utf8[1] & 0x3f) << 24)
123	2.16k	\| ((wc_uint32)(utf8[2] & 0x3f) << 18)
124	2.16k	\| ((wc_uint32)(utf8[3] & 0x3f) << 12)
125	2.16k	\| ((wc_uint32)(utf8[4] & 0x3f) << 6)
126	2.16k	\| (wc_uint32)(utf8[5] & 0x3f);
127	2.16k	if (ucs < WC_C_UTF8_L6)
128	200	break;
129	1.96k	return ucs;
130	0	default:
131	0	break;
132	127k	}
133	9.72k	return WC_C_UCS4_ERROR;
134	127k	}
135
136		Str
137		wc_conv_from_utf8(Str is, wc_ces ces)
138	1.76k	{
139	1.76k	Str os;
140	1.76k	wc_uchar sp = (wc_uchar )is->ptr;
141	1.76k	wc_uchar *ep = sp + is->length;
142	1.76k	wc_uchar *p;
143	1.76k	wc_uchar *q = NULL;
144	1.76k	int state = WC_UTF8_NOSTATE;
145	1.76k	size_t next = 0;
146	1.76k	wc_uint32 ucs;
147	1.76k	wc_status st;
148
149	4.46k	for (p = sp; p < ep && *p < 0x80; p++)
150	2.70k	;
151	1.76k	if (p == ep)
152	82	return is;
153	1.68k	os = Strnew_size(is->length + is->length / 3);
154	1.68k	if (p > sp)
155	348	Strcat_charp_n(os, is->ptr, (int)(p - sp));
156
157	1.68k	st.tag = NULL;
158	1.68k	st.ntag = 0;
159	59.8M	for (; p < ep; p++) {
160	59.8M	switch (state) {
161	58.9M	case WC_UTF8_NOSTATE:
162	58.9M	next = WC_UTF8_MAP[*p];
163	58.9M	switch (next) {
164	35.6M	case 1:
165	35.6M	wtf_push_ucs(os, (wc_uint32)*p, &st);
166	35.6M	break;
167	3.38M	case 8:
168	3.38M	Strcat_char(os, (char)*p);
169	3.38M	break;
170	18.9M	case 0:
171	19.1M	case 7:
172	19.1M	wtf_push_unknown(os, p, 1);
173	19.1M	break;
174	742k	default:
175	742k	q = p;
176	742k	next--;
177	742k	state = WC_UTF8_NEXT;
178	742k	break;
179	58.9M	}
180	58.9M	break;
181	58.9M	case WC_UTF8_NEXT:
182	928k	if (WC_UTF8_MAP[*p]) {
183	614k	wtf_push_unknown(os, q, p - q + 1);
184	614k	state = WC_UTF8_NOSTATE;
185	614k	break;
186	614k	}
187	314k	if (--next)
188	186k	break;
189	127k	state = WC_UTF8_NOSTATE;
190	127k	ucs = wc_utf8_to_ucs(q);
191	127k	if (ucs == WC_C_UCS4_ERROR \|\|
192	118k	(ucs >= WC_C_UCS2_SURROGATE && ucs <= WC_C_UCS2_SURROGATE_END))
193	10.1k	wtf_push_unknown(os, q, p - q + 1);
194	117k	else if (ucs != WC_C_UCS2_BOM)
195	117k	wtf_push_ucs(os, ucs, &st);
196	127k	break;
197	59.8M	}
198	59.8M	}
199	1.68k	switch (state) {
200	50	case WC_UTF8_NEXT:
201	50	wtf_push_unknown(os, q, p - q);
202	50	break;
203	1.68k	}
204	1.68k	return os;
205	1.68k	}
206
207		static int
208		wc_push_tag_to_utf8(Str os, int ntag)
209	4.95M	{
210	4.95M	char *p;
211
212	4.95M	if (ntag) {
213	2.47M	p = wc_ucs_get_tag(ntag);
214	2.47M	if (p == NULL)
215	152	ntag = 0;
216	2.47M	}
217	4.95M	if (ntag) {
218	2.47M	wc_ucs_to_utf8(WC_C_LANGUAGE_TAG, utf8_buf);
219	2.47M	Strcat_charp(os, (char *)utf8_buf);
220	44.5M	for (; *p; p++) {
221	42.0M	wc_ucs_to_utf8(WC_C_LANGUAGE_TAG0 \| *p, utf8_buf);
222	42.0M	Strcat_charp(os, (char *)utf8_buf);
223	42.0M	}
224	2.47M	} else {
225	2.47M	wc_ucs_to_utf8(WC_C_CANCEL_TAG, utf8_buf);
226	2.47M	Strcat_charp(os, (char *)utf8_buf);
227	2.47M	}
228	4.95M	return ntag;
229	4.95M	}
230
231		void
232		wc_push_to_utf8(Str os, wc_wchar_t cc, wc_status *st)
233	5.68M	{
234	5.89M	while (1) {
235	5.89M	switch (WC_CCS_SET(cc.ccs)) {
236	1.02M	case WC_CCS_US_ASCII:
237	1.02M	if (st->ntag)
238	921k	st->ntag = wc_push_tag_to_utf8(os, 0);
239	1.02M	Strcat_char(os, (char)(cc.code & 0x7f));
240	1.02M	return;
241	208k	case WC_CCS_UCS2:
242	210k	case WC_CCS_UCS4:
243	210k	if (st->ntag)
244	225	st->ntag = wc_push_tag_to_utf8(os, 0);
245	210k	wc_ucs_to_utf8(cc.code, utf8_buf);
246	210k	Strcat_charp(os, (char *)utf8_buf);
247	210k	return;
248	2.76M	case WC_CCS_UCS_TAG:
249	2.76M	if (WcOption.use_language_tag && wc_ucs_tag_to_tag(cc.code) != st->ntag)
250	2.47M	st->ntag = wc_push_tag_to_utf8(os, wc_ucs_tag_to_tag(cc.code));
251	2.76M	wc_ucs_to_utf8(wc_ucs_tag_to_ucs(cc.code), utf8_buf);
252	2.76M	Strcat_charp(os, (char *)utf8_buf);
253	2.76M	return;
254	394	case WC_CCS_ISO_8859_1:
255	394	if (st->ntag)
256	195	st->ntag = wc_push_tag_to_utf8(os, 0);
257	394	wc_ucs_to_utf8((cc.code \| 0x80), utf8_buf);
258	394	Strcat_charp(os, (char *)utf8_buf);
259	394	return;
260	647	case WC_CCS_UNKNOWN_W:
261	647	if (!WcOption.no_replace) {
262	647	if (st->ntag)
263	198	st->ntag = wc_push_tag_to_utf8(os, 0);
264	647	Strcat_charp(os, WC_REPLACE_W);
265	647	}
266	647	return;
267	1.69M	case WC_CCS_UNKNOWN:
268	1.69M	if (!WcOption.no_replace) {
269	1.69M	if (st->ntag)
270	1.55M	st->ntag = wc_push_tag_to_utf8(os, 0);
271	1.69M	Strcat_charp(os, WC_REPLACE);
272	1.69M	}
273	1.69M	return;
274	210k	default:
275	210k	if (WcOption.ucs_conv &&
276	210k	(cc.code = wc_any_to_ucs(cc)) != WC_C_UCS4_ERROR)
277	208k	cc.ccs = WC_CCS_UCS2;
278	1.84k	else
279	1.84k	cc.ccs = WC_CCS_IS_WIDE(cc.ccs) ? WC_CCS_UNKNOWN_W : WC_CCS_UNKNOWN;
280	210k	continue;
281	5.89M	}
282	5.89M	}
283	5.68M	}
284
285		void
286		wc_push_to_utf8_end(Str os, wc_status *st)
287	475	{
288	475	if (st->ntag)
289	86	st->ntag = wc_push_tag_to_utf8(os, 0);
290	475	return;
291	475	}
292
293		Str
294		wc_char_conv_from_utf8(wc_uchar c, wc_status *st)
295	0	{
296	0	static Str os;
297	0	static wc_uchar buf[6];
298	0	static size_t nbuf, next;
299	0	wc_uint32 ucs;
300
301	0	if (st->state == -1) {
302	0	st->state = WC_UTF8_NOSTATE;
303	0	os = Strnew_size(8);
304	0	st->tag = NULL;
305	0	st->ntag = 0;
306	0	nbuf = 0;
307	0	}
308
309	0	switch (st->state) {
310	0	case WC_UTF8_NOSTATE:
311	0	switch (next = WC_UTF8_MAP[c]) {
312	0	case 1:
313	0	wtf_push_ucs(os, (wc_uint32)c, st);
314	0	break;
315	0	case 8:
316	0	Strcat_char(os, (char)c);
317	0	break;
318	0	case 0:
319	0	case 7:
320	0	break;
321	0	default:
322	0	buf[nbuf++] = c;
323	0	next--;
324	0	st->state = WC_UTF8_NEXT;
325	0	return NULL;
326	0	}
327	0	break;
328	0	case WC_UTF8_NEXT:
329	0	if (WC_UTF8_MAP[c])
330	0	break;
331	0	buf[nbuf++] = c;
332	0	if (--next)
333	0	return NULL;
334	0	ucs = wc_utf8_to_ucs(buf);
335	0	if (ucs == WC_C_UCS4_ERROR \|\|
336	0	(ucs >= WC_C_UCS2_SURROGATE && ucs <= WC_C_UCS2_SURROGATE_END))
337	0	break;
338	0	if (ucs != WC_C_UCS2_BOM)
339	0	wtf_push_ucs(os, ucs, st);
340	0	break;
341	0	}
342	0	st->state = -1;
343	0	return os;
344	0	}
345
346		#endif