/**

 * @file

 * Conversion between different character encodings

 *

 * @authors

 * Copyright (C) 2017 Tobias Angele <toogley@mailbox.org>

 * Copyright (C) 2017-2023 Richard Russon <rich@flatcap.org>

 * Copyright (C) 2018-2023 Pietro Cerutti <gahr@gahr.ch>

 * Copyright (C) 2023 Steinar H Gunderson <steinar+neomutt@gunderson.no>

 *

 * @copyright

 * This program is free software: you can redistribute it and/or modify it under

 * the terms of the GNU General Public License as published by the Free Software

 * Foundation, either version 2 of the License, or (at your option) any later

 * version.

 *

 * This program is distributed in the hope that it will be useful, but WITHOUT

 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS

 * FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more

 * details.

 *

 * You should have received a copy of the GNU General Public License along with

 * this program.  If not, see <http://www.gnu.org/licenses/>.

 */

/**

 * @page mutt_charset Conversion between different character encodings

 *

 * Conversion between different character encodings

 */

#include "config.h"

#include <errno.h>

#include <iconv.h>

#include <langinfo.h>

#include <limits.h>

#include <stdbool.h>

#include <stdio.h>

#include <string.h>

#include "charset.h"

#include "buffer.h"

#include "list.h"

#include "logging2.h"

#include "memory.h"

#include "pool.h"

#include "queue.h"

#include "regex3.h"

#include "slist.h"

#include "string2.h"

#ifdef ENABLE_NLS

#include <libintl.h>

#endif

#ifndef EILSEQ

#define EILSEQ EINVAL

#endif

/**

 * ReplacementChar - When a Unicode character can't be displayed, use this instead

 */

wchar_t ReplacementChar = '?';

/**

 * CharsetIsUtf8 - Is the user's current character set utf-8?

 */

bool CharsetIsUtf8 = false;

/**

 * struct Lookup - Regex to String lookup table

 *

 * This is used by 'charset-hook' and 'iconv-hook'.

 */

struct Lookup

{

  enum LookupType type;        ///< Lookup type

  struct Regex regex;          ///< Regular expression

  char *replacement;           ///< Alternative charset to use

  TAILQ_ENTRY(Lookup) entries; ///< Linked list

};

TAILQ_HEAD(LookupList, Lookup);

/// Lookup table of preferred character set names

static struct LookupList Lookups = TAILQ_HEAD_INITIALIZER(Lookups);

/**

 * struct IconvCacheEntry - Cached iconv conversion descriptor

 */

struct IconvCacheEntry

{

  char *fromcode1; ///< Source character set

  char *tocode1;   ///< Destination character set

  iconv_t cd;      ///< iconv conversion descriptor

};

/// Max size of the iconv cache

#define ICONV_CACHE_SIZE 16

/// Cache of iconv conversion descriptors

static struct IconvCacheEntry IconvCache[ICONV_CACHE_SIZE];

/// Number of iconv descriptors in the cache

static int IconvCacheUsed = 0;

/**

 * struct MimeNames - MIME name lookup entry

 */

struct MimeNames

{

  const char *key;

  const char *pref;

};

/**

 * PreferredMimeNames - Lookup table of preferred charsets

 *

 * The following list has been created manually from the data under:

 * http://www.isi.edu/in-notes/iana/assignments/character-sets

 * Last update: 2000-09-07

 *

 * @note It includes only the subset of character sets for which a preferred

 * MIME name is given.

 */

static const struct MimeNames PreferredMimeNames[] = {

  // clang-format off

  { "ansi_x3.4-1968",        "us-ascii"      },

  { "iso-ir-6",              "us-ascii"      },

  { "iso_646.irv:1991",      "us-ascii"      },

  { "ascii",                 "us-ascii"      },

  { "iso646-us",             "us-ascii"      },

  { "us",                    "us-ascii"      },

  { "ibm367",                "us-ascii"      },

  { "cp367",                 "us-ascii"      },

  { "csASCII",               "us-ascii"      },

  { "csISO2022KR",           "iso-2022-kr"   },

  { "csEUCKR",               "euc-kr"        },

  { "csISO2022JP",           "iso-2022-jp"   },

  { "csISO2022JP2",          "iso-2022-jp-2" },

  { "ISO_8859-1:1987",       "iso-8859-1"    },

  { "iso-ir-100",            "iso-8859-1"    },

  { "iso_8859-1",            "iso-8859-1"    },

  { "latin1",                "iso-8859-1"    },

  { "l1",                    "iso-8859-1"    },

  { "IBM819",                "iso-8859-1"    },

  { "CP819",                 "iso-8859-1"    },

  { "csISOLatin1",           "iso-8859-1"    },

  { "ISO_8859-2:1987",       "iso-8859-2"    },

  { "iso-ir-101",            "iso-8859-2"    },

  { "iso_8859-2",            "iso-8859-2"    },

  { "latin2",                "iso-8859-2"    },

  { "l2",                    "iso-8859-2"    },

  { "csISOLatin2",           "iso-8859-2"    },

  { "ISO_8859-3:1988",       "iso-8859-3"    },

  { "iso-ir-109",            "iso-8859-3"    },

  { "ISO_8859-3",            "iso-8859-3"    },

  { "latin3",                "iso-8859-3"    },

  { "l3",                    "iso-8859-3"    },

  { "csISOLatin3",           "iso-8859-3"    },

  { "ISO_8859-4:1988",       "iso-8859-4"    },

  { "iso-ir-110",            "iso-8859-4"    },

  { "ISO_8859-4",            "iso-8859-4"    },

  { "latin4",                "iso-8859-4"    },

  { "l4",                    "iso-8859-4"    },

  { "csISOLatin4",           "iso-8859-4"    },

  { "ISO_8859-6:1987",       "iso-8859-6"    },

  { "iso-ir-127",            "iso-8859-6"    },

  { "iso_8859-6",            "iso-8859-6"    },

  { "ECMA-114",              "iso-8859-6"    },

  { "ASMO-708",              "iso-8859-6"    },

  { "arabic",                "iso-8859-6"    },

  { "csISOLatinArabic",      "iso-8859-6"    },

  { "ISO_8859-7:1987",       "iso-8859-7"    },

  { "iso-ir-126",            "iso-8859-7"    },

  { "ISO_8859-7",            "iso-8859-7"    },

  { "ELOT_928",              "iso-8859-7"    },

  { "ECMA-118",              "iso-8859-7"    },

  { "greek",                 "iso-8859-7"    },

  { "greek8",                "iso-8859-7"    },

  { "csISOLatinGreek",       "iso-8859-7"    },

  { "ISO_8859-8:1988",       "iso-8859-8"    },

  { "iso-ir-138",            "iso-8859-8"    },

  { "ISO_8859-8",            "iso-8859-8"    },

  { "hebrew",                "iso-8859-8"    },

  { "csISOLatinHebrew",      "iso-8859-8"    },

  { "ISO_8859-5:1988",       "iso-8859-5"    },

  { "iso-ir-144",            "iso-8859-5"    },

  { "ISO_8859-5",            "iso-8859-5"    },

  { "cyrillic",              "iso-8859-5"    },

  { "csISOLatinCyrillic",    "iso-8859-5"    },

  { "ISO_8859-9:1989",       "iso-8859-9"    },

  { "iso-ir-148",            "iso-8859-9"    },

  { "ISO_8859-9",            "iso-8859-9"    },

  { "latin5",                "iso-8859-9"    },  /* this is not a bug */

  { "l5",                    "iso-8859-9"    },

  { "csISOLatin5",           "iso-8859-9"    },

  { "ISO_8859-10:1992",      "iso-8859-10"   },

  { "iso-ir-157",            "iso-8859-10"   },

  { "latin6",                "iso-8859-10"   },  /* this is not a bug */

  { "l6",                    "iso-8859-10"   },

  { "csISOLatin6",           "iso-8859-10"   },

  { "csKOI8r",               "koi8-r"        },

  { "MS_Kanji",              "Shift_JIS"     },  /* Note the underscore! */

  { "csShiftJis",            "Shift_JIS"     },

  { "Extended_UNIX_Code_Packed_Format_for_Japanese",

                             "euc-jp"        },

  { "csEUCPkdFmtJapanese",   "euc-jp"        },

  { "csGB2312",              "gb2312"        },

  { "csbig5",                "big5"          },

  /* End of official brain damage.

   * What follows has been taken from glibc's localedata files.  */

  { "iso_8859-13",           "iso-8859-13"   },

  { "iso-ir-179",            "iso-8859-13"   },

  { "latin7",                "iso-8859-13"   },  /* this is not a bug */

  { "l7",                    "iso-8859-13"   },

  { "iso_8859-14",           "iso-8859-14"   },

  { "latin8",                "iso-8859-14"   },  /* this is not a bug */

  { "l8",                    "iso-8859-14"   },

  { "iso_8859-15",           "iso-8859-15"   },

  { "latin9",                "iso-8859-15"   },  /* this is not a bug */

  /* Suggested by Ionel Mugurel Ciobica <tgakic@sg10.chem.tue.nl> */

  { "latin0",                "iso-8859-15"   },  /* this is not a bug */

  { "iso_8859-16",           "iso-8859-16"   },

  { "latin10",               "iso-8859-16"   },  /* this is not a bug */

  { "646",                   "us-ascii"      },

  /* http://www.sun.com/software/white-papers/wp-unicode/ */

  { "eucJP",                 "euc-jp"        },

  { "PCK",                   "Shift_JIS"     },

  { "ko_KR-euc",             "euc-kr"        },

  { "zh_TW-big5",            "big5"          },

  /* seems to be common on some systems */

  { "sjis",                  "Shift_JIS"     },

  { "euc-jp-ms",             "eucJP-ms"      },

  /* If you happen to encounter system-specific brain-damage with respect to

   * character set naming, please add it above this comment, and submit a patch

   * to <neomutt-devel@neomutt.org> */

  { NULL, NULL },

  // clang-format on

};

/**

 * lookup_new - Create a new Lookup

 * @retval ptr New Lookup

 */

static struct Lookup *lookup_new(void)

{

  return MUTT_MEM_CALLOC(1, struct Lookup);

}

/**

 * lookup_free - Free a Lookup

 * @param ptr Lookup to free

 */

static void lookup_free(struct Lookup **ptr)

{

  if (!ptr || !*ptr)

    return;

  struct Lookup *l = *ptr;

  FREE(&l->replacement);

  FREE(&l->regex.pattern);

  if (l->regex.regex)

    regfree(l->regex.regex);

  FREE(&l->regex.regex);

  FREE(&l->regex);

  FREE(ptr);

}

/**

 * lookup_charset - Look for a preferred character set name

 * @param type Type, e.g. #MUTT_LOOKUP_CHARSET

 * @param cs   Character set

 * @retval ptr Charset string

 *

 * If the character set matches one of the regexes,

 * then return the replacement name.

 */

static const char *lookup_charset(enum LookupType type, const char *cs)

{

  if (!cs)

    return NULL;

  struct Lookup *l = NULL;

  TAILQ_FOREACH(l, &Lookups, entries)

  {

    if (l->type != type)

      continue;

    if (mutt_regex_match(&l->regex, cs))

      return l->replacement;

  }

  return NULL;

}

/**

 * mutt_ch_convert_nonmime_string - Try to convert a string using a list of character sets

 * @param[in]     assumed_charset From $assumed_charset

 * @param[in]     charset         From $charset

 * @param[in,out] ps              String to be converted

 * @retval 0  Success

 * @retval -1 Error

 *

 * Work through `$assumed_charset` looking for a character set conversion that

 * works.  Failing that, try mutt_ch_get_default_charset().

 */

int mutt_ch_convert_nonmime_string(const struct Slist *const assumed_charset,

                                   const char *charset, char **ps)

{

  if (!ps)

    return -1;

  char *u = *ps;

  const size_t ulen = mutt_str_len(u);

  if (ulen == 0)

    return 0;

  const struct ListNode *np = NULL;

  STAILQ_FOREACH(np, &assumed_charset->head, entries)

  {

    char const *c = np->data;

    size_t n = mutt_str_len(c);

    char *fromcode = MUTT_MEM_MALLOC(n + 1, char);

    mutt_str_copy(fromcode, c, n + 1);

    char *s = mutt_strn_dup(u, ulen);

    int m = mutt_ch_convert_string(&s, fromcode, charset, MUTT_ICONV_NO_FLAGS);

    FREE(&fromcode);

    if (m == 0)

    {

      FREE(ps);

      *ps = s;

      return 0;

    }

    FREE(&s);

  }

  mutt_ch_convert_string(ps, mutt_ch_get_default_charset(assumed_charset),

                         charset, MUTT_ICONV_HOOK_FROM);

  return -1;

}

/**

 * mutt_ch_canonical_charset - Canonicalise the charset of a string

 * @param buf Buffer for canonical character set name

 * @param buflen Length of buffer

 * @param name Name to be canonicalised

 *

 * This first ties off any charset extension such as "//TRANSLIT",

 * canonicalizes the charset and re-adds the extension

 */

void mutt_ch_canonical_charset(char *buf, size_t buflen, const char *name)

{

  if (!buf || !name)

    return;

  char in[1024] = { 0 };

  char scratch[1024 + 10] = { 0 };

  struct Buffer *canon = buf_pool_get();

  mutt_str_copy(in, name, sizeof(in));

  char *ext = strchr(in, '/');

  if (ext)

    *ext++ = '\0';

  if (mutt_istr_equal(in, "utf-8") || mutt_istr_equal(in, "utf8"))

  {

    buf_strcpy(canon, "utf-8");

    goto out;

  }

  /* catch some common iso-8859-something misspellings */

  size_t plen;

  if ((plen = mutt_istr_startswith(in, "8859")) && (in[plen] != '-'))

    snprintf(scratch, sizeof(scratch), "iso-8859-%s", in + plen);

  else if ((plen = mutt_istr_startswith(in, "8859-")))

    snprintf(scratch, sizeof(scratch), "iso-8859-%s", in + plen);

  else if ((plen = mutt_istr_startswith(in, "iso8859")) && (in[plen] != '-'))

    snprintf(scratch, sizeof(scratch), "iso_8859-%s", in + plen);

  else if ((plen = mutt_istr_startswith(in, "iso8859-")))

    snprintf(scratch, sizeof(scratch), "iso_8859-%s", in + plen);

  else

    mutt_str_copy(scratch, in, sizeof(scratch));

  for (size_t i = 0; PreferredMimeNames[i].key; i++)

  {

    if (mutt_istr_equal(scratch, PreferredMimeNames[i].key))

    {

      buf_strcpy(canon, PreferredMimeNames[i].pref);

      goto out;

    }

  }

  buf_strcpy(canon, scratch);

  buf_lower(canon); // for cosmetics' sake

out:

  if (ext && (*ext != '\0'))

  {

    buf_addch(canon, '/');

    buf_addstr(canon, ext);

  }

  mutt_str_copy(buf, buf_string(canon), buflen);

  buf_pool_release(&canon);

}

/**

 * mutt_ch_chscmp - Are the names of two character sets equivalent?

 * @param cs1 First character set

 * @param cs2 Second character set

 * @retval true  Names are equivalent

 * @retval false Names differ

 *

 * Charsets may have extensions that mutt_ch_canonical_charset() leaves intact;

 * we expect 'cs2' to originate from neomutt code, not user input (i.e. 'cs2'

 * does _not_ have any extension) we simply check if the shorter string is a

 * prefix for the longer.

 */

bool mutt_ch_chscmp(const char *cs1, const char *cs2)

{

  if (!cs1 || !cs2)

    return false;

  char buf[256] = { 0 };

  mutt_ch_canonical_charset(buf, sizeof(buf), cs1);

  int len1 = mutt_str_len(buf);

  int len2 = mutt_str_len(cs2);

  return mutt_istrn_equal(((len1 > len2) ? buf : cs2),

                          ((len1 > len2) ? cs2 : buf), MIN(len1, len2));

}

/**

 * mutt_ch_get_default_charset - Get the default character set

 * @param assumed_charset From $assumed_charset

 * @retval ptr Name of the default character set

 *

 * @warning This returns a pointer to a static buffer.  Do not free it.

 */

const char *mutt_ch_get_default_charset(const struct Slist *const assumed_charset)

{

  static char fcharset[128];

  const char *c = NULL;

  if (assumed_charset && (assumed_charset->count > 0))

    c = STAILQ_FIRST(&assumed_charset->head)->data;

  else

    c = "us-ascii";

  mutt_str_copy(fcharset, c, sizeof(fcharset));

  return fcharset;

}

/**

 * mutt_ch_get_langinfo_charset - Get the user's choice of character set

 * @retval ptr Charset string

 *

 * Get the canonical character set used by the user's locale.

 * The caller must free the returned string.

 */

char *mutt_ch_get_langinfo_charset(void)

{

  char buf[1024] = { 0 };

  mutt_ch_canonical_charset(buf, sizeof(buf), nl_langinfo(CODESET));

  if (buf[0] != '\0')

    return mutt_str_dup(buf);

  return mutt_str_dup("iso-8859-1");

}

/**

 * mutt_ch_lookup_add - Add a new character set lookup

 * @param type    Type of character set, e.g. #MUTT_LOOKUP_CHARSET

 * @param pat     Pattern to match

 * @param replace Replacement string

 * @param err     Buffer for error message

 * @retval true  Lookup added to list

 * @retval false Regex string was invalid

 *

 * Add a regex for a character set and a replacement name.

 */

bool mutt_ch_lookup_add(enum LookupType type, const char *pat,

                        const char *replace, struct Buffer *err)

{

  if (!pat || !replace)

    return false;

  regex_t *rx = MUTT_MEM_CALLOC(1, regex_t);

  int rc = REG_COMP(rx, pat, REG_ICASE);

  if (rc != 0)

  {

    regerror(rc, rx, err->data, err->dsize);

    FREE(&rx);

    return false;

  }

  struct Lookup *l = lookup_new();

  l->type = type;

  l->replacement = mutt_str_dup(replace);

  l->regex.pattern = mutt_str_dup(pat);

  l->regex.regex = rx;

  l->regex.pat_not = false;

  TAILQ_INSERT_TAIL(&Lookups, l, entries);

  return true;

}

/**

 * mutt_ch_lookup_remove - Remove all the character set lookups

 *

 * Empty the list of replacement character set names.

 */

void mutt_ch_lookup_remove(void)

{

  struct Lookup *l = NULL;

  struct Lookup *tmp = NULL;

  TAILQ_FOREACH_SAFE(l, &Lookups, entries, tmp)

  {

    TAILQ_REMOVE(&Lookups, l, entries);

    lookup_free(&l);

  }

}

/**

 * mutt_ch_charset_lookup - Look for a replacement character set

 * @param chs Character set to lookup

 * @retval ptr  Replacement character set (if a 'charset-hook' matches)

 * @retval NULL No matching hook

 *

 * Look through all the 'charset-hook's.

 * If one matches return the replacement character set.

 */

const char *mutt_ch_charset_lookup(const char *chs)

{

  return lookup_charset(MUTT_LOOKUP_CHARSET, chs);

}

/**

 * mutt_ch_iconv_open - Set up iconv for conversions

 * @param tocode   Current character set

 * @param fromcode Target character set

 * @param flags    Flags, e.g. #MUTT_ICONV_HOOK_FROM

 * @retval ptr iconv handle for the conversion

 *

 * Like iconv_open, but canonicalises the charsets, applies charset-hooks,

 * recanonicalises, and finally applies iconv-hooks. Parameter flags=0 skips

 * charset-hooks, while MUTT_ICONV_HOOK_FROM applies them to fromcode. Callers

 * should use flags=0 when fromcode can safely be considered true, either some

 * constant, or some value provided by the user; MUTT_ICONV_HOOK_FROM should be

 * used only when fromcode is unsure, taken from a possibly wrong incoming MIME

 * label, or such. Misusing MUTT_ICONV_HOOK_FROM leads to unwanted interactions

 * in some setups.

 *

 * Since calling iconv_open() repeatedly can be expensive, we keep a cache of

 * the most recently used iconv_t objects, kept in LRU order. This means that

 * you should not call iconv_close() on the object yourself. All remaining

 * objects in the cache will exit when main() calls mutt_ch_cache_cleanup().

 *

 * @note By design charset-hooks should never be, and are never, applied

 * to tocode.

 *

 * @note The top-well-named MUTT_ICONV_HOOK_FROM acts on charset-hooks,

 * not at all on iconv-hooks.

 */

iconv_t mutt_ch_iconv_open(const char *tocode, const char *fromcode, uint8_t flags)

{

  char tocode1[128] = { 0 };

  char fromcode1[128] = { 0 };

  const char *tocode2 = NULL, *fromcode2 = NULL;

  const char *tmp = NULL;

  /* transform to MIME preferred charset names */

  mutt_ch_canonical_charset(tocode1, sizeof(tocode1), tocode);

  mutt_ch_canonical_charset(fromcode1, sizeof(fromcode1), fromcode);

  /* maybe apply charset-hooks and recanonicalise fromcode,

   * but only when caller asked us to sanitize a potentially wrong

   * charset name incoming from the wild exterior. */

  if (flags & MUTT_ICONV_HOOK_FROM)

  {

    tmp = mutt_ch_charset_lookup(fromcode1);

    if (tmp)

      mutt_ch_canonical_charset(fromcode1, sizeof(fromcode1), tmp);

  }

  /* check if we have this pair cached already */

  for (int i = 0; i < IconvCacheUsed; i++)

  {

    if (strcmp(tocode1, IconvCache[i].tocode1) == 0 &&

        strcmp(fromcode1, IconvCache[i].fromcode1) == 0)

    {

      iconv_t cd = IconvCache[i].cd;

      /* make room for this one at the top */

      struct IconvCacheEntry top = IconvCache[i];

      for (int j = i - 1; j >= 0; j--)

      {

        IconvCache[j + 1] = IconvCache[j];

      }

      IconvCache[0] = top;

      if (iconv_t_valid(cd))

      {

        /* reset state */

        iconv(cd, NULL, NULL, NULL, NULL);

      }

      return cd;

    }

  }

  /* not found in cache */

  /* always apply iconv-hooks to suit system's iconv tastes */

  tocode2 = mutt_ch_iconv_lookup(tocode1);

  tocode2 = tocode2 ? tocode2 : tocode1;

  fromcode2 = mutt_ch_iconv_lookup(fromcode1);

  fromcode2 = fromcode2 ? fromcode2 : fromcode1;

  /* call system iconv with names it appreciates */

  iconv_t cd = iconv_open(tocode2, fromcode2);

  if (IconvCacheUsed == ICONV_CACHE_SIZE)

  {

    mutt_debug(LL_DEBUG2, "iconv: dropping %s -> %s from the cache\n",

               IconvCache[IconvCacheUsed - 1].fromcode1,

               IconvCache[IconvCacheUsed - 1].tocode1);

    /* get rid of the oldest entry */

    FREE(&IconvCache[IconvCacheUsed - 1].fromcode1);

    FREE(&IconvCache[IconvCacheUsed - 1].tocode1);

    if (iconv_t_valid(IconvCache[IconvCacheUsed - 1].cd))

    {

      iconv_close(IconvCache[IconvCacheUsed - 1].cd);

    }

    IconvCacheUsed--;

  }

  /* make room for this one at the top */

  for (int j = IconvCacheUsed - 1; j >= 0; j--)

  {

    IconvCache[j + 1] = IconvCache[j];

  }

  IconvCacheUsed++;

  mutt_debug(LL_DEBUG2, "iconv: adding %s -> %s to the cache\n", fromcode1, tocode1);

  IconvCache[0].fromcode1 = strdup(fromcode1);

  IconvCache[0].tocode1 = strdup(tocode1);

  IconvCache[0].cd = cd;

  return cd;

}

/**

 * mutt_ch_iconv - Change the encoding of a string

 * @param[in]     cd           Iconv conversion descriptor

 * @param[in,out] inbuf        Buffer to convert

 * @param[in,out] inbytesleft  Length of buffer to convert

 * @param[in,out] outbuf       Buffer for the result

 * @param[in,out] outbytesleft Length of result buffer

 * @param[in]     inrepls      Input replacement characters

 * @param[in]     outrepl      Output replacement characters

 * @param[out]    iconverrno   Errno if iconv() fails, 0 if it succeeds

 * @retval num Characters converted

 *

 * Like iconv, but keeps going even when the input is invalid

 * If you're supplying inrepls, the source charset should be stateless;

 * if you're supplying an outrepl, the target charset should be.

 */

size_t mutt_ch_iconv(iconv_t cd, const char **inbuf, size_t *inbytesleft,

                     char **outbuf, size_t *outbytesleft, const char **inrepls,

                     const char *outrepl, int *iconverrno)

{

  size_t rc = 0;

  const char *ib = *inbuf;

  size_t ibl = *inbytesleft;

  char *ob = *outbuf;

  size_t obl = *outbytesleft;

  while (true)

  {

    errno = 0;

    const size_t ret1 = iconv(cd, (ICONV_CONST char **) &ib, &ibl, &ob, &obl);

    if (ret1 != ICONV_ILLEGAL_SEQ)

      rc += ret1;

    if (iconverrno)

      *iconverrno = errno;

    if (ibl && obl && (errno == EILSEQ))

    {

      if (inrepls)

      {

        /* Try replacing the input */

        const char **t = NULL;

        for (t = inrepls; *t; t++)

        {

          const char *ib1 = *t;

          size_t ibl1 = strlen(*t);

          char *ob1 = ob;

          size_t obl1 = obl;

          iconv(cd, (ICONV_CONST char **) &ib1, &ibl1, &ob1, &obl1);

          if (ibl1 == 0)

          {

            ib++;

            ibl--;

            ob = ob1;

            obl = obl1;

            rc++;

            break;

          }

        }

        if (*t)

          continue;

      }

      /* Replace the output */

      if (!outrepl)

        outrepl = "?";

      iconv(cd, NULL, NULL, &ob, &obl);

      if (obl)

      {

        int n = strlen(outrepl);

        if (n > obl)

        {

          outrepl = "?";

          n = 1;

        }

        memcpy(ob, outrepl, n);

        ib++;

        ibl--;

        ob += n;

        obl -= n;

        rc++;

        iconv(cd, NULL, NULL, NULL, NULL); /* for good measure */

        continue;

      }

    }

    *inbuf = ib;

    *inbytesleft = ibl;

    *outbuf = ob;

    *outbytesleft = obl;

    return rc;

  }

}

/**

 * mutt_ch_iconv_lookup - Look for a replacement character set

 * @param chs Character set to lookup

 * @retval ptr  Replacement character set (if a 'iconv-hook' matches)

 * @retval NULL No matching hook

 *

 * Look through all the 'iconv-hook's.

 * If one matches return the replacement character set.

 */

const char *mutt_ch_iconv_lookup(const char *chs)

{

  return lookup_charset(MUTT_LOOKUP_ICONV, chs);

}

/**

 * mutt_ch_check - Check whether a string can be converted between encodings

 * @param[in] s     String to check

 * @param[in] slen  Length of the string to check

 * @param[in] from  Current character set

 * @param[in] to    Target character set

 * @retval 0  Success

 * @retval -1 Error in iconv_open()

 * @retval >0 Errno as set by iconv()

 */

int mutt_ch_check(const char *s, size_t slen, const char *from, const char *to)

{

  if (!s || !from || !to)

    return -1;

  int rc = 0;

  iconv_t cd = mutt_ch_iconv_open(to, from, MUTT_ICONV_NO_FLAGS);

  if (!iconv_t_valid(cd))

    return -1;

  size_t outlen = MB_LEN_MAX * slen;

  char *out = MUTT_MEM_MALLOC(outlen + 1, char);

  char *saved_out = out;

  const size_t convlen = iconv(cd, (ICONV_CONST char **) &s, &slen, &out, &outlen);

  if (convlen == ICONV_ILLEGAL_SEQ)

    rc = errno;

  FREE(&saved_out);

  return rc;

}

/**

 * mutt_ch_convert_string - Convert a string between encodings

 * @param[in,out] ps    String to convert

 * @param[in]     from  Current character set

 * @param[in]     to    Target character set

 * @param[in]     flags Flags, e.g. #MUTT_ICONV_HOOK_FROM

 * @retval 0      Success

 * @retval -1     Invalid arguments or failure to open an iconv channel

 * @retval errno  Failure in iconv conversion

 *

 * Parameter flags is given as-is to mutt_ch_iconv_open().

 * See there for its meaning and usage policy.

 */

int mutt_ch_convert_string(char **ps, const char *from, const char *to, uint8_t flags)

{

  if (!ps)

    return -1;

  char *s = *ps;

  if (!s || (*s == '\0'))

    return 0;

  if (!to || !from)

    return -1;

  const char *repls[] = { "\357\277\275", "?", 0 };

  int rc = 0;

  iconv_t cd = mutt_ch_iconv_open(to, from, flags);

  if (!iconv_t_valid(cd))

    return -1;

  const char **inrepls = NULL;

  const char *outrepl = NULL;

  if (mutt_ch_is_utf8(to))

    outrepl = "\357\277\275";

  else if (mutt_ch_is_utf8(from))

    inrepls = repls;

  else

    outrepl = "?";

  const char *ib = s;

  size_t ibl = strlen(s);

  if (ibl >= (SIZE_MAX / MB_LEN_MAX))

  {

    return -1;

  }

  size_t obl = MB_LEN_MAX * ibl;

  char *buf = MUTT_MEM_MALLOC(obl + 1, char);

  char *ob = buf;

  mutt_ch_iconv(cd, &ib, &ibl, &ob, &obl, inrepls, outrepl, &rc);

  iconv(cd, 0, 0, &ob, &obl);

  *ob = '\0';

  FREE(ps);

  *ps = buf;

  mutt_str_adjust(ps);

  return rc;

}

/**

 * mutt_ch_check_charset - Does iconv understand a character set?

 * @param cs     Character set to check

 * @param strict Check strictly by using iconv

 * @retval true Character set is valid

 *

 * If `strict` is false, then finding a matching character set in

 * #PreferredMimeNames will be enough.

 * If `strict` is true, or the charset is not in #PreferredMimeNames, then

 * iconv() with be run.

 */

bool mutt_ch_check_charset(const char *cs, bool strict)

{

  if (!cs)

    return false;

  if (mutt_ch_is_utf8(cs))

    return true;

  if (!strict)

  {

    for (int i = 0; PreferredMimeNames[i].key; i++)

    {

      if (mutt_istr_equal(PreferredMimeNames[i].key, cs) ||

          mutt_istr_equal(PreferredMimeNames[i].pref, cs))

      {

        return true;

      }

    }

  }

  iconv_t cd = mutt_ch_iconv_open(cs, cs, MUTT_ICONV_NO_FLAGS);

  if (iconv_t_valid(cd))

  {

    return true;

  }

  return false;

}

/**

 * mutt_ch_fgetconv_open - Prepare a file for charset conversion

 * @param fp    FILE ptr to prepare

 * @param from  Current character set

 * @param to    Destination character set

 * @param flags Flags, e.g. #MUTT_ICONV_HOOK_FROM

 * @retval ptr fgetconv handle

 *

 * Parameter flags is given as-is to mutt_ch_iconv_open().

 */

struct FgetConv *mutt_ch_fgetconv_open(FILE *fp, const char *from, const char *to, uint8_t flags)

{

  iconv_t cd = ICONV_T_INVALID;

  if (from && to)

    cd = mutt_ch_iconv_open(to, from, flags);

  struct FgetConv *fc = MUTT_MEM_CALLOC(1, struct FgetConv);

  fc->fp = fp;

  fc->cd = cd;

  if (iconv_t_valid(cd))

  {

    static const char *repls[] = { "\357\277\275", "?", 0 };

    fc->p = fc->bufo;

    fc->ob = fc->bufo;

    fc->ib = fc->bufi;

    fc->ibl = 0;

    fc->inrepls = mutt_ch_is_utf8(to) ? repls : repls + 1;

  }

  return fc;

}

/**

 * mutt_ch_fgetconv_close - Close an fgetconv handle

 * @param[out] ptr fgetconv handle

 */

void mutt_ch_fgetconv_close(struct FgetConv **ptr)

{

  if (!ptr || !*ptr)

    return;

  FREE(ptr);

}