/src/httrack/src/htslib.c

Source
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors

SPDX-License-Identifier: GPL-3.0-or-later

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.

Please visit our Website: http://www.httrack.com
*/

/* ------------------------------------------------------------ */
/* File: Subroutines                                            */
/* Author: Xavier Roche                                         */
/* ------------------------------------------------------------ */

/* Internal engine bytecode */
#define HTS_INTERNAL_BYTECODE

// Fichier librairie .c

#include "htscore.h"
#include "htssitemap.h"
#include "htswarc.h"
#include "htschanges.h"
#include "htssinglefile.h"

/* specific definitions */
#include "htsbase.h"
#include "htsio.h"
#include "htsnet.h"
#include "htsbauth.h"
#include "htsthread.h"
#include "htsback.h"
#include "htsftp.h"
#include "htswrap.h"
#include "htsmd5.h"
#include "htsmodules.h"
#include "htscharset.h"
#include "htsencoding.h"
#include "htscodec.h"

#ifdef _WIN32
#include <direct.h>
#else
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#ifdef HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#endif /* _WIN32 */
#include <stdarg.h>

#include <ctype.h>
#include <string.h>
#include <time.h>
#include <stdarg.h>

#ifndef _WIN32
#include <sys/time.h>
#else
#include <sys/timeb.h>
#endif
#include <fcntl.h>

// pour utimbuf
#ifdef _WIN32
#include <sys/utime.h>
#else
#include <utime.h>
#endif /* _WIN32 */

#include <sys/stat.h>

#ifdef __ANDROID__
#define timezone 0
#endif
/* END specific definitions */

/* Windows might be missing va_copy */
#ifdef _WIN32
#ifndef va_copy
#define va_copy(dst, src) ((dst) = (src))
#endif
#endif

// Debugging
#if _HTS_WIDE
FILE *DEBUG_fp = NULL;
#endif

/* variables globales */
int _DEBUG_HEAD;
FILE *ioinfo;

#if HTS_USEOPENSSL
SSL_CTX *openssl_ctx = NULL;
#endif
int IPV6_resolver = 0;

/* détection complémentaire */
const char *hts_detect[] = {
  "archive",
  "background",
  "data",                       // OBJECT
  "data-src",
  "data-srcset",
  "dynsrc",
  "lowsrc",
  "profile",                    // element META
  "src",
  "srcset",                     // HTML5 responsive images (<img>, <source>)
  "swurl",
  "url",
  "usemap",
  "longdesc",                   // accessibility
  "xlink:href",                 // xml/svg tag
  "poster",                     // HTML5
  ""
};

/* détecter début */
const char *hts_detectbeg[] = {
  "hotspot",                    /* hotspot1=..,hotspot2=.. */
  ""
};

/* ne pas détcter de liens dedans */
const char *hts_nodetect[] = {
  "accept-charset",
  "accesskey",
  "action",
  "align",
  "alt",
  "axes",
  "axis",
  "char",
  "charset",
  "cite",
  "class",
  "classid",
  "code",
  "color",
  "datetime",
  "dir",
  "enctype",
  "face",
  "height",
  "id",
  "lang",
  "language",
  "media",
  "method",
  "name",
  "prompt",
  "scheme",
  "size",
  "style",
  "target",
  "title",
  "type",
  "valign",
  "version",
  "width",
  ""
};

/* détection de mini-code javascript */
/* ALSO USED: detection based on the name: onXXX="<tag>" where XXX starts with upper case letter */
const char *hts_detect_js[] = {
  "onAbort",
  "onBlur",
  "onChange",
  "onClick",
  "onDblClick",
  "onDragDrop",
  "onError",
  "onFocus",
  "onKeyDown",
  "onKeyPress",
  "onKeyUp",
  "onLoad",
  "onMouseDown",
  "onMouseMove",
  "onMouseOut",
  "onMouseOver",
  "onMouseUp",
  "onMove",
  "onReset",
  "onResize",
  "onSelect",
  "onSubmit",
  "onUnload",
  "style",                      /* hack for CSS code data */
  ""
};

const char *hts_main_mime[] = {
  "application",
  "audio",
  "image",
  "message",
  "multipart",
  "text",
  "video",
  ""
};

/* détection "...URL=<url>" */
const char *hts_detectURL[] = {
  "content",
  ""
};

/* tags où l'URL doit être réécrite mais non capturée */
const char *hts_detectandleave[] = {
  "action",
  ""
};

/* ne pas renommer les types renvoyés (souvent types inconnus) */
const char *hts_mime_keep[] = {
  "application/octet-stream",
  "text/plain",
  "application/xml",
  "text/xml",
  ""
};

/* bogus servers returns these mime types when the extension is seen within the filename */
const char *hts_mime_bogus_multiple[] = {
  "application/x-wais-source",  /* src (src.rpm) */
  ""
};

/* pas de type mime connu, mais extension connue */
const char *hts_ext_dynamic[] = {
  "php3",
  "php",
  "php4",
  "php2",
  "cgi",
  "asp",
  "jsp",
  "pl",
  /*"exe", */
  "cfm",
  "nsf",                        /* lotus */
  ""
};

/* types MIME 
   note: application/octet-stream should not be used here
*/
const char *hts_mime[][2] = {
  {"application/acad", "dwg"},
  {"application/arj", "arj"},
  {"application/clariscad", "ccad"},
  {"application/drafting", "drw"},
  {"application/dxf", "dxf"},
  {"application/excel", "xls"},
  {"application/i-deas", "unv"},
  {"application/iges", "isg"},
  {"application/iges", "iges"},
  {"application/mac-binhex40", "hqx"},
  {"application/mac-compactpro", "cpt"},
  {"application/msword", "doc"},
  {"application/msword", "w6w"},
  {"application/msword", "word"},
  {"application/mswrite", "wri"},
  /*{"application/octet-stream","dms"}, */
  /*{"application/octet-stream","lzh"}, */
  /*{"application/octet-stream","lha"}, */
  /*{"application/octet-stream","bin"}, */
  {"application/oda", "oda"},
  {"application/pdf", "pdf"},
  {"application/postscript", "ps"},
  {"application/postscript", "ai"},
  {"application/postscript", "eps"},
  {"application/powerpoint", "ppt"},
  {"application/pro_eng", "prt"},
  {"application/pro_eng", "part"},
  {"application/rtf", "rtf"},
  {"application/set", "set"},
  {"application/sla", "stl"},
  {"application/smil", "smi"},
  {"application/smil", "smil"},
  {"application/smil", "sml"},
  {"application/solids", "sol"},
  {"application/STEP", "stp"},
  {"application/STEP", "step"},
  {"application/vda", "vda"},
  {"application/x-authorware-map", "aam"},
  {"application/x-authorware-seg", "aas"},
  {"application/x-authorware-bin", "aab"},
  {"application/x-bzip2", "bz2"},
  {"application/x-cocoa", "cco"},
  {"application/x-csh", "csh"},
  {"application/x-director", "dir"},
  {"application/x-director", "dcr"},
  {"application/x-director", "dxr"},
  {"application/x-mif", "mif"},
  {"application/x-dvi", "dvi"},
  {"application/x-gzip", "gz"},
  {"application/x-gzip", "gzip"},
  {"application/x-hdf", "hdf"},
  {"application/x-javascript", "js"},
  {"application/x-koan", "skp"},
  {"application/x-koan", "skd"},
  {"application/x-koan", "skt"},
  {"application/x-koan", "skm"},
  {"application/x-latex", "latex"},
  {"application/x-netcdf", "nc"},
  {"application/x-netcdf", "cdf"},
  /* {"application/x-sh","sh"}, */
  /* {"application/x-csh","csh"}, */
  /* {"application/x-ksh","ksh"}, */
  {"application/x-shar", "shar"},
  {"application/x-stuffit", "sit"},
  {"application/x-tcl", "tcl"},
  {"application/x-tex", "tex"},
  {"application/x-texinfo", "texinfo"},
  {"application/x-texinfo", "texi"},
  {"application/x-troff", "t"},
  {"application/x-troff", "tr"},
  {"application/x-troff", "roff"},
  {"application/x-troff-man", "man"},
  {"application/x-troff-me", "ms"},
  {"application/x-wais-source", "src"},
  {"application/zip", "zip"},
  {"application/x-zip-compressed", "zip"},
  {"application/x-bcpio", "bcpio"},
  {"application/x-cdlink", "vcd"},
  {"application/x-cpio", "cpio"},
  {"application/x-gtar", "tgz"},
  {"application/x-gtar", "gtar"},
  {"application/x-shar", "shar"},
  {"application/x-shockwave-flash", "swf"},
  {"application/x-sv4cpio", "sv4cpio"},
  {"application/x-sv4crc", "sv4crc"},
  {"application/x-tar", "tar"},
  {"application/x-ustar", "ustar"},
  {"application/x-winhelp", "hlp"},
  {"application/xml", "xml"},
  {"audio/midi", "mid"},
  {"audio/midi", "midi"},
  {"audio/midi", "kar"},
  {"audio/mpeg", "mp3"},
  {"audio/mpeg", "mpga"},
  {"audio/mpeg", "mp2"},
  {"audio/basic", "au"},
  {"audio/basic", "snd"},
  {"audio/x-aiff", "aif"},
  {"audio/x-aiff", "aiff"},
  {"audio/x-aiff", "aifc"},
  {"audio/x-pn-realaudio", "rm"},
  {"audio/x-pn-realaudio", "ram"},
  {"audio/x-pn-realaudio", "ra"},
  {"audio/x-pn-realaudio-plugin", "rpm"},
  {"audio/x-wav", "wav"},
  {"chemical/x-pdb", "pdb"},
  {"chemical/x-pdb", "xyz"},
  {"drawing/x-dwf", "dwf"},
  {"image/gif", "gif"},
  {"image/ief", "ief"},
  {"image/jpeg", "jpg"},
  {"image/jpeg", "jpe"},
  {"image/jpeg", "jpeg"},
  {"image/pict", "pict"},
  {"image/png", "png"},
  {"image/tiff", "tiff"},
  {"image/tiff", "tif"},
  {"image/svg+xml", "svg"},
  {"image/svg-xml", "svg"},
  {"image/x-cmu-raster", "ras"},
  {"image/x-freehand", "fh4"},
  {"image/x-freehand", "fh7"},
  {"image/x-freehand", "fh5"},
  {"image/x-freehand", "fhc"},
  {"image/x-freehand", "fh"},
  {"image/x-portable-anymap", "pnm"},
  {"image/x-portable-bitmap", "pgm"},
  {"image/x-portable-pixmap", "ppm"},
  {"image/x-rgb", "rgb"},
  {"image/x-xbitmap", "xbm"},
  {"image/x-xpixmap", "xpm"},
  {"image/x-xwindowdump", "xwd"},
  {"model/mesh", "msh"},
  {"model/mesh", "mesh"},
  {"model/mesh", "silo"},
  {"multipart/x-zip", "zip"},
  {"multipart/x-gzip", "gzip"},
  {"text/css", "css"},
  {"text/html", "html"},
  {"text/html", "htm"},
  {"text/plain", "txt"},
  {"text/plain", "g"},
  {"text/plain", "h"},
  {"text/plain", "c"},
  {"text/plain", "cc"},
  {"text/plain", "hh"},
  {"text/plain", "m"},
  {"text/plain", "f90"},
  {"text/richtext", "rtx"},
  {"text/tab-separated-values", "tsv"},
  {"text/x-setext", "etx"},
  {"text/x-sgml", "sgml"},
  {"text/x-sgml", "sgm"},
  {"text/xml", "xml"},
  {"text/xml", "dtd"},
  {"video/mpeg", "mpeg"},
  {"video/mpeg", "mpg"},
  {"video/mpeg", "mpe"},
  {"video/quicktime", "qt"},
  {"video/quicktime", "mov"},
  {"video/x-msvideo", "avi"},
  {"video/x-sgi-movie", "movie"},
  {"x-conference/x-cooltalk", "ice"},
  /*{"application/x-httpd-cgi","cgi"}, */
  {"x-world/x-vrml", "wrl"},

  /* More from w3schools.com */
  {"application/envoy", "evy"},
  {"application/fractals", "fif"},
  {"application/futuresplash", "spl"},
  {"application/hta", "hta"},
  {"application/internet-property-stream", "acx"},
  {"application/msword", "dot"},
  {"application/olescript", "axs"},
  {"application/pics-rules", "prf"},
  {"application/pkcs10", "p10"},
  {"application/pkix-crl", "crl"},
  {"application/set-payment-initiation", "setpay"},
  {"application/set-registration-initiation", "setreg"},
  {"application/vnd.ms-excel", "xls"},
  {"application/vnd.ms-excel", "xla"},
  {"application/vnd.ms-excel", "xlc"},
  {"application/vnd.ms-excel", "xlm"},
  {"application/vnd.ms-excel", "xlt"},
  {"application/vnd.ms-excel", "xlw"},
  {"application/vnd.ms-pkicertstore", "sst"},
  {"application/vnd.ms-pkiseccat", "cat"},
  {"application/vnd.ms-powerpoint", "ppt"},
  {"application/vnd.ms-powerpoint", "pot"},
  {"application/vnd.ms-powerpoint", "pps"},
  {"application/vnd.ms-project", "mpp"},
  {"application/vnd.ms-works", "wcm"},
  {"application/vnd.ms-works", "wdb"},
  {"application/vnd.ms-works", "wks"},
  {"application/vnd.ms-works", "wps"},
  {"application/vnd.oasis.opendocument.chart", "odc"},
  {"application/vnd.oasis.opendocument.database", "odb"},
  {"application/vnd.oasis.opendocument.formula", "odf"},
  {"application/vnd.oasis.opendocument.graphics", "odg"},
  {"application/vnd.oasis.opendocument.graphics-template", "otg"},
  {"application/vnd.oasis.opendocument.image", "odi"},
  {"application/vnd.oasis.opendocument.presentation", "odp"},
  {"application/vnd.oasis.opendocument.presentation-template", "otp"},
  {"application/vnd.oasis.opendocument.spreadsheet", "ods"},
  {"application/vnd.oasis.opendocument.spreadsheet-template", "ots"},
  {"application/vnd.oasis.opendocument.text", "odt"},
  {"application/vnd.oasis.opendocument.text-master", "odm"},
  {"application/vnd.oasis.opendocument.text-template", "ott"},
  {"application/vnd.oasis.opendocument.text-web", "oth"},
  {"application/vnd.openxmlformats-officedocument.presentationml.presentation", "pptx"},
  {"application/vnd.openxmlformats-officedocument.presentationml.slide", "sldx"},
  {"application/vnd.openxmlformats-officedocument.presentationml.slideshow", "ppsx"},
  {"application/vnd.openxmlformats-officedocument.presentationml.template", "potx"},
  {"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", "xlsx"},
  {"application/vnd.openxmlformats-officedocument.spreadsheetml.template", "xltx"},
  {"application/vnd.openxmlformats-officedocument.wordprocessingml.document", "docx"},
  {"application/vnd.openxmlformats-officedocument.wordprocessingml.template", "dotx"},
  {"application/x-compress", "z"},
  {"application/x-compressed", "tgz"},
  {"application/x-internet-signup", "ins"},
  {"application/x-internet-signup", "isp"},
  {"application/x-iphone", "iii"},
  {"application/x-javascript", "js"},
  {"application/x-msaccess", "mdb"},
  {"application/x-mscardfile", "crd"},
  {"application/x-msclip", "clp"},
  {"application/x-msmediaview", "m13"},
  {"application/x-msmediaview", "m14"},
  {"application/x-msmediaview", "mvb"},
  {"application/x-msmetafile", "wmf"},
  {"application/x-msmoney", "mny"},
  {"application/x-mspublisher", "pub"},
  {"application/x-msschedule", "scd"},
  {"application/x-msterminal", "trm"},
  {"application/x-perfmon", "pma"},
  {"application/x-perfmon", "pmc"},
  {"application/x-perfmon", "pml"},
  {"application/x-perfmon", "pmr"},
  {"application/x-perfmon", "pmw"},
  {"application/x-pkcs12", "p12"},
  {"application/x-pkcs12", "pfx"},
  {"application/x-pkcs7-certificates", "p7b"},
  {"application/x-pkcs7-certificates", "spc"},
  {"application/x-pkcs7-certreqresp", "p7r"},
  {"application/x-pkcs7-mime", "p7c"},
  {"application/x-pkcs7-mime", "p7m"},
  {"application/x-pkcs7-signature", "p7s"},
  {"application/x-troff-me", "me"},
  {"application/x-x509-ca-cert", "cer"},
  {"application/x-x509-ca-cert", "crt"},
  {"application/x-x509-ca-cert", "der"},
  {"application/ynd.ms-pkipko", "pko"},
  {"audio/mid", "mid"},
  {"audio/mid", "rmi"},
  {"audio/mpeg", "mp3"},
  {"audio/x-mpegurl", "m3u"},
  {"image/bmp", "bmp"},
  {"image/cis-cod", "cod"},
  {"image/pipeg", "jfif"},
  {"image/x-cmx", "cmx"},
  {"image/x-icon", "ico"},
  {"image/x-portable-bitmap", "pbm"},
  {"message/rfc822", "mht"},
  {"message/rfc822", "mhtml"},
  {"message/rfc822", "nws"},
  {"text/css", "css"},
  {"text/h323", "323"},
  {"text/html", "stm"},
  {"text/iuls", "uls"},
  {"text/plain", "bas"},
  {"text/scriptlet", "sct"},
  {"text/webviewhtml", "htt"},
  {"text/x-component", "htc"},
  {"text/x-vcard", "vcf"},
  {"video/mpeg", "mp2"},
  {"video/mpeg", "mpa"},
  {"video/mpeg", "mpv2"},
  {"video/x-la-asf", "lsf"},
  {"video/x-la-asf", "lsx"},
  {"video/x-ms-asf", "asf"},
  {"video/x-ms-asf", "asr"},
  {"video/x-ms-asf", "asx"},
  {"video/x-ms-wmv", "wmv"},
  {"x-world/x-vrml", "flr"},
  {"x-world/x-vrml", "vrml"},
  {"x-world/x-vrml", "wrz"},
  {"x-world/x-vrml", "xaf"},
  {"x-world/x-vrml", "xof"},

  /* Various */
  {"application/ogg", "ogg"},

  {"application/x-java-vm", "class"},
  {"application/x-bittorrent","torrent"},

  {"", ""}
};

/* Modern web formats (post-2010), kept in their own table: appending to the
   legacy hts_mime[] above makes clang-format reflow its whole initializer.
   Scanned after hts_mime[], so it never shadows a legacy mapping. */
static const char *hts_mime_modern[][2] = {
    {"image/webp", "webp"},
    {"image/avif", "avif"},
    {"image/heic", "heic"},
    {"font/woff", "woff"},
    {"font/woff2", "woff2"},
    {"font/ttf", "ttf"},
    {"font/otf", "otf"},
    {"application/json", "json"},
    {"application/ld+json", "jsonld"},
    {"application/manifest+json", "webmanifest"},
    {"application/wasm", "wasm"},
    {"text/javascript", "js"},
    {"text/javascript", "mjs"},
    {"text/markdown", "md"},
    {"video/mp4", "mp4"},
    {"video/webm", "webm"},
    {"video/ogg", "ogv"},
    {"video/mp2t", "ts"},
    {"audio/mp4", "m4a"},
    {"audio/aac", "aac"},
    {"audio/ogg", "oga"},
    {"audio/opus", "opus"},
    {"audio/flac", "flac"},
    {"audio/webm", "weba"},
    {"application/x-7z-compressed", "7z"},
    {"application/x-rar-compressed", "rar"},
    {"application/zstd", "zst"},
    {"", ""}};

// Reserved (RFC2396)
#define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
#define CHAR_RESERVED(c)                                                       \
  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
// Delimiters (RFC2396)
#define CHAR_DELIM(c)                                                          \
  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
// Unwise (RFC2396)
#define CHAR_UNWISE(c)                                                         \
  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
// Special (escape chars) (RFC2396 + >127 )
#define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
#define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
#define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
// We try to avoid them and encode them instead
#define CHAR_XXAVOID(c)                                                        \
  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
   CIS(c, '&') || CIS(c, '!'))
#define CHAR_MARK(c)                                                           \
  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))

// conversion éventuelle / vers antislash
#ifdef _WIN32
char *antislash(char *catbuff, const char *s) {
  char *a;

  strcpybuff(catbuff, s);
  while(a = strchr(catbuff, '/'))
    *a = '\\';
  return catbuff;
}
#endif

// Initialize a htsblk structure
void hts_init_htsblk(htsblk * r) {
  memset(r, 0, sizeof(htsblk)); // effacer
  r->soc = INVALID_SOCKET;
  r->msg[0] = '\0';
  r->statuscode = STATUSCODE_INVALID;
  r->totalsize = -1;
}

// ouverture d'une liaison http, envoi d'une requète
// mode: 0 GET  1 HEAD  [2 POST]
// treat: traiter header?
// waitconnect: attendre le connect()
// note: dans retour, on met les params du proxy
T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
                  const char *xsend, const char *adr, const char *fil,
                  htsblk *retour) {
  T_SOC soc = INVALID_SOCKET;
  char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];

  // retour prédéfini: erreur
  if (retour) {
    retour->adr = NULL;
    retour->size = 0;
    retour->msg[0] = '\0';
    retour->statuscode = STATUSCODE_NON_FATAL;  // a priori erreur non fatale
  }
#if HDEBUG
  printf("adr=%s\nfichier=%s\n", adr, fil);
#endif

  // ouvrir liaison
#if HDEBUG
  printf("Création d'une socket sur %s\n", adr);
#endif

#if CNXDEBUG
  printf("..newhttp\n");
#endif

  /* connexion */
  if (retour) {
    /* no proxy, or proxy not usable here (local file) */
    if ((!(retour->req.proxy.active)) || (strcmp(adr, "file://") == 0)) {
      soc = newhttp(opt, adr, retour, -1, waitconnect);
    } else {
      // to the proxy; https tunnels to the origin via CONNECT in back_wait
      // (#85)
      soc = newhttp(opt, retour->req.proxy.name, retour, retour->req.proxy.port,
                    waitconnect);
    }
  } else {
    soc = newhttp(opt, adr, NULL, -1, waitconnect);
  }

  // copier index socket retour
  if (retour)
    retour->soc = soc;

  /* Check for errors */
  if (soc == INVALID_SOCKET) {
    if (retour) {
      if (!strnotempty(retour->msg)) {
#ifdef _WIN32
        int last_errno = WSAGetLastError();

        htsblk_failf(retour, "Connect error: %s", strerror(last_errno));
#else
        int last_errno = errno;

        htsblk_failf(retour, "Connect error: %s", strerror(last_errno));
#endif
      }
    }
  }
  // --------------------
  // court-circuit (court circuite aussi le proxy..)
  // LOCAL_SOCKET_ID est une pseudo-socket locale
  if (soc == LOCAL_SOCKET_ID) {
    retour->is_file = 1;        // fichier local
    if (mode == 0) {            // GET

      // Test en cas de file:///C|...
      if (!fexist
          (fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
          unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil))))
        if (fexist
            (fconv
             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
             unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil + 1)))) {
          strcpybuff(tempo_fil, fil + 1);
          fil = tempo_fil;
        }
      // Ouvrir
      retour->totalsize = fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
        unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)));       // taille du fichier
      retour->msg[0] = '\0';
      soc = INVALID_SOCKET;
      if (retour->totalsize < 0)
        strcpybuff(retour->msg, "Unable to open local file");
      else {
        // Note: On passe par un FILE* (plus propre)
        retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
          unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
        if (retour->fp == NULL)
          soc = INVALID_SOCKET;
        else
          soc = LOCAL_SOCKET_ID;
      }
      retour->soc = soc;
      if (soc != INVALID_SOCKET) {
        retour->statuscode = HTTP_OK;   // OK
        strcpybuff(retour->msg, "OK");
        guess_httptype_sized(opt, retour->contenttype,
                             sizeof(retour->contenttype), fil);
      } else if (strnotempty(retour->msg) == 0)
        strcpybuff(retour->msg, "Unable to open local file");
      return soc;               // renvoyer
    } else {                    // HEAD ou POST : interdit sur un local!!!! (c'est idiot!)
      strcpybuff(retour->msg, "Unexpected Head/Post local request");
      soc = INVALID_SOCKET;     // erreur
      retour->soc = soc;
      return soc;
    }
  }
  // --------------------

  if (soc != INVALID_SOCKET) {
    char rcvd[1100];

    rcvd[0] = '\0';
#if HDEBUG
    printf("Ok, connexion réussie, id=%d\n", soc);
#endif

    // connecté?
    if (waitconnect) {
      http_sendhead(opt, NULL, mode, xsend, adr, fil, NULL, NULL, retour);
    }

    if (soc != INVALID_SOCKET) {

#if HDEBUG
      printf("Attente de la réponse:\n");
#endif

      // si GET (réception d'un fichier), réceptionner en-tête d'abord,
      // et ensuite le corps
      // si POST on ne réceptionne rien du tout, c'est après que l'on fera
      // une réception standard pour récupérer l'en tête
      if ((treat) && (waitconnect)) {   // traiter (attendre!) en-tête
        // Réception de la status line et de l'en-tête (norme RFC1945)

        // status-line à récupérer
        finput_line(soc, rcvd, 1024);
        // some buggy servers send a leading \n (RFC)
        if (strnotempty(rcvd) == 0)
          finput_line(soc, rcvd, 1024);

        // traiter status-line
        treatfirstline(retour, rcvd);

#if HDEBUG
        printf("Status-Code=%d\n", retour->statuscode);
#endif

        // en-tête

        // header // ** !attention! HTTP/0.9 non supporté
        do {
          const hts_boolean cut = finput_line(soc, rcvd, 1024);

#if HDEBUG
          printf(">%s\n", rcvd);
#endif
          if (cut)
            hts_log_print(NULL, LOG_WARNING, "Over-long header dropped");
          else if (strnotempty(rcvd))
            treathead(NULL, NULL, NULL, retour, rcvd);

        } while(strnotempty(rcvd));

      } else { // si GET, on recevra l'en tête APRES
        if (retour)
          retour->totalsize = -1;
      }
    }

  }

  return soc;
}

/* Buffer printing */
typedef struct buff_struct {
  /** Buffer **/
  char *buffer;
  /** Buffer capacity in bytes **/
  size_t capacity;
  /** Buffer write position ; MUST point to a valid \0. **/
  size_t pos;
} buff_struct;

static void print_buffer(buff_struct*const str, const char *format, ...)
  HTS_PRINTF_FUN(2, 3);

/* Prints on a static buffer. asserts in case of overflow. */
static void print_buffer(buff_struct*const str, const char *format, ...) {
  size_t result;
  va_list args;
  size_t remaining;
  char *position;

  /* Security check. */
  assertf(str != NULL);
  assertf(str->pos < str->capacity);

  /* Print */
  position = &str->buffer[str->pos];
  remaining = str->capacity - str->pos;
  va_start(args, format);
  result = (size_t) vsnprintf(position, remaining, format, args);
  va_end(args);
  assertf(result < remaining);

  /* Increment. */
  str->pos += strlen(position);
  assertf(str->pos < str->capacity);
}

/* Append the request "Cookie:" header line for every stored cookie matching
   domain/path. RFC 6265 form: bare "name=value" pairs joined by "; ", no
   $Version/$Path attributes (those are RFC 2965 syntax that modern servers
   reject, issue #151). Returns the number of cookies emitted. */
static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
                                const char *domain, const char *path) {
  char buffer[8192];
  char *b;
  int cook = 0;
  int max_cookies = 8;

  if (cookie == NULL)
    return 0;
  b = cookie->data;
  do {
    b = cookie_find(b, "", domain, path); // next matching cookie
    if (b != NULL) {
      max_cookies--;
      if (!cook) {
        print_buffer(bstr, "Cookie: ");
        cook = 1;
      } else
        print_buffer(bstr, "; ");
      print_buffer(bstr, "%s", cookie_get(buffer, b, 5));
      print_buffer(bstr, "=%s", cookie_get(buffer, b, 6));
      b = cookie_nextfield(b);
    }
  } while (b != NULL && max_cookies > 0);
  if (cook)
    print_buffer(bstr, H_CRLF);
  return cook;
}

/* Build the request Cookie line for domain/path into dst (always
   NUL-terminated). Returns the number of cookies emitted. */
int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
                       char *dst, size_t dst_size) {
  buff_struct bstr = {dst, dst_size, 0};

  assertf(dst != NULL && dst_size > 0);
  dst[0] = '\0';
  return append_cookie_header(&bstr, cookie, domain, path);
}

hts_boolean hts_body_missing_unexpectedly(const htsblk *r) {
  if (r->adr != NULL || r->is_write != 0)
    return HTS_FALSE;
  if (HTTP_IS_REDIRECT(r->statuscode) ||
      r->statuscode == HTTP_PRECONDITION_FAILED ||
      r->statuscode == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE)
    return HTS_FALSE;
  return HTS_TRUE;
}

hts_boolean http_headers_have_field(const char *headers,
                                    const char *field_name) {
  size_t len;
  const char *line;

  if (headers == NULL || field_name == NULL)
    return HTS_FALSE;
  len = strlen(field_name);
  if (len != 0 && field_name[len - 1] == ':') /* the tree spells it both ways */
    len--;
  if (len == 0)
    return HTS_FALSE;
  /* anchored, so a folded line or a name inside a value cannot match */
  for (line = headers; *line != '\0';) {
    size_t i = 0;

    while (i < len && streql(line[i], field_name[i]))
      i++;
    /* RFC 7230 forbids a space before the colon: tolerating one would drop our
       line for a malformed one the server then ignores */
    if (i == len && line[len] == ':')
      return HTS_TRUE;
    while (*line != '\0' && *line != '\n' && *line != '\r')
      line++;
    while (*line == '\n' || *line == '\r')
      line++;
  }
  return HTS_FALSE;
}

/* Dropping a line drops its folded continuations too: kept, that value would
   attach to whichever header precedes it in the request. */
static void append_custom_headers(buff_struct *bstr, const char *headers) {
  const size_t base = bstr->pos;
  hts_boolean keep = HTS_TRUE;
  const char *line;

  for (line = headers; *line != '\0';) {
    const char *eol = line;
    const char *next;

    while (*eol != '\0' && *eol != '\r' && *eol != '\n')
      eol++;
    next = eol;
    if (next[0] == '\r' && next[1] == '\n')
      next += 2;
    else if (*next != '\0')
      next++;

    if (*line != ' ' && *line != '\t') { /* not a folded continuation */
      const char *colon = line;
      char field[128];

      keep = HTS_TRUE;
      while (colon < eol && *colon != ':')
        colon++;
      if (colon > line && colon < eol &&
          (size_t) (colon - line) < sizeof(field)) {
        const size_t len = (size_t) (colon - line);
        const char cut = bstr->buffer[base];

        memcpy(field, line, len);
        field[len] = '\0';
        /* only what the engine wrote, so the box may repeat its own fields */
        bstr->buffer[base] = '\0';
        keep = !http_headers_have_field(bstr->buffer, field);
        bstr->buffer[base] = cut;
      }
    }
    if (keep)
      print_buffer(bstr, "%.*s", (int) (next - line), line);
    line = next;
  }
}

void http_append_custom_headers(char *dst, size_t dst_size,
                                const char *headers) {
  buff_struct bstr = {dst, dst_size, 0};

  assertf(dst != NULL && dst_size > 0);
  bstr.pos = strlen(dst);
  append_custom_headers(&bstr, headers);
}

// envoi d'une requète
int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
                  const char *xsend, const char *adr, const char *fil,
                  const char *referer_adr, const char *referer_fil,
                  htsblk * retour) {
  char BIGSTK buffer_head_request[16384];
  buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };

  int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
  const char *search_tag = NULL;

  /* the extra-headers box wins: skip the engine's own line (#1337, #1340) */
  const char *const custom = retour->req.headers;

  /* adr and the referer come off the network and can carry raw CR/LF; capped
     at their own source buffers, so the worst case emitted does not grow. */
  char BIGSTK esc[HTS_URLMAXSIZE * 2];

  // Initialize buffer
  buffer_head_request[0] = '\0';

  // possibilité non documentée: >post: et >postfile:
  // si présence d'un tag >post: alors executer un POST
  // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
  // si présence d'un tag >postfile: alors envoyer en tête brut contenu dans le fichier en question
  // exemple: http://www.example.com/test.cgi?foo>postfile:post0.txt
  search_tag = strstr(fil, POSTTOK ":");
  if (!search_tag) {
    search_tag = strstr(fil, POSTTOK "file:");
    if (search_tag) {           // postfile
      if (mode == 0) {          // GET!
        FILE *fp =
          FOPEN(unescape_http(OPT_GET_BUFF(opt), 
                OPT_GET_BUFF_SIZE(opt), search_tag + strlen(POSTTOK) + 5), "rb");
        if (fp) {
          char BIGSTK line[1100];
          char BIGSTK protocol[256], url[HTS_URLMAXSIZE * 2], method[256];

          linput(fp, line, 1000);
          /* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
          if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
            size_t ret;
            // http proxy: absolute-URI; socks/CONNECT tunnel: origin-form
            if (retour->req.proxy.active &&
                !hts_proxy_is_socks(retour->req.proxy.name) &&
                !hts_proxy_is_connect(retour->req.proxy.name)) {
              /* only adr is raw here: the other three are %s-scanned */
              print_buffer(&bstr, "%s http://%s%s %s\r\n", method,
                           escape_check_url_addr(adr, esc, sizeof(esc)), url,
                           protocol);
            } else {
              print_buffer(&bstr,
                       "%s %s %s\r\n", method, url, protocol);
            }
            // lire le reste en brut
            ret = fread(&bstr.buffer[bstr.pos],
                        bstr.capacity - bstr.pos, 1, fp);
            if ((int) ret < 0) {
              return -1;
            }
            bstr.pos += strlen(&bstr.buffer[bstr.pos]);
          }
          fclose(fp);
        }
      }
    }
  }
  // Fin postfile

  if (bstr.pos == 0) { // PAS POSTFILE
    // Type de requète?
    if ((search_tag) && (mode == 0)) {
      print_buffer(&bstr, "POST ");
    } else if (mode == 0) {     // GET
      print_buffer(&bstr, "GET ");
    } else {                    // if (mode==1) {
      if (!retour->req.http11)  // forcer HTTP/1.0
        print_buffer(&bstr, "GET ");       // certains serveurs (cgi) buggent avec HEAD
      else
        print_buffer(&bstr, "HEAD ");
    }

    // an http proxy needs an absolute URI; a socks or CONNECT tunnel does not
    if (retour->req.proxy.active &&
        !hts_proxy_is_socks(retour->req.proxy.name) &&
        !hts_proxy_is_connect(retour->req.proxy.name) &&
        (strncmp(adr, "https://", 8) != 0)) {
      if (!link_has_authority(adr)) {   // default http
#if HDEBUG
        printf("Proxy Use: for %s%s proxy %d port %d\n", adr, fil,
               retour->req.proxy.name, retour->req.proxy.port);
#endif
        print_buffer(&bstr, "http://%s",
                     escape_check_url_addr(jump_identification_const(adr), esc,
                                           sizeof(esc)));
      } else {                  // ftp:// en proxy http
#if HDEBUG
        printf("Proxy Use for ftp: for %s%s proxy %d port %d\n", adr, fil,
               retour->req.proxy.name, retour->req.proxy.port);
#endif
        direct_url = 1;         // ne pas analyser user/pass
        print_buffer(&bstr, "%s", escape_check_url_addr(adr, esc, sizeof(esc)));
      }
    }
    // NOM DU FICHIER
    // on slash doit être présent en début, sinon attention aux bad request! (400)
    if (*fil != '/')
      print_buffer(&bstr, "/");

    {
      char BIGSTK tempo[HTS_URLMAXSIZE * 2];

      tempo[0] = '\0';
      if (search_tag)
        strncatbuff(tempo, fil, (int) (search_tag - fil));
      else
        strcpybuff(tempo, fil);
      inplace_escape_check_url(tempo, sizeof(tempo));
      print_buffer(&bstr, "%s", tempo);  // avec échappement
    }

    // protocole
    if (!retour->req.http11) { // forcer HTTP/1.0
      print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
    } else { // Requète 1.1
      print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
    }

    /* supplemental data */
    if (xsend)
      print_buffer(&bstr, "%s", xsend);  // éventuelles autres lignes

    // https/connect://: auth rides the CONNECT; socks: the handshake
    if (retour->req.proxy.active &&
        !hts_proxy_is_socks(retour->req.proxy.name) &&
        !hts_proxy_is_connect(retour->req.proxy.name) &&
        strncmp(adr, "https://", 8) != 0) {
      if (link_has_authorization(retour->req.proxy.name) &&
          !http_headers_have_field(custom, "Proxy-Authorization")) {
        const char *a = jump_identification_const(retour->req.proxy.name);
        const char *astart = jump_protocol_const(retour->req.proxy.name);
        char autorisation[1100];
        char user_pass[256];

        autorisation[0] = user_pass[0] = '\0';
        //
        strncatbuff(user_pass, astart, (int) (a - astart) - 1);
        strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), user_pass));
        code64((unsigned char *) user_pass, (int) strlen(user_pass),
               (unsigned char *) autorisation, 0);
        print_buffer(&bstr, "Proxy-Authorization: Basic %s"H_CRLF,
                     autorisation);
#if HDEBUG
        printf("Proxy-Authenticate, %s (code: %s)\n", user_pass, autorisation);
#endif
      }
    }
    // Referer?
    if (referer_adr != NULL && referer_fil != NULL && strnotempty(referer_adr)
        && strnotempty(referer_fil)
      ) {                       // non vide
      if ((strcmp(referer_adr, "file://") != 0) &&
          (/* no https referer to http urls */
           (strncmp(referer_adr, "https://", 8) != 0) /* referer is not https */
           || (strncmp(adr, "https://", 8) ==
               0) /* or referer AND addresses are https */
           ) &&
          !http_headers_have_field(custom, "Referer")) { // not file://
        /* one escape per piece, so neither can outgrow its own source buffer */
        print_buffer(
            &bstr, "Referer: http://%s",
            escape_check_url_addr(jump_identification_const(referer_adr), esc,
                                  sizeof(esc)));
        print_buffer(&bstr, "%s" H_CRLF,
                     escape_check_url_addr(referer_fil, esc, sizeof(esc)));
      }
    }
    // HTTP field: referer
    else if (strnotempty(retour->req.referer) &&
             !http_headers_have_field(custom, "Referer")) {
      print_buffer(&bstr, "Referer: %s"H_CRLF, retour->req.referer);
    }
    // POST?
    if (mode == 0) {            // GET!
      if (search_tag) {
        print_buffer(&bstr, "Content-length: %d" H_CRLF,
                (int) (strlen
                       (unescape_http
                        (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                         search_tag + strlen(POSTTOK) + 1))));
      }
    }
    // send stored cookies matching this host/path
    if (cookie != NULL && !http_headers_have_field(custom, "Cookie")) {
      append_cookie_header(&bstr, cookie, jump_identification_const(adr), fil);
    }
    // gérer le keep-alive (garder socket)
    if (retour->req.http11 && !retour->req.nokeepalive) {
      print_buffer(&bstr, "Connection: keep-alive" H_CRLF);
    } else {
      print_buffer(&bstr, "Connection: close" H_CRLF);
    }

    {
      const char *real_adr = jump_identification_const(adr);

      // Mandatory per RFC2616
      if (!direct_url &&
          !http_headers_have_field(custom, "Host")) { // not ftp:// for example
        print_buffer(&bstr, "Host: %s" H_CRLF,
                     escape_check_url_addr(real_adr, esc, sizeof(esc)));
      }

      // HTTP field: from
      if (strnotempty(retour->req.from) &&
          !http_headers_have_field(custom, "From")) { // HTTP from
        print_buffer(&bstr, "From: %s" H_CRLF, retour->req.from);
      }

      if (retour->req.user_agent_send && strnotempty(retour->req.user_agent) &&
          !http_headers_have_field(custom, "User-Agent")) {
        print_buffer(&bstr, "User-Agent: %s" H_CRLF, retour->req.user_agent);
      }

      // Accept
      if (strnotempty(retour->req.accept) &&
          !http_headers_have_field(custom, "Accept")) {
        print_buffer(&bstr, "Accept: %s" H_CRLF, retour->req.accept);
      }

      // Accept-language
      if (strnotempty(retour->req.lang_iso) &&
          !http_headers_have_field(custom, "Accept-Language")) {
        print_buffer(&bstr, "Accept-Language: %s"H_CRLF, retour->req.lang_iso);
      }

      // Compression accepted ?
      if (retour->req.http11 &&
          !http_headers_have_field(custom, "Accept-Encoding")) {
        hts_boolean compressible =
            (!retour->req.range_used && !retour->req.nocompression);
        hts_boolean secure = HTS_FALSE;

#if HTS_USEOPENSSL
        secure = retour->ssl ? HTS_TRUE : HTS_FALSE;
#endif
        print_buffer(&bstr, "Accept-Encoding: %s" H_CRLF,
                     hts_acceptencoding(compressible, secure));
      }

      /* Authentification */
      {
        char autorisation[1100];
        const char *a;

        autorisation[0] = '\0';
        if (link_has_authorization(adr)) {      // ohh une authentification!
          const char *a = jump_identification_const(adr);
          const char *astart = jump_protocol_const(adr);

          if (!direct_url) {    // pas ftp:// par exemple
            char user_pass[256];

            user_pass[0] = '\0';
            strncatbuff(user_pass, astart, (int) (a - astart) - 1);
            strcpybuff(user_pass, 
              unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), user_pass));
            code64((unsigned char *) user_pass, (int) strlen(user_pass),
                   (unsigned char *) autorisation, 0);
            if (strcmp(fil, "/robots.txt"))     /* pas robots.txt */
              bauth_add(cookie, astart, fil, autorisation);
          }
        } else if ((a = bauth_check(cookie, real_adr, fil)))
          strcpybuff(autorisation, a);
        if (strnotempty(autorisation) &&
            !http_headers_have_field(custom, "Authorization")) {
          print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
        }
      }
    }

    // Custom header(s), minus the fields the request already carries
    if (strnotempty(custom)) {
      append_custom_headers(&bstr, custom);
    }

    // CRLF de fin d'en tête
    print_buffer(&bstr, H_CRLF);

    // données complémentaires?
    if (search_tag)
      if (mode == 0)            // GET!
        print_buffer(&bstr, "%s",
                   unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                                 search_tag + strlen(POSTTOK) + 1));
  }
#if HDEBUG
#endif
  if (_DEBUG_HEAD) {
    if (ioinfo) {
      fprintf(ioinfo, "[%d] request for %s%s:\r\n", retour->debugid,
              jump_identification_const(adr), fil);
      fprintfio(ioinfo, bstr.buffer, "<<< ");
      fprintf(ioinfo, "\r\n");
      fflush(ioinfo);
    }
  }                             // Fin test pas postfile
  //

  // Stash the raw request for the WARC request record (freed at
  // back_clear_entry)
  if (StringNotEmpty(opt->warc_file))
    warc_stash_request(retour, bstr.buffer);

  // Callback
  {
    int test_head =
      RUN_CALLBACK6(opt, sendhead, bstr.buffer, adr, fil, referer_adr, referer_fil,
                    retour);
    if (test_head != 1) {
      deletesoc_r(retour);
      strcpybuff(retour->msg, "Header refused by external wrapper");
      retour->soc = INVALID_SOCKET;
    }
  }

  // Envoi
  HTS_STAT.last_request = mtime_local();
  if (sendc(retour, bstr.buffer) < 0) {        // ERREUR, socket rompue?...
    deletesoc_r(retour);        // fermer tout de même
    // et tenter de reconnecter

    strcpybuff(retour->msg, "Write error");
    retour->soc = INVALID_SOCKET;
  }

  // RX'98
  return 0;
}

// traiter 1ere ligne d'en tête
void treatfirstline(htsblk * retour, const char *rcvd) {
  const char *a = rcvd;

  // exemple:
  // HTTP/1.0 200 OK
  if (*a) {
    // note: certains serveurs buggés renvoient HTTP/1.0\n200 OK ou " HTTP/1.0 200 OK"
    while((*a == ' ') || (*a == 10) || (*a == 13) || (*a == 9))
      a++;                      // épurer espaces au début
    if (strfield(a, "HTTP/")) {
      // sauter HTTP/1.x
      while((*a != ' ') && (*a != '\0') && (*a != 10) && (*a != 13)
            && (*a != 9))
        a++;
      if (*a != '\0') {
        while((*a == ' ') || (*a == 10) || (*a == 13) || (*a == 9))
          a++;                  // épurer espaces
        if ((*a >= '0') && (*a <= '9')) {
          sscanf(a, "%d", &(retour->statuscode));
          // sauter 200
          while((*a != ' ') && (*a != '\0') && (*a != 10) && (*a != 13)
                && (*a != 9))
            a++;
          while((*a == ' ') || (*a == 10) || (*a == 13) || (*a == 9))
            a++;                // épurer espaces
          if ((strlen(a) > 1) && (strlen(a) < 64))      // message retour
            strcpybuff(retour->msg, a);
          else
            infostatuscode(retour->msg, retour->statuscode);
          // type MIME par défaut2
          strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
        } else {                // pas de code!
          retour->statuscode = STATUSCODE_INVALID;
          strcpybuff(retour->msg, "Unknown response structure");
        }
      } else {                  // euhh??
        retour->statuscode = STATUSCODE_INVALID;
        strcpybuff(retour->msg, "Unknown response structure");
      }
    } else {
      if (*a == '<') {
        /* This is dirty .. */
        retour->statuscode = HTTP_OK;
        retour->keep_alive = 0;
        strcpybuff(retour->msg, "Unknown, assuming junky server");
        strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
      } else if (strnotempty(a)) {
        retour->statuscode = STATUSCODE_INVALID;
        strcpybuff(retour->msg, "Unknown (not HTTP/xx) response structure");
      } else {
        /* This is dirty .. */
        retour->statuscode = HTTP_OK;
        retour->keep_alive = 0;
        strcpybuff(retour->msg, "Unknown, assuming junky server");
        strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
      }
    }
  } else {                      // vide!
    /*
       retour->statuscode=STATUSCODE_INVALID;
       strcpybuff(retour->msg,"Empty reponse or internal error");
     */
    /* This is dirty .. */
    retour->statuscode = HTTP_OK;
    strcpybuff(retour->msg, "Unknown, assuming junky server");
    strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
  }
}

// traiter ligne par ligne l'en tête
// gestion des cookies
void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
               char *rcvd) {
  int p;

  if ((p = strfield(rcvd, "Content-length:")) != 0) {
#if HDEBUG
    printf("ok, Content-length: détecté\n");
#endif
    if (sscanf(rcvd + p, LLintP, &(retour->totalsize)) == 1) {
      if (retour->totalsize == 0) {
        retour->empty = 1;
      }
    }
  } else if ((p = strfield(rcvd, "Content-Disposition:")) != 0) {
    while(is_realspace(*(rcvd + p)))
      p++;                      // sauter espaces
    if ((int) strlen(rcvd + p) < 250) { // pas trop long?
      char tmp[256];
      char *a = NULL;

      strcpybuff(tmp, rcvd + p);
      a = strstr(tmp, "filename=");
      if (a) {
        a += strlen("filename=");
        while(is_space(*a))
          a++;
        if (a) {
          char *c = NULL;

          while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
            a = c + 1;
          hts_rtrim(a, HTS_SPACES);
          if ((int) strlen(a) < 200) { // pas trop long?
            strcpybuff(retour->cdispo, a);
          }
        }
      }
    }
  } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
    while(is_realspace(*(rcvd + p)))
      p++;                      // sauter espaces
    if ((int) strlen(rcvd + p) < 64) { // pas trop long?
      strcpybuff(retour->lastmodified, rcvd + p);
    }
  } else if ((p = strfield(rcvd, "Date:")) != 0) {
    if (strnotempty(retour->lastmodified) == 0) {       /* pas encore de last-modified */
      while(is_realspace(*(rcvd + p)))
        p++;                    // sauter espaces
      if ((int) strlen(rcvd + p) < 64) { // pas trop long?
        strcpybuff(retour->lastmodified, rcvd + p);
      }
    }
  } else if ((p = strfield(rcvd, "Etag:")) != 0) {      /* Etag */
    if (retour) {
      while(is_realspace(*(rcvd + p)))
        p++;                    // sauter espaces
      if ((int) strlen(rcvd + p) < 64)  // pas trop long?
        strcpybuff(retour->etag, rcvd + p);
      else                      // erreur.. ignorer
        retour->etag[0] = '\0';
    }
  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
    while(is_realspace(*(rcvd + p)))
      p++;                      // sauter espaces
    if (strfield(rcvd + p, "chunked")) {
      retour->is_chunk = 1; // chunked
#if HDEBUG
      printf("ok, Transfer-Encoding: détecté\n");
#endif
    }
  } else if ((p = strfield(rcvd, "Content-type:")) != 0) {
    if (retour) {
      char tempo[1100];

      // éviter les text/html; charset=foo
      {
        char *a = strchr(rcvd + p, ';');

        if (a) {                // extended information
          *a = '\0';
          a++;
          while(is_space(*a))
            a++;
          if (strfield(a, "charset")) {
            a += 7;
            while(is_space(*a))
              a++;
            if (*a == '=') {
              a++;
              while(is_space(*a))
                a++;
              if (*a == '\"')
                a++;
              while(is_space(*a))
                a++;
              if (*a) {
                char *chs = a;

                while(*a && !is_space(*a) && *a != '\"' && *a != ';')
                  a++;
                *a = '\0';
                if (*chs) {
                  if (strlen(chs) < sizeof(retour->charset) - 2) {
                    strcpybuff(retour->charset, chs);
                  }
                }
              }
            }
          }
        }
      }
      // An empty/whitespace Content-Type value yields no token: keep the
      // sentinel default rather than reading an uninitialized tempo.
      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
        if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
          strcpybuff(retour->contenttype, tempo);
        else
          strcpybuff(retour->contenttype,
                     "application/octet-stream-unknown"); // erreur
      }
    }
  } else if ((p = strfield(rcvd, "Content-Range:")) != 0) {
    // Content-Range: bytes 0-70870/70871
    const char *a;

    for(a = rcvd + p; is_space(*a); a++) ;
    if (strncasecmp(a, "bytes ", 6) == 0) {
      for(a += 6; is_space(*a); a++) ;
      if (sscanf
          (a, LLintP "-" LLintP "/" LLintP, &retour->crange_start,
           &retour->crange_end, &retour->crange) != 3) {
        retour->crange_start = 0;
        retour->crange_end = 0;
        retour->crange = 0;
        a = strchr(rcvd + p, '/');
        if (a != NULL) {
          a++;
          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
            retour->crange_start = 0;
            retour->crange_end = retour->crange - 1;
          } else {
            retour->crange = 0;
          }
        }
      }
      // A valid Content-Range has no negative field; reject hostile values so
      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
      if (retour->crange_start < 0 || retour->crange_end < 0 ||
          retour->crange < 0) {
        retour->crange_start = retour->crange_end = retour->crange = 0;
      }
    }
  } else if ((p = strfield(rcvd, "Connection:")) != 0) {
    char *a = rcvd + p;

    while(is_space(*a))
      a++;
    if (*a) {
      if (strfield(a, "Keep-Alive")) {
        if (!retour->keep_alive) {
          retour->keep_alive_max = 10;
          retour->keep_alive_t = 15;
        }
        retour->keep_alive = 1;
      } else {
        retour->keep_alive = 0;
      }
    }
  } else if ((p = strfield(rcvd, "Keep-Alive:")) != 0) {
    char *a = rcvd + p;

    while(is_space(*a))
      a++;
    if (*a) {
      char *p;

      retour->keep_alive = 1;
      retour->keep_alive_max = 10;
      retour->keep_alive_t = 15;
      if ((p = strstr(a, "timeout="))) {
        p += strlen("timeout=");
        sscanf(p, "%d", &retour->keep_alive_t);
      }
      if ((p = strstr(a, "max="))) {
        p += strlen("max=");
        sscanf(p, "%d", &retour->keep_alive_max);
      }
      if (retour->keep_alive_max <= 1 || retour->keep_alive_t < 1) {
        retour->keep_alive = 0;
      }
    }
  } else if ((p = strfield(rcvd, "TE:")) != 0) {
    char *a = rcvd + p;

    while(is_space(*a))
      a++;
    if (*a) {
      if (strfield(a, "trailers")) {
        retour->keep_alive_trailers = 1;
      }
    }
  } else if ((p = strfield(rcvd, "Content-Encoding:")) != 0) {
    if (retour) {
      char tempo[1100];
      char *a = rcvd + p;

      while(is_space(*a))
        a++;
      {
        char *a = strchr(rcvd + p, ';');

        if (a)
          *a = '\0';
      }
      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
        strcpybuff(retour->contentencoding, tempo);
      else
        retour->contentencoding[0] = '\0';
      /* A coding to undo (or one we can not undo, which must fail the fetch
         rather than save the coded bytes as the page) */
      if (hts_codec_parse(retour->contentencoding) != HTS_CODEC_IDENTITY)
        retour->compressed = 1;
    }
  } else if ((p = strfield(rcvd, "Location:")) != 0) {
    if (retour) {
      if (retour->location) {
        while(is_realspace(*(rcvd + p)))
          p++; // skip spaces
        if (strlen(rcvd + p) < HTS_LOCATION_SIZE)
          strlcpybuff(retour->location, rcvd + p, HTS_LOCATION_SIZE);
        else {
          /* no opt here, so this only reaches a registered log callback */
          hts_log_print(NULL, LOG_WARNING,
                        "Location header too long (%d bytes), redirect ignored",
                        (int) strlen(rcvd + p));
          retour->location[0] = '\0';
        }
      }
    }
  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
             (cookie)) {        // ohh un cookie
    char *a = rcvd + p;         // pointeur
    char domain[256];           // domaine cookie (.netscape.com)
    char path[256];             // chemin (/)
    char cook_name[256];        // nom cookie (MYCOOK)
    char BIGSTK cook_value[8192];       // valeur (ID=toto,S=1234)

#if DEBUG_COOK
    printf("set-cookie detected\n");
#endif
    /* Over-long host overflows the fail-safe domain[] copy (abort); drop it. */
    if (adr && strlen(jump_identification_const(adr)) >= sizeof(domain))
      return;
    while(*a) {
      char *token_st, *token_end;
      char *value_st, *value_end;
      char name[256];
      char BIGSTK value[8192];
      int next = 0;

      name[0] = value[0] = '\0';
      //

      // initialiser cookie lu actuellement
      if (adr)
        strcpybuff(domain, jump_identification_const(adr));   // domaine
      strcpybuff(path, "/");    // chemin (/)
      strcpybuff(cook_name, "");        // nom cookie (MYCOOK)
      strcpybuff(cook_value, "");       // valeur (ID=toto,S=1234)
      // boucler jusqu'au prochain cookie ou la fin
      do {
        char *start_loop = a;

        while(is_space(*a))
          a++;                  // sauter espaces
        token_st = a;           // départ token
        while((!is_space(*a)) && (*a) && (*a != ';') && (*a != '='))
          a++;                  // arrêter si espace, point virgule
        token_end = a;
        while(is_space(*a))
          a++;                  // sauter espaces
        if (*a == '=') {        // name=value
          a++;
          while(is_space(*a))
            a++;                // sauter espaces
          value_st = a;
          while((*a != ';') && (*a))
            a++; // prochain ;
          value_end = a;
          // vérifier débordements
          if ((((int) (token_end - token_st)) < 200)
              && (((int) (value_end - value_st)) < 8000)
              && (((int) (token_end - token_st)) > 0)
              && (((int) (value_end - value_st)) > 0)) {
            int name_len = (int) (token_end - token_st);
            int value_len = (int) (value_end - value_st);

            name[0] = '\0';
            value[0] = '\0';
            strncatbuff(name, token_st, name_len);
            strncatbuff(value, value_st, value_len);
#if DEBUG_COOK
            printf("detected cookie-av: name=\"%s\" value=\"%s\"\n", name,
                   value);
#endif
            if (strfield2(name, "domain")) {
              if (value_len < sizeof(domain) - 1) {
                strcpybuff(domain, value);
              } else {
                cook_name[0] = 0;
                break;
              }
            } else if (strfield2(name, "path")) {
              if (value_len < sizeof(path) - 1) {
                strcpybuff(path, value);
              } else {
                cook_name[0] = 0;
                break;
              }
            } else if (strfield2(name, "max-age")) {
              // ignoré..
            } else if (strfield2(name, "expires")) {
              // ignoré..
            } else if (strfield2(name, "version")) {
              // ignoré..
            } else if (strfield2(name, "comment")) {
              // ignoré
            } else if (strfield2(name, "secure")) {     // ne devrait pas arriver ici
              // ignoré
            } else {
              if (value_len < sizeof(cook_value) - 1
                  && name_len < sizeof(cook_name) - 1) {
                if (strnotempty(cook_name) == 0) {      // noter premier: nom et valeur cookie
                  strcpybuff(cook_name, name);
                  strcpybuff(cook_value, value);
                } else {        // prochain cookie
                  a = start_loop;       // on devra recommencer à cette position
                  next = 1;     // enregistrer
                }
              } else {
                cook_name[0] = 0;
                break;
              }
            }
          }
        }
        if (!next) {
          while((*a != ';') && (*a))
            a++;                // prochain
          while(*a == ';')
            a++;                // sauter ;
        }
      } while((*a) && (!next));
      if (strnotempty(cook_name)) {     // cookie?
#if DEBUG_COOK
        printf
          ("new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
           cook_name, cook_value, domain, path);
#endif
        cookie_add(cookie, cook_name, cook_value, domain, path);
      }
    }
  }
}

// HTTP status code -> reason phrase (per RFC), or NULL if unknown.
HTSEXT_API const char *infostatuscode_const(int statuscode) {
  // O(1) dispatch (the compiler builds a jump table); the phrases are static.
  switch (statuscode) {
  case 100:
    return "Continue";
  case 101:
    return "Switching Protocols";
  case 200:
    return "OK";
  case 201:
    return "Created";
  case 202:
    return "Accepted";
  case 203:
    return "Non-Authoritative Information";
  case 204:
    return "No Content";
  case 205:
    return "Reset Content";
  case 206:
    return "Partial Content";
  case 300:
    return "Multiple Choices";
  case 301:
    return "Moved Permanently";
  case 302:
    return "Moved Temporarily";
  case 303:
    return "See Other";
  case 304:
    return "Not Modified";
  case 305:
    return "Use Proxy";
  case 306:
    return "Undefined 306 error";
  case 307:
    return "Temporary Redirect";
  case 400:
    return "Bad Request";
  case 401:
    return "Unauthorized";
  case 402:
    return "Payment Required";
  case 403:
    return "Forbidden";
  case 404:
    return "Not Found";
  case 405:
    return "Method Not Allowed";
  case 406:
    return "Not Acceptable";
  case 407:
    return "Proxy Authentication Required";
  case 408:
    return "Request Time-out";
  case 409:
    return "Conflict";
  case 410:
    return "Gone";
  case 411:
    return "Length Required";
  case 412:
    return "Precondition Failed";
  case 413:
    return "Request Entity Too Large";
  case 414:
    return "Request-URI Too Large";
  case 415:
    return "Unsupported Media Type";
  case 416:
    return "Requested Range Not Satisfiable";
  case 417:
    return "Expectation Failed";
  case 429:
    return "Too Many Requests";
  case 451:
    return "Unavailable For Legal Reasons";
  case 500:
    return "Internal Server Error";
  case 501:
    return "Not Implemented";
  case 502:
    return "Bad Gateway";
  case 503:
    return "Service Unavailable";
  case 504:
    return "Gateway Time-out";
  case 505:
    return "HTTP Version Not Supported";
  default:
    return NULL;
  }
}

// Write the status code's reason phrase into msg. For an unknown code, keep any
// caller-provided message, otherwise fall back to a default. Callers provide a
// buffer of at least 64 bytes (the longest reason phrase is 31).
HTSEXT_API void infostatuscode(char *msg, int statuscode) {
  const char *const text = infostatuscode_const(statuscode);

  if (text != NULL) {
    strlcpybuff(msg, text, 64);
  } else if (strnotempty(msg) == 0) {
    strlcpybuff(msg, "Unknown error", 64);
  }
}

// check if data is available
int check_readinput(htsblk * r) {
  if (r->soc != INVALID_SOCKET) {
    fd_set fds;                 // poll structures
    struct timeval tv;          // structure for select
    const int soc = (int) r->soc;

    assertf(soc == r->soc);
    FD_ZERO(&fds);
    FD_SET(soc, &fds);
    tv.tv_sec = 0;
    tv.tv_usec = 0;
    select(soc + 1, &fds, NULL, NULL, &tv);
    if (FD_ISSET(soc, &fds))
      return 1;
    else
      return 0;
  } else
    return 0;
}

// check if data is available
int check_readinput_t(T_SOC soc, int timeout) {
  if (soc != INVALID_SOCKET) {
    fd_set fds;                 // poll structures
    struct timeval tv;          // structure for select
    const int isoc = (int) soc;

    assertf(isoc == soc);
    FD_ZERO(&fds);
    FD_SET(isoc, &fds);
    tv.tv_sec = timeout;
    tv.tv_usec = 0;
    select(isoc + 1, &fds, NULL, NULL, &tv);
    if (FD_ISSET(isoc, &fds))
      return 1;
    else
      return 0;
  } else
    return 0;
}

// wait until the socket is writable, up to timeout seconds
int check_writeinput_t(T_SOC soc, int timeout) {
  if (soc != INVALID_SOCKET) {
    fd_set fds;
    struct timeval tv;
    const int isoc = (int) soc;

    assertf(isoc == soc);
    FD_ZERO(&fds);
    FD_SET(isoc, &fds);
    tv.tv_sec = timeout;
    tv.tv_usec = 0;
    select(isoc + 1, NULL, &fds, NULL, &tv);
    return FD_ISSET(isoc, &fds) ? 1 : 0;
  } else
    return 0;
}

/* Class the live errno now: back_wait() sees it after free() and fflush() and
   would rewrite this as a retryable connection error. */
static void classify_write_error(htsblk *r) {
  r->statuscode =
      check_fatal_io_errno() ? STATUSCODE_IO_FATAL : STATUSCODE_IO_ERROR;
  strcpybuff(r->msg, "Write error on disk");
}

// Read one block: bufl is a byte count, or one of the HTS_XFREAD_* line modes.
// Note: the +1 in the mallocs is the trailing NUL appended to the data.
LLint http_xfread1(htsblk * r, int bufl) {
  int nl = -1;

  // EOF
  if (r->totalsize >= 0 && r->size == r->totalsize) {
    return READ_EOF;
  }

  if (bufl > 0) {
    if (!r->is_write) {         // stocker en mémoire
      // In-memory content must fit a 32-bit index (allocs below add 1, reads
      // use int offsets): reject a hostile Content-Length or endless stream.
      const LLint inmem_want =
          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
      if (inmem_want >= INT32_MAX) {
        r->statuscode = STATUSCODE_INVALID;
        strcpybuff(r->msg, "In-memory content too large");
        return READ_ERROR;
      }
      if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
        if (r->adr == NULL) {
          r->adr = (char *) malloct((size_t) r->totalsize + 1);
          r->size = 0;
        }
        if (r->adr != NULL) {
          // lecture
          /* Signed first: a body longer than its announced totalsize makes
             this negative, and as a size_t it would drive a read past adr. */
          const LLint remaining = r->totalsize - r->size;
          const size_t req_size = remaining > 0 ? (size_t) remaining : 0;

          nl = req_size > 0 ? hts_read(r, r->adr + ((int) r->size), (int) req_size) : 0;        /* NO 32 bit overlow possible here (no 4GB html!) */
          // nouvelle taille
          if (nl >= 0)
            r->size += nl;

          /*
             if (r->size >= r->totalsize)
             nl = -1;  // break
           */

          r->adr[r->size] = '\0';       // caractère NULL en fin au cas où l'on traite des HTML
        }

      } else {                  // inconnu..
        // réserver de la mémoire?
        if (r->adr == NULL) {
#if HDEBUG
          printf("..alloc xfread\n");
#endif
          r->adr = (char *) malloct(bufl + 1);
          r->size = 0;
        } else {
#if HDEBUG
          printf("..realloc xfread1\n");
#endif
          r->adr = (char *) realloct(r->adr, (int) r->size + bufl + 1);
        }

        if (r->adr != NULL) {
          // lecture
          nl = hts_read(r, r->adr + (int) r->size, bufl);
          if (nl > 0) {
            // resize
            r->adr = (char *) realloct(r->adr, (int) r->size + nl + 1);
            // nouvelle taille
            r->size += nl;
            // octet nul
            if (r->adr)
              r->adr[r->size] = '\0';

          }                     // sinon on a fini
#if HDEBUG
          else if (nl < 0)
            printf("..end read (%d)\n", nl);
#endif
        }
#if HDEBUG
        else
          printf("..-> error\n");
#endif
      }

      // pas de adr=erreur
      if (r->adr == NULL)
        nl = READ_ERROR;

    } else {                    // stocker sur disque
      char *buff;

      buff = (char *) malloct(bufl);
      if (buff != NULL) {
        // lecture
        nl = hts_read(r, buff, bufl);
        // nouvelle taille
        if (nl > 0) {
          r->size += nl;
          if (!hts_fwrite_exact(buff, (size_t) nl, r->out)) {
            classify_write_error(r);
            nl = READ_ERROR;
          }
        }

        // libérer bloc tempo
        freet(buff);
      } else
        nl = READ_ERROR;

      /* The tail stdio held is written here, and glibc's later fclose() returns
         0 once this flush has taken the error. */
      if ((nl < 0) && (r->out != NULL) && fflush(r->out) != 0 &&
          !statuscode_is_write_error(r->statuscode)) {
        classify_write_error(r);
        nl = READ_ERROR;
      }

    }                           // stockage disque ou mémoire

  } else { // line modes: byte by byte, CR dropped
    int count = 256;
    int tot_nl = 0;
    int lf_detected = 0;
    int at_beginning = 1;

    do {
      nl = READ_INTERNAL_ERROR;
      count--;
      if (r->adr == NULL) {
        r->adr = (char *) malloct(HTS_LINE_BLOCK_SIZE);
        r->size = 0;
      }
      if (r->adr != NULL) {
        if (r->size < HTS_LINE_BLOCK_SIZE - 2) {
          // lecture
          nl = hts_read(r, r->adr + r->size, 1);
          if (nl > 0) {
            // exit on a blank line (LF seen twice, or LF as the first byte),
            // or on the first LF in HTS_XFREAD_LINE mode
            if (*(r->adr + r->size) == 10) {
              if (lf_detected || (at_beginning) || (bufl < 0))
                count = -1;
              lf_detected = 1;
            }
            if (*(r->adr + r->size) != 13) {    // sauter caractères 13
              if ((*(r->adr + r->size) != 10)
                  && (*(r->adr + r->size) != 13)
                ) {
                // restart for new line
                lf_detected = 0;
              }
              (r->size)++;
              at_beginning = 0;
            }
            *(r->adr + r->size) = '\0'; // terminer par octet nul
          }
        }
      }
      if (nl >= 0) {
        tot_nl += nl;
        if (!check_readinput(r))
          count = -1;
      }
    } while((nl >= 0) && (count > 0));
    if (nl >= 0) {
      nl = tot_nl;
    }
  }
  /* Ahead of the EOF test below: r->size is advanced above the fwrite that
     failed, so a body completed by that very read would report a clean EOF. */
  if (nl == READ_ERROR && statuscode_is_write_error(r->statuscode)) {
    return READ_ERROR;
  }
  // EOF
  if (r->totalsize >= 0 && r->size == r->totalsize) {
    return READ_EOF;
  } else {
    return nl;
  }
}

hts_boolean socket_set_nonblocking(T_SOC soc, hts_boolean nonblocking) {
#ifdef _WIN32
  unsigned long p = nonblocking ? 1 : 0;

  return ioctlsocket(soc, FIONBIO, &p) == 0 ? HTS_TRUE : HTS_FALSE;
#else
  const int flags = fcntl(soc, F_GETFL, 0);
  const int wanted = nonblocking ? (flags | O_NONBLOCK) : (flags & ~O_NONBLOCK);

  if (flags == -1 || fcntl(soc, F_SETFL, wanted) == -1)
    return HTS_FALSE;
  return HTS_TRUE;
#endif
}

int connect_socket_error(T_SOC soc) {
  int soerr = 0;
  socklen_t len = (socklen_t) sizeof(soerr);

  if (getsockopt(soc, SOL_SOCKET, SO_ERROR, (char *) &soerr, &len) != 0)
    return -1;
  return soerr;
}

// Crée un lien (http) vers une adresse internet iadr
// retour: structure (adresse, taille, message si erreur (si !adr))
// peut ouvrir avec des connect() non bloquants: waitconnect=0/1
T_SOC newhttp(httrackp * opt, const char *_iadr, htsblk * retour, int port,
              int waitconnect) {
  return newhttp_addr(opt, _iadr, retour, port, waitconnect, 0, NULL);
}

T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
                   int waitconnect, int addr_index, int *addr_count) {
  T_SOC soc;                    // descipteur de la socket

  if (addr_count != NULL) {
    *addr_count = 0;
  }

  if (strcmp(_iadr, "file://") != 0) {  /* non fichier */
    SOCaddr server;
    SOCaddr addrs[HTS_MAXADDRNUM];
    int naddr;
    const char *error = "unknown error";

    // tester un éventuel id:pass et virer id:pass@ si détecté
    const char *const iadr = jump_identification_const(_iadr);
    const char *resolve_host = iadr;
    char BIGSTK iadr2[HTS_URLMAXSIZE * 2];

    SOCaddr_clear(server);

#if HDEBUG
    printf("gethostbyname\n");
#endif

    // tester un éventuel port
    if (port == -1) {
      const char *a = jump_toport_const(iadr);

#if HTS_USEOPENSSL
      if (retour->ssl)
        port = 443;
      else
        port = 80;              // port par défaut
#else
      port = 80;                // port par défaut
#endif

      if (a != NULL) {
        // folding a nonsense port into 0..65535 crawls one neither the link nor
        // a port filter named; an empty "host:" just means the default (#614)
        if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
          if (retour != NULL) {
            snprintf(retour->msg, sizeof(retour->msg), "Invalid port: %s",
                     a + 1);
          }
          return INVALID_SOCKET;
        }

        iadr2[0] = '\0';
        // the address itself, without the ":port"
        strncatbuff(iadr2, iadr, (int) (a - iadr));
        resolve_host = iadr2;
      }
    }

    // resolve the full address list and pick the requested candidate; the
    // scheduler retries the next index when a connect fails (dead IPv6 etc.)
    naddr =
        hts_dns_resolve_all(opt, resolve_host, addrs, HTS_MAXADDRNUM, &error);
    if (addr_count != NULL) {
      *addr_count = naddr;
    }
    if (addr_index >= 0 && addr_index < naddr) {
      SOCaddr_copy_SOCaddr(server, addrs[addr_index]);
    }

    if (!SOCaddr_is_valid(server)) {
#if DEBUG
      printf("erreur gethostbyname\n");
#endif
      if (retour != NULL) {
#ifdef _WIN32
        snprintf(retour->msg, sizeof(retour->msg),
                 "Unable to get server's address: %s", error);
#else
        snprintf(retour->msg, sizeof(retour->msg),
                 "Unable to get server's address: %s", error);
#endif
      }
      return INVALID_SOCKET;
    }

    // make a copy for external clients
    SOCaddr_copy_SOCaddr(retour->address, server);
    retour->address_size = SOCaddr_size(retour->address);

    // créer ("attachement") une socket (point d'accès) internet,en flot
#if HDEBUG
    printf("socket\n");
#endif
#if HTS_WIDE_DEBUG
    DEBUG_W("socket\n");
#endif
    soc = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
    if (retour != NULL) {
      retour->debugid = HTS_STAT.stat_sockid++;
    }
#if HTS_WIDE_DEBUG
    DEBUG_W("socket()=%d\n" _(int) soc);
#endif
    if (soc == INVALID_SOCKET) {
      if (retour != NULL) {
#ifdef _WIN32
        int last_errno = WSAGetLastError();

        htsblk_failf(retour, "Unable to create a socket: %s",
                     strerror(last_errno));
#else
        int last_errno = errno;

        htsblk_failf(retour, "Unable to create a socket: %s",
                     strerror(last_errno));
#endif
      }
      return INVALID_SOCKET;    // erreur création socket impossible
    }
    // bind this address
    if (retour != NULL && strnotempty(retour->req.proxy.bindhost)) {
      const char *error = "unknown error";
      SOCaddr bind_addr;

      if (hts_dns_resolve2(opt, retour->req.proxy.bindhost,
                             &bind_addr, &error) == NULL
          || bind(soc, &SOCaddr_sockaddr(bind_addr), 
                  SOCaddr_size(bind_addr)) != 0) {
        snprintf(retour->msg, sizeof(retour->msg),
                 "Unable to bind the specificied server address: %s", error);
        deletesoc(soc);
        return INVALID_SOCKET;
      }
    }
    // structure: connexion au domaine internet, port 80 (ou autre)
    SOCaddr_initport(server, port);
#if HDEBUG
    printf("==%d\n", soc);
#endif

    // connexion non bloquante?
    if (!waitconnect) {
      if (!socket_set_nonblocking(soc, HTS_TRUE)) {
#ifdef _WIN32
        const int last_errno = WSAGetLastError();
#else
        const int last_errno = errno;
#endif
        snprintf(retour->msg, sizeof(retour->msg),
                 "Non-blocking socket failed: %s", strerror(last_errno));
        deletesoc(soc);
        return INVALID_SOCKET;
      }
    }
    // Connexion au serveur lui même
#if HDEBUG
    printf("connect\n");
#endif
    HTS_STAT.last_connect = mtime_local();

#if HTS_WIDE_DEBUG
    DEBUG_W("connect\n");
#endif
    if (connect(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
      // bloquant
      if (waitconnect) {
#if HDEBUG
        printf("unable to connect!\n");
#endif
        if (retour != NULL) {
#ifdef _WIN32
          const int last_errno = WSAGetLastError();

          htsblk_failf(retour, "Unable to connect to the server: %s",
                       strerror(last_errno));
#else
          const int last_errno = errno;

          htsblk_failf(retour, "Unable to connect to the server: %s",
                       strerror(last_errno));
#endif
        }
        /* Close the socket and notify the error!!! */
        deletesoc(soc);
        return INVALID_SOCKET;
      }
    }
#if HTS_WIDE_DEBUG
    DEBUG_W("connect done\n");
#endif

#if HDEBUG
    printf("connexion établie\n");
#endif

    // A partir de maintenant, on peut envoyer et recevoir des données
    // via le flot identifié par soc (socket): write(soc,adr,taille) et
    // read(soc,adr,taille)

  } else {                      // on doit ouvrir un fichier local!
    // il sera géré de la même manière qu'une socket (c'est idem!)

    soc = LOCAL_SOCKET_ID;      // pseudo-socket locale..
    // soc sera remplacé lors d'un http_fopen() par un handle véritable!

  }                             // teste fichier local ou http

  return soc;
}

// couper http://www.truc.fr/pub/index.html -> www.truc.fr /pub/index.html
// retour=-1 si erreur.
// si file://... alors adresse=file:// (et coupe le ?query dans ce cas)
int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
  int pos = 0;
  int scheme = 0;

  // effacer adrfil->adr et adrfil->fil
  adrfil->adr[0] = adrfil->fil[0] = '\0';

  // Reject an over-long URL: a root-relative path is copied whole into fil[].
  if (strlen(url) >= HTS_URLMAXSIZE * 2)
    return -1;

#if HDEBUG
  printf("protocol: %s\n", url);
#endif

  // Scheme?
  {
    const char *a = url;

    while(isalpha((unsigned char) *a))
      a++;
    if (*a == ':')
      scheme = 1;
  }

  // 1. optional scheme ":"
  if ((pos = strfield(url, "file:"))) { // fichier local!! (pour les tests)
    //!!p+=3;
    strcpybuff(adrfil->adr, "file://");
  } else if ((pos = strfield(url, "http:"))) {  // HTTP
    //!!p+=3;
  } else if ((pos = strfield(url, "ftp:"))) {   // FTP
    strcpybuff(adrfil->adr, "ftp://");  // FTP!!
    //!!p+=3;
#if HTS_USEOPENSSL
  } else if ((pos = strfield(url, "https:"))) {     // HTTPS
    strcpybuff(adrfil->adr, "https://");
#endif
  } else if (scheme) {
    return -1;                  // erreur non reconnu
  } else
    pos = 0;

  // 2. optional "//" authority
  if (strncmp(url + pos, "//", 2) == 0)
    pos += 2;

  // (url+pos) now points to the path (not net path)

  //## if (adrfil->adr[0]!=lOCAL_CHAR) {    // adrfil->adresse normale http
  if (!strfield(adrfil->adr, "file:")) {        // PAS adrfil->file://
    const char *p, *q;

    p = url + pos;

    // p pointe sur le début de l'adrfil->adresse, ex: www.truc.fr/sommaire/index.html
    q = strchr(jump_identification_const(p), '/');
    if (q == 0)
      q = strchr(jump_identification_const(p), '?');  // http://www.foo.com?bar=1
    if (q == 0)
      q = p + strlen(p);        // pointe sur \0
    // q pointe sur le chemin, ex: index.html?query=recherche

    // chemin www... trop long!!
    if ((((int) (q - p))) > HTS_URLMAXSIZE) {
      return -1;                // erreur
    }
    // recopier adrfil->adresse www..
    strncatbuff(adrfil->adr, p, ((int) (q - p)));
    // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!

    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
      return -1;

    // recopier chemin /pub/..
    if (q[0] != '/')            // page par défaut (/)
      strcatbuff(adrfil->fil, "/");
    strcatbuff(adrfil->fil, q);
    // SECURITE:
    // simplifier url pour les ../
    fil_simplifie(adrfil->fil);
  } else {                      // localhost adrfil->file://
    const char *p;
    size_t i;
    char *a;

    p = url + pos;
    if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
      strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
    } else {
      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
        strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
        strcatbuff(adrfil->fil, p);
      } else {
        strcatbuff(adrfil->fil, p);     // adrfil->file://C:\..
      }
    }

    a = strchr(adrfil->fil, '?');
    if (a)
      *a = '\0';                /* couper query (inutile pour adrfil->file:// lors de la requête) */
    // adrfil->filtrer les \\ -> / pour les fichiers DOS
    for(i = 0; adrfil->fil[i] != '\0'; i++)
      if (adrfil->fil[i] == '\\')
        adrfil->fil[i] = '/';
    // collapse ../ like the http branch above (path-traversal safety)
    fil_simplifie(adrfil->fil);
  }

  // no hostname
  if (!strnotempty(adrfil->adr))
    return -1;                  // erreur non reconnu

  // nommer au besoin.. (non utilisé normalement)
  if (!strnotempty(adrfil->fil))
    strcpybuff(adrfil->fil, "default-index.html");

  // case insensitive pour adrfil->adresse
  {
    char *a = jump_identification(adrfil->adr);

    while(*a) {
      if ((*a >= 'A') && (*a <= 'Z'))
        *a += 'a' - 'A';
      a++;
    }
  }

  return 0;
}

/* simplify ../ and ./ */
void fil_simplifie(char *f) {
  char *a, *b;
  char *rollback[128];
  int rollid = 0;
  char lc = '/';
  int query = 0;
  int wasAbsolute = (*f == '/');

  for(a = b = f; *a != '\0';) {
    if (*a == '?')
      query = 1;
    if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '/') {        /* foo/./bar or ./foo  */
      a += 2;
    } else if (query == 0 && lc == '/' && a[0] == '.' && a[1] == '.' && (a[2] == '/' || a[2] == '\0')) {        /* foo/../bar or ../foo or .. */
      if (a[2] == '\0')
        a += 2;
      else
        a += 3;
      if (rollid > 1) {
        rollid--;
        b = rollback[rollid - 1];
      } else {                  /* too many ../ */
        rollid = 0;
        b = f;
        if (wasAbsolute)
          b++;                  /* after the / */
      }
    } else {
      *b++ = lc = *a;
      if (*a == '/') {
        rollback[rollid++] = b;
        if (rollid >= 127) {
          *f = '\0';            /* ERROR */
          break;
        }
      }
      a++;
    }
  }
  *b = '\0';
  if (*f == '\0') {
    if (wasAbsolute) {
      f[0] = '/';
      f[1] = '\0';
    } else {
      f[0] = '.';
      f[1] = '/';
      f[2] = '\0';
    }
  }
}

// fermer liaison fichier ou socket
void deletehttp(htsblk * r) {
#if HTS_DEBUG_CLOSESOCK
  DEBUG_W("deletehttp: (htsblk*) 0x%p\n" _(void *)r);
#endif
#if HTS_USEOPENSSL
  /* Free OpenSSL structures */
  if (r->ssl_con) {
    SSL_shutdown(r->ssl_con);
    SSL_free(r->ssl_con);
    r->ssl_con = NULL;
  }
#endif
  if (r->soc != INVALID_SOCKET) {
    if (r->is_file) {
      if (r->fp)
        fclose(r->fp);
      r->fp = NULL;
    } else {
      if (r->soc != LOCAL_SOCKET_ID)
        deletesoc_r(r);
    }
    r->soc = INVALID_SOCKET;
  }
}

// free the addr buffer
// always returns 1
int deleteaddr(htsblk * r) {
  if (r->adr != NULL) {
    freet(r->adr);
    r->adr = NULL;
  }
  if (r->headers != NULL) {
    freet(r->headers);
    r->headers = NULL;
  }
  return 1;
}

// fermer une socket
void deletesoc(T_SOC soc) {
  if (soc != INVALID_SOCKET && soc != LOCAL_SOCKET_ID) {
#if HTS_WIDE_DEBUG
    DEBUG_W("close %d\n" _(int) soc);
#endif
#ifdef _WIN32
    if (closesocket(soc) != 0) {
      int err = WSAGetLastError();

      fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
              strerror(err));
    }
#else
    if (close(soc) != 0) {
      const int err = errno;

      fprintf(stderr, "* error closing socket " T_SOCP ": %s\n", soc,
              strerror(err));
    }
#endif
#if HTS_WIDE_DEBUG
    DEBUG_W(".. done\n");
#endif
  }
}

/* Will also clean other things */
void deletesoc_r(htsblk * r) {
#if HTS_USEOPENSSL
  if (r->ssl_con) {
    SSL_shutdown(r->ssl_con);
    SSL_free(r->ssl_con);
    r->ssl_con = NULL;
  }
#endif
  if (r->soc != INVALID_SOCKET) {
    deletesoc(r->soc);
    r->soc = INVALID_SOCKET;
  }
}

// renvoi le nombre de secondes depuis 1970
TStamp time_local(void) {
  return ((TStamp) time(NULL));
}

// number of millisec since 1970
HTSEXT_API TStamp mtime_local(void) {
#ifndef _WIN32
  struct timeval tv;
  if (gettimeofday(&tv, NULL) != 0) {
    assert(! "gettimeofday");
  }

  return (TStamp) (((TStamp) tv.tv_sec * (TStamp) 1000) +
                   ((TStamp) tv.tv_usec / (TStamp) 1000));
#else
  struct timeb B;
  ftime(&B);
  return (TStamp) (((TStamp) B.time * (TStamp) 1000)
                   + ((TStamp) B.millitm));
#endif
}

// convertit un nombre de secondes en temps (chaine)
void sec2str(char *st, TStamp t) {
  int j, h, m, s;

  j = (int) (t / (3600 * 24));
  t -= ((TStamp) j) * (3600 * 24);
  h = (int) (t / (3600));
  t -= ((TStamp) h) * 3600;
  m = (int) (t / 60);
  t -= ((TStamp) m) * 60;
  s = (int) t;

  if (j > 0)
    sprintf(st, "%d days, %d hours %d minutes %d seconds", j, h, m, s);
  else if (h > 0)
    sprintf(st, "%d hours %d minutes %d seconds", h, m, s);
  else if (m > 0)
    sprintf(st, "%d minutes %d seconds", m, s);
  else
    sprintf(st, "%d seconds", s);
}

// idem, plus court (chaine)
HTSEXT_API void qsec2str(char *st, TStamp t) {
  int j, h, m, s;

  j = (int) (t / (3600 * 24));
  t -= ((TStamp) j) * (3600 * 24);
  h = (int) (t / (3600));
  t -= ((TStamp) h) * 3600;
  m = (int) (t / 60);
  t -= ((TStamp) m) * 60;
  s = (int) t;

  if (j > 0)
    sprintf(st, "%dd,%02dh,%02dmin%02ds", j, h, m, s);
  else if (h > 0)
    sprintf(st, "%dh,%02dmin%02ds", h, m, s);
  else if (m > 0)
    sprintf(st, "%dmin%02ds", m, s);
  else
    sprintf(st, "%ds", s);
}

// heure actuelle, GMT, format rfc (taille buffer 256o)
void time_gmt_rfc822(char *s) {
  time_t tt;
  struct tm tmv;

  tt = time(NULL);
  /* no local-time fallback: it would format local time and still label it GMT
   * (#806) */
  if (hts_gmtime(tt, &tmv))
    time_rfc822(s, &tmv);
  else
    s[0] = '\0';
}

void hts_now_iso8601(char out[32]) {
  time_t t = time(NULL);
  struct tm tmv;

  if (!hts_gmtime(t, &tmv))
    memset(&tmv, 0, sizeof(tmv));
  strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &tmv);
}

// heure actuelle, format rfc (taille buffer 256o)
void time_local_rfc822(char *s) {
  time_t tt;
  struct tm tmv;

  tt = time(NULL);
  if (hts_localtime(tt, &tmv))
    time_rfc822_local(s, &tmv);
  else
    s[0] = '\0';
}

/* convertir une chaine en temps */
struct tm *convert_time_rfc822(struct tm *result, const char *s) {
  char months[] = "jan feb mar apr may jun jul aug sep oct nov dec";
  char str[256];
  char *a;

  /* */
  int result_mm = -1;
  int result_dd = -1;
  int result_n1 = -1;
  int result_n2 = -1;
  int result_n3 = -1;
  int result_n4 = -1;

  /* */

  if ((int) strlen(s) > 200)
    return NULL;
  strcpybuff(str, s);
  hts_lowcase(str);
  /* éliminer :,- */
  while((a = strchr(str, '-')))
    *a = ' ';
  while((a = strchr(str, ':')))
    *a = ' ';
  while((a = strchr(str, ',')))
    *a = ' ';
  /* tokeniser */
  a = str;
  while(*a) {
    char *first, *last;
    char tok[256];

    /* découper mot */
    while(*a == ' ')
      a++;                      /* sauter espaces */
    first = a;
    while((*a) && (*a != ' '))
      a++;
    last = a;
    tok[0] = '\0';
    if (first != last) {
      char *pos;

      strncatbuff(tok, first, (int) (last - first));
      /* analyser */
      if ((pos = strstr(months, tok))) {        /* month always in letters */
        result_mm = ((int) (pos - months)) / 4;
      } else {
        int number;

        if (sscanf(tok, "%d", &number) == 1) {  /* number token */
          if (result_dd < 0)    /* day always first number */
            result_dd = number;
          else if (result_n1 < 0)
            result_n1 = number;
          else if (result_n2 < 0)
            result_n2 = number;
          else if (result_n3 < 0)
            result_n3 = number;
          else if (result_n4 < 0)
            result_n4 = number;
        }                       /* sinon, bruit de fond(+1GMT for exampel) */
      }
    }
  }
  if ((result_n1 >= 0) && (result_mm >= 0) && (result_dd >= 0)
      && (result_n2 >= 0) && (result_n3 >= 0) && (result_n4 >= 0)) {
    if (result_n4 >= 1000) {    /* Sun Nov  6 08:49:37 1994 */
      result->tm_year = result_n4 - 1900;
      result->tm_hour = result_n1;
      result->tm_min = result_n2;
      result->tm_sec = max(result_n3, 0);
    } else {                    /* Sun, 06 Nov 1994 08:49:37 GMT or Sunday, 06-Nov-94 08:49:37 GMT */
      result->tm_hour = result_n2;
      result->tm_min = result_n3;
      result->tm_sec = max(result_n4, 0);
      if (result_n1 <= 50)      /* 00 means 2000 */
        result->tm_year = result_n1 + 100;
      else if (result_n1 < 1000)        /* 99 means 1999 */
        result->tm_year = result_n1;
      else                      /* 2000 */
        result->tm_year = result_n1 - 1900;
    }
    result->tm_isdst = 0;       /* assume GMT */
    result->tm_yday = -1;       /* don't know */
    result->tm_wday = -1;       /* don't know */
    result->tm_mon = result_mm;
    result->tm_mday = result_dd;
    return result;
  }
  return NULL;
}

static time_t getGMT(struct tm *tm) {
  time_t t = timegm(tm);

  if (t != (time_t) - 1 && t != (time_t) 0) {
    return (time_t) t;
  }
  return (time_t) -1;
}

/* sets file time. -1 if error */
/* Note: utf-8 */
int set_filetime(const char *file, struct tm *tm_time) {
  time_t t = getGMT(tm_time);

  if (t != (time_t) - 1) {
    STRUCT_UTIMBUF tim;

    memset(&tim, 0, sizeof(tim));
    tim.actime = tim.modtime = t;
    return UTIME(file, &tim);
  }
  return -1;
}

/* sets file time from RFC822 date+time, -1 if error*/
/* Note: utf-8 */
int set_filetime_rfc822(const char *file, const char *date) {
  struct tm buffer;
  struct tm *tm_s = convert_time_rfc822(&buffer, date);

  if (tm_s) {
    return set_filetime(file, tm_s);
  } else
    return -1;
}

/* Note: utf-8 */
time_t get_filetime(const char *file) {
  STRUCT_STAT buf;

  if (STAT(file, &buf) != 0)
    return (time_t) -1;
  return (time_t) buf.st_mtime;
}

/* Note: utf-8 */
int get_filetime_rfc822(const char *file, char *date) {
  STRUCT_STAT buf;

  date[0] = '\0';
  if (STAT(file, &buf) == 0) {
    struct tm tmv;
    time_t tt = buf.st_mtime;

    /* no local-time fallback: it would format local time and still label it GMT
     * (#806) */
    if (hts_gmtime(tt, &tmv)) {
      time_rfc822(date, &tmv);
      return 1;
    }
  }
  return 0;
}

// heure au format rfc (taille buffer 256o)
void time_rfc822(char *s, struct tm *A) {
  if (A == NULL) {
    int localtime_returned_null = 0;

    assertf(localtime_returned_null);
  }
  strftime(s, 256, "%a, %d %b %Y %H:%M:%S GMT", A);
}

// heure locale au format rfc (taille buffer 256o)
void time_rfc822_local(char *s, struct tm *A) {
  if (A == NULL) {
    int localtime_returned_null = 0;

    assertf(localtime_returned_null);
  }
  strftime(s, 256, "%a, %d %b %Y %H:%M:%S", A);
}

// conversion en b,Kb,Mb
HTSEXT_API char *int2bytes(strc_int2bytes2 * strc, LLint n) {
  char **a = int2bytes2(strc, n);

  strcpybuff(strc->catbuff, a[0]);
  strcatbuff(strc->catbuff, a[1]);
  return strc->catbuff;
}

// conversion en b/s,Kb/s,Mb/s
HTSEXT_API char *int2bytessec(strc_int2bytes2 * strc, long int n) {
  char buff[256];
  char **a = int2bytes2(strc, n);

  strcpybuff(buff, a[0]);
  strcatbuff(buff, a[1]);
  return concat(strc->catbuff, sizeof(strc->catbuff), buff, "/s");
}
HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
  sprintf(strc->buff2, "%d", n);
  return strc->buff2;
}

// conversion en b,Kb,Mb, nombre et type séparés
// limite: 2.10^9.10^6B

/* See http://physics.nist.gov/cuu/Units/binary.html */
#define ToLLint(a) ((LLint)(a))
#define ToLLintKiB (ToLLint(1024))
#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
#define ToLLintPiB                                                             \
  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
  if (n < ToLLintKiB) {
    sprintf(strc->buff1, "%d", (int) (LLint) n);
    strcpybuff(strc->buff2, "B");
  } else if (n < ToLLintMiB) {
    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
            (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
    strcpybuff(strc->buff2, "KiB");
  } else if (n < ToLLintGiB) {
    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
    strcpybuff(strc->buff2, "MiB");
  } else if (n < ToLLintTiB) {
    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintGiB))),
            (int) ((LLint) (((n % (ToLLintGiB)) * 100) / (ToLLintGiB))));
    strcpybuff(strc->buff2, "GiB");
  } else if (n < ToLLintPiB) {
    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintTiB))),
            (int) ((LLint) (((n % (ToLLintTiB)) * 100) / (ToLLintTiB))));
    strcpybuff(strc->buff2, "TiB");
  } else {
    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintPiB))),
            (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
    strcpybuff(strc->buff2, "PiB");
  }
  strc->buffadr[0] = strc->buff1;
  strc->buffadr[1] = strc->buff2;
  return strc->buffadr;
}

#ifdef _WIN32
#else
// ignore sigpipe?
int sig_ignore_flag(int setflag) {      // flag ignore
  static int flag = 0;          /* YES, this one is true static */

  if (setflag >= 0)
    flag = setflag;
  return flag;
}
#endif

// envoi de texte (en têtes généralement) sur la socket soc
int sendc(htsblk * r, const char *s) {
  int n, ssz = (int) strlen(s);

#ifdef _WIN32
#else
  sig_ignore_flag(1);
#endif
#if HDEBUG
  write(0, s, ssz);
#endif

#if HTS_USEOPENSSL
  if (r->ssl) {
    n = SSL_write(r->ssl_con, s, ssz);
  } else
#endif
    n = send(r->soc, s, ssz, 0);

#ifdef _WIN32
#else
  sig_ignore_flag(0);
#endif

  return (n == ssz) ? n : -1;
}

// Like linput, but in memory (optimized). A line too long for `s` is clipped
// but consumed whole: resuming inside it reads its tail back as another line.
int binput(const char *buff, char *s, int max) {
  int count = 0;
  int destCount = 0;

  // Note: \0 will return 1
  while (buff != NULL && buff[count] != '\0' && buff[count] != '\n') {
    if (buff[count] != '\r' && destCount < max) {
      s[destCount++] = buff[count];
    }
    count++;
  }
  s[destCount] = '\0';

  // then return the supplemental jump offset
  return count + 1;
}

hts_boolean binput_line(const char *buff, const char *end, char *s, int max,
                        int *offset) {
  hts_boolean cut = HTS_FALSE;
  const char *p;
  int j = 0;

  *offset = 0;
  s[0] = '\0';
  if (buff == NULL || buff >= end)
    return HTS_FALSE;
  for (p = buff; p < end && *p != '\0' && *p != '\n'; p++) {
    if (*p == '\r')
      continue;
    if (j < max - 1)
      s[j++] = *p;
    else
      cut = HTS_TRUE; /* keep walking: the tail must not become more headers */
  }
  s[j] = '\0';
  *offset = (int) (p - buff) + 1; /* binput's convention: step over the \n */
  return cut;
}

hts_boolean finput_line(T_SOC fd, char *s, int max) {
  hts_boolean cut = HTS_FALSE;
  int j = 0;
  char c;

  for (;;) {
    if (read((int) fd, &c, 1) <= 0 || c == 10)
      break;
    if (c == 13)
      continue;
    if (j < max - 1)
      s[j++] = c;
    else
      cut = HTS_TRUE; /* keep draining: the tail must not become more headers */
  }
  s[j] = '\0';
  return cut;
}

// Lecture d'une ligne (peut être unicode à priori)
int linput(FILE * fp, char *s, int max) {
  int c;
  int j = 0;

  do {
    c = fgetc(fp);
    if (c != EOF) {
      switch (c) {
      case 13:
        break;                  // sauter CR
      case 10:
        c = -1;
        break;
      case 9:
      case 12:
        break;                  // sauter ces caractères
      default:
        s[j++] = (char) c;
        break;
      }
    }
  } while((c != -1) && (c != EOF) && (j < (max - 1)));
  s[j] = '\0';
  return j;
}
int linputsoc(T_SOC soc, char *s, int max) {
  int c;
  int j = 0;

  do {
    unsigned char ch;

    if (recv(soc, &ch, 1, 0) == 1) {
      c = ch;
    } else {
      c = EOF;
    }
    if (c != EOF) {
      switch (c) {
      case 13:
        break;                  // sauter CR
      case 10:
        c = -1;
        break;
      case 9:
      case 12:
        break;                  // sauter ces caractères
      default:
        s[j++] = (char) c;
        break;
      }
    }
  } while((c != -1) && (c != EOF) && (j < (max - 1)));
  s[j] = '\0';
  return j;
}
int linputsoc_t(T_SOC soc, char *s, int max, int timeout) {
  if (check_readinput_t(soc, timeout)) {
    return linputsoc(soc, s, max);
  }
  return -1;
}
int linput_trim(FILE * fp, char *s, int max) {
  int rlen = 0;
  char *ls = (char *) malloct(max + 1);

  s[0] = '\0';
  if (ls) {
    char *a;

    // lire ligne
    rlen = linput(fp, ls, max);
    if (rlen) {
      // sauter espaces et tabs en fin
      while((rlen > 0)
            && ((ls[max(rlen - 1, 0)] == ' ')
                || (ls[max(rlen - 1, 0)] == '\t')))
        ls[--rlen] = '\0';
      // sauter espaces en début
      a = ls;
      while((rlen > 0) && ((*a == ' ') || (*a == '\t'))) {
        a++;
        rlen--;
      }
      if (rlen > 0) {
        memcpy(s, a, rlen);     // can copy \0 chars
        s[rlen] = '\0';
      }
    }
    //
    freet(ls);
  }
  return rlen;
}
int linput_cpp(FILE * fp, char *s, int max) {
  int rlen = 0;

  s[0] = '\0';
  do {
    int ret;

    if (rlen > 0)
      if (s[rlen - 1] == '\\')
        s[--rlen] = '\0';       // couper \ final
    // lire ligne
    ret = linput_trim(fp, s + rlen, max - rlen);
    if (ret > 0)
      rlen += ret;
  } while((s[max(rlen - 1, 0)] == '\\') && (rlen < max));
  return rlen;
}

// idem avec les car spéciaux
void rawlinput(FILE * fp, char *s, int max) {
  int c;
  int j = 0;

  do {
    c = fgetc(fp);
    if (c != EOF) {
      switch (c) {
      case 13:
        break;                  // sauter CR
      case 10:
        c = -1;
        break;
      default:
        s[j++] = (char) c;
        break;
      }
    }
  } while((c != -1) && (c != EOF) && (j < (max - 1)));
  s[j++] = '\0';
}

//cherche chaine, case insensitive
const char *strstrcase(const char *s, const char *o) {
  while(*s && strfield(s, o) == 0)
    s++;
  if (*s == '\0')
    return NULL;
  return s;
}

// Unicode detector
// See http://www.unicode.org/unicode/reports/tr28/
// (sect Table 3.1B. Legal UTF-8 Byte Sequences)
typedef struct {
  unsigned int pos;
  unsigned char data[4];
} t_auto_seq;

// char between a and b
#define CHAR_BETWEEN(c, a, b)       ( (c) >= 0x##a ) && ( (c) <= 0x##b )
// sequence start
#define SEQBEG                      ( inseq == 0 )
// in this block
#define BLK(n,a, b)                 ( (seq.pos >= n) && ((err = CHAR_BETWEEN(seq.data[n], a, b))) )
#define ELT(n,a)                    BLK(n,a,a)
// end
#define SEQEND                      ((ok = 1))
// sequence started, character will fail if error
#define IN_SEQ                      ( (inseq = 1) )
// decoding error
#define BAD_SEQ                     ( (ok == 0) && (inseq != 0) && (!err) )
// no sequence started
#define NO_SEQ                      ( inseq == 0 )

void map_characters(unsigned char *buffer, unsigned int size, unsigned int *map) {
  unsigned int i;

  memset(map, 0, sizeof(unsigned int) * 256);
  for(i = 0; i < size; i++) {
    map[buffer[i]]++;
  }
}

// le fichier est-il un fichier html?
//  0 : non
//  1 : oui
// -1 : on sait pas
// -2 : on sait pas, pas d'extension
int ishtml(httrackp * opt, const char *fil) {
  /* User-defined MIME types (overrides ishtml()) */
  char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2];
  char mime[256];
  char *a;

  strcpybuff(fil_noquery, fil);
  if ((a = strchr(fil_noquery, '?')) != NULL) {
    *a = '\0';
  }
  if (get_userhttptype(opt, mime, sizeof(mime), fil_noquery)) {
    if (is_html_mime_type(mime)) {
      return 1;
    } else {
      return 0;
    }
  }

  if (!strnotempty(fil_noquery)) {
    return -2;
  }

  /* Search for known ext */
  for (a = hts_lastcharptr(fil_noquery);
       *a != '.' && *a != '/' && a > fil_noquery; a--)
    ;
  if (*a == '.') {              // a une extension
    char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2];
    char *b;
    int ret;
    char *dotted = a;

    fil_noquery[0] = '\0';
    a++;                        // pointer sur extension
    strncatbuff(fil_noquery, a, HTS_URLMAXSIZE);
    b = strchr(fil_noquery, '?');
    if (b)
      *b = '\0';
    ret = ishtml_ext(fil_noquery);      // retour
    if (ret == -1) {
      switch (is_knowntype(opt, dotted)) {
      case 1:
        ret = 0;                // connu, non html
        break;
      case 2:
        ret = 1;                // connu, html
        break;
      default:
        ret = -1;               // inconnu..
        break;
      }
    }
    return ret;
  } else
    return -2;                  // indéterminé, par exemple /truc
}

// idem, mais pour uniquement l'extension
int ishtml_ext(const char *a) {
  int html = 0;

  //
  if (strfield2(a, "html"))
    html = 1;
  else if (strfield2(a, "htm"))
    html = 1;
  else if (strfield2(a, "shtml"))
    html = 1;
  else if (strfield2(a, "phtml"))
    html = 1;
  else if (strfield2(a, "htmlx"))
    html = 1;
  else if (strfield2(a, "shtm"))
    html = 1;
  else if (strfield2(a, "phtm"))
    html = 1;
  else if (strfield2(a, "htmx"))
    html = 1;
  //
  // insuccès..
  else {
#if 1
    html = -1;                  // inconnu..
#else
    // XXXXXX not suitable (ext)
    switch (is_knownext(a)) {
    case 1:
      html = 0;                 // connu, non html
      break;
    case 2:
      html = 1;                 // connu, html
      break;
    default:
      html = -1;                // inconnu..
      break;
    }
#endif
  }
  return html;
}

// error (404,500..)
int ishttperror(int err) {
  switch (err / 100) {
  case 4:
  case 5:
    return 1;
    break;
  }
  return 0;
}

/* Declare a non-const version of FUN */
#define DECLARE_NON_CONST_VERSION(FUN) \
char *FUN(char *source) { \
  const char *const ret = FUN ##_const(source); \
  return ret != NULL ? source + ( ret - source ) : NULL; \
}

// retourne le pointeur ou le pointeur + offset si il existe dans la chaine un @ signifiant 
// une identification
HTSEXT_API const char *jump_identification_const(const char *source) {
  const char *a, *trytofind;

  if (strcmp(source, "file://") == 0)
    return source;
  // rechercher dernier @ (car parfois email transmise dans adresse!)
  // mais sauter ftp:// éventuel
  a = jump_protocol_const(source);
  trytofind = strrchr_limit(a, '@', strchr(a, '/'));
  return trytofind != NULL ? trytofind : a;
}

HTSEXT_API DECLARE_NON_CONST_VERSION(jump_identification)

HTSEXT_API const char *jump_normalized_const(const char *source) {
  if (strcmp(source, "file://") == 0)
    return source;
  source = jump_identification_const(source);
  if (strfield(source, "www") && source[3] != '\0') {
    if (source[3] == '.') {     // www.foo.com -> foo.com
      source += 4;
    } else {                    // www-4.foo.com -> foo.com
      const char *a = source + 3;

      while(*a && (isdigit(*a) || *a == '-'))
        a++;
      if (*a == '.') {
        source = a + 1;
      }
    }
  }
  return source;
}

HTSEXT_API DECLARE_NON_CONST_VERSION(jump_normalized)

static int sortNormFnc(const void *a_, const void *b_) {
  const char *const*const a = (const char *const*) a_;
  const char *const*const b = (const char *const*) b_;

  return strcmp(*a + 1, *b + 1);
}

/* Path normalizer core: optionally collapse redundant '//' (DO_SLASH) and/or
   sort query arguments (DO_QUERY) so equivalent URLs dedupe. */
static char *fil_normalized_ex(const char *source, char *dest, int do_slash,
                               int do_query) {
  char lastc = 0;
  int gotquery = 0;
  size_t ampargs = 0;
  size_t i, j;
  char *query = NULL;

  for(i = j = 0; source[i] != '\0'; i++) {
    if (!gotquery && source[i] == '?') {
      gotquery = 1;
      ampargs = 1;
    }
    if (do_slash && !gotquery && lastc == '/' && source[i] == '/') {
      // foo//bar -> foo/bar
    } else {
      if (gotquery && source[i] == '&') {
        ampargs++;
      }
      dest[j++] = source[i];
    }
    lastc = source[i];
  }
  dest[j++] = '\0';

  /* Sort arguments (&foo=1&bar=2 == &bar=2&foo=1) */
  if (do_query && ampargs > 1) {
    char **amps = malloct(ampargs * sizeof(char *));
    char *copyBuff = NULL;
    size_t qLen = 0;

    assertf(amps != NULL);
    gotquery = 0;
    for(i = j = 0; dest[i] != '\0'; i++) {
      if ((gotquery && dest[i] == '&') || (!gotquery && dest[i] == '?')) {
        if (!gotquery) {
          gotquery = 1;
          query = &dest[i];
          qLen = strlen(query);
        }
        assertf(j < ampargs);
        amps[j++] = &dest[i];
        dest[i] = '\0';
      }
    }
    assertf(gotquery);
    assertf(j == ampargs);

    /* Sort 'em all */
    qsort(amps, ampargs, sizeof(char *), sortNormFnc);

    /* Replace query by sorted query */
    copyBuff = malloct(qLen + 1);
    assertf(copyBuff != NULL);
    {
      htsbuff cb = htsbuff_ptr(copyBuff, qLen + 1);

      for (i = 0; i < ampargs; i++) {
        htsbuff_cat(&cb, i == 0 ? "?" : "&");
        htsbuff_cat(&cb, amps[i] + 1);
      }
      assertf(cb.len == qLen);
    }
    /* query points into dest where the original qLen-byte query was */
    strlcpybuff(query, copyBuff, qLen + 1);

    /* Cleanup */
    freet(amps);
    freet(copyBuff);
  }

  return dest;
}

HTSEXT_API char *fil_normalized(const char *source, char *dest) {
  return fil_normalized_ex(source, dest, 1, 1);
}

/* Is query key ARG[0..keylen) in the comma-separated STRIP list? "*" = all;
   case-sensitive, space-trimmed tokens. */
static int hts_query_key_stripped(const char *arg, size_t keylen,
                                  const char *strip) {
  const char *p = strip;

  while (*p != '\0') {
    const char *start = p;
    size_t toklen;

    while (*p != '\0' && *p != ',')
      p++;
    toklen = (size_t) (p - start);
    while (toklen > 0 && *start == ' ') {
      start++;
      toklen--;
    }
    while (toklen > 0 && start[toklen - 1] == ' ')
      toklen--;
    if (toklen == 1 && start[0] == '*')
      return 1;
    if (toklen == keylen && strncmp(start, arg, keylen) == 0)
      return 1;
    if (*p == ',')
      p++;
  }
  return 0;
}

/* see htscore.h */
char *fil_normalized_filtered_ex(const char *source, char *dest,
                                 const char *strip, int do_slash,
                                 int do_query) {
  const char *query;
  char BIGSTK tmp[HTS_URLMAXSIZE * 2];
  htsbuff cb;
  int wrote = 0;

  /* No strip list, or no query: plain normalization. */
  if (strip == NULL || *strip == '\0' ||
      (query = strchr(source, '?')) == NULL) {
    return fil_normalized_ex(source, dest, do_slash, do_query);
  }

  /* Copy the path, re-emit kept query args, let fil_normalized() sort. Walk
     every field incl. empty/trailing ("a&","?&&") so the result is a fixpoint
     (the read re-normalizes it; a dropped empty arg would miss dedup). */
  cb = htsbuff_ptr(tmp, sizeof(tmp));
  htsbuff_catn(&cb, source, (size_t) (query - source));
  for (query++;;) {
    const char *const arg = query;
    const char *eq = NULL;
    size_t keylen, arglen;

    while (*query != '\0' && *query != '&') {
      if (eq == NULL && *query == '=')
        eq = query;
      query++;
    }
    arglen = (size_t) (query - arg);
    keylen = eq != NULL ? (size_t) (eq - arg) : arglen;
    if (!hts_query_key_stripped(arg, keylen, strip)) {
      htsbuff_catc(&cb, wrote ? '&' : '?');
      htsbuff_catn(&cb, arg, arglen);
      wrote = 1;
    }
    if (*query == '\0')
      break;
    query++;
  }
  return fil_normalized_ex(tmp, dest, do_slash, do_query);
}

/* see htscore.h */
char *fil_normalized_filtered(const char *source, char *dest,
                              const char *strip) {
  return fil_normalized_filtered_ex(source, dest, strip, 1, 1);
}

/* see htscore.h */
const char *hts_query_strip_keys(const char *rules, const char *adr,
                                 const char *fil, char *dest, size_t destsize) {
  const char *p, *q;
  const char *result = NULL;
  /* holds a host and a path, each up to HTS_URLMAXSIZE * 2 */
  char BIGSTK url[HTS_URLMAXSIZE * 4];

  if (rules == NULL || *rules == '\0' || destsize == 0)
    return NULL;

  /* Match string = normalized host/path, query removed. jump_normalized_const
     collapses www+scheme/auth so read and write (double-normalized) agree;
     query excluded keeps the decision on host/path only. Clipped rather than
     aborting: a longer one is hostile input, and which rule it picks is a
     filter decision, not a memory error. */
  (void) strclipbuff(url, sizeof(url), jump_normalized_const(adr));
  if (fil[0] != '/')
    (void) strclipbuff(&url[strlen(url)], sizeof(url) - strlen(url), "/");
  q = strchr(fil, '?');
  {
    const size_t room = sizeof(url) - strlen(url) - 1;
    const size_t fillen = q != NULL ? (size_t) (q - fil) : strlen(fil);

    strlncatbuff(url, fil, sizeof(url), fillen < room ? fillen : room);
  }

  /* Walk the '\n' entries; last match wins (like the +/- filter eval). Each is
     "pattern=keys"; no '=' is the bare form, pattern "*". */
  for (p = rules; *p != '\0';) {
    const char *const line = p;
    const char *eol, *eq, *keys;
    char BIGSTK pat[HTS_URLMAXSIZE * 2];

    while (*p != '\0' && *p != '\n')
      p++;
    eol = p;
    if (*p == '\n')
      p++;
    if (eol == line)
      continue;
    eq = memchr(line, '=', (size_t) (eol - line));
    if (eq != NULL) {
      size_t patlen = (size_t) (eq - line);

      if (patlen >= sizeof(pat))
        patlen = sizeof(pat) - 1;
      memcpy(pat, line, patlen);
      pat[patlen] = '\0';
      keys = eq + 1;
    } else {
      pat[0] = '*';
      pat[1] = '\0';
      keys = line;
    }
    if (strjoker(url, pat, NULL, NULL) != NULL) {
      size_t klen = (size_t) (eol - keys);

      if (klen >= destsize)
        klen = destsize - 1;
      memcpy(dest, keys, klen);
      dest[klen] = '\0';
      result = dest;
    }
  }
  return result;
}

/* see htscore.h */
const char *hts_host_alias_rules(httrackp *opt) {
  return StringNotEmpty(opt->host_alias) ? StringBuff(opt->host_alias) : NULL;
}

/* Last rule of RULES whose alias list matches HOST (strjoker, last wins as in
   the +/- filter list): its canonical host, of length *CANONLEN, or NULL. FULL
   is the scheme://host form a scheme-qualified pattern matches; NOWWW collapses
   the www. prefix of both the pattern and the host. */
static const char *hts_host_alias_match(const char *rules, const char *host,
                                        const char *full,
                                        hts_boolean collapse_www,
                                        size_t *canonlen) {
  const char *p;
  const char *canon = NULL;

  /* Walk the '\n' entries "alias[,alias...]=canonical" */
  for (p = rules; *p != '\0';) {
    const char *const line = p;
    const char *eol, *eq, *pat;

    while (*p != '\0' && *p != '\n')
      p++;
    eol = p;
    if (*p == '\n')
      p++;
    eq = memchr(line, '=', (size_t) (eol - line));
    if (eq == NULL || eq == line || eq + 1 == eol)
      continue; /* not "alias=canonical" */
    for (pat = line; pat < eq;) {
      const char *const sep = memchr(pat, ',', (size_t) (eq - pat));
      const char *const end = sep != NULL ? sep : eq;
      const char *start = pat;
      size_t len = (size_t) (end - pat);
      char BIGSTK glob[HTS_URLMAXSIZE * 2];

      while (len > 0 && (*start == ' ' || *start == '\t')) {
        start++; /* "a.com, b.com=c.com" */
        len--;
      }
      while (len > 0 && (start[len - 1] == ' ' || start[len - 1] == '\t' ||
                         start[len - 1] == '/'))
        len--; /* a trailing slash is not a path */
      if (len != 0 && len < sizeof(glob)) {
        char *ghost;

        memcpy(glob, start, len);
        glob[len] = '\0';
        /* the collapse applies to the pattern too, so a rule may spell either
           www.example.com or example.com under -%u */
        ghost = glob + (jump_protocol_const(glob) - glob);
        if (collapse_www) {
          const char *const bare = jump_normalized_const(ghost);

          if (bare != ghost)
            memmove(ghost, bare, strlen(bare) + 1);
        }
        /* a rule that names no scheme keeps matching every scheme */
        if (strjoker(strstr(glob, "://") != NULL ? full : host,
                     strstr(glob, "://") != NULL ? glob : ghost, NULL,
                     NULL) != NULL) {
          const char *end = eol;

          canon = eq + 1;
          /* Trim here, where the canonical is produced: the resolver compares
             one match against the next to see a chain settle, and an untrimmed
             one never equals the trimmed bytes it emits. */
          while (canon < end && (*canon == ' ' || *canon == '\t'))
            canon++;
          while (end > canon &&
                 (end[-1] == ' ' || end[-1] == '\t' || end[-1] == '/'))
            end--;
          *canonlen = (size_t) (end - canon);
          break;
        }
      }
      pat = end + 1;
    }
  }
  return canon;
}

/* Build "scheme://host" from URL's scheme and HOST, for the rules that name a
   scheme. HTS_FALSE if it does not fit DEST. */
static hts_boolean hts_host_alias_subject(const char *url, const char *host,
                                          char *dest, size_t destsize) {
  /* An address with no scheme is the engine's plain-http form. Spell that one
     out, or a rule naming http:// could never match anything. */
  const hts_boolean bare =
      jump_protocol_const(url) == url ? HTS_TRUE : HTS_FALSE;
  const char *const scheme = bare ? "http://" : url;
  const size_t schemelen =
      bare ? strlen("http://") : (size_t) (jump_protocol_const(url) - url);
  const size_t hostlen = strlen(host);

  if (schemelen >= destsize || hostlen >= destsize - schemelen)
    return HTS_FALSE;
  memcpy(dest, scheme, schemelen);
  memcpy(dest + schemelen, host, hostlen);
  dest[schemelen + hostlen] = '\0';
  return HTS_TRUE;
}

/* Chain hops resolved before giving up. A longer chain, or a loop, is a
   misconfiguration; the fold leaves those hosts alone rather than settle on an
   arbitrary hop, which would depend on how often a link is named. */
enum { HTS_HOST_ALIAS_MAXHOPS = 8 };

/* hts_host_alias(), reporting through SETTLED (may be NULL) whether the chain
   ended within HTS_HOST_ALIAS_MAXHOPS. */
static const char *hts_host_alias_resolve(const char *rules, const char *adr,
                                          hts_boolean collapse_www, char *dest,
                                          size_t destsize,
                                          hts_boolean *settled) {
  const char *proto, *host, *hostsubj, *canon;
  char BIGSTK fullsubj[HTS_URLMAXSIZE * 2];
  char schemebuf[32]; /* longest jump_protocol_const knows is "socks5h://" */
  size_t canonlen = 0, schemelen;
  int hop;

  if (settled != NULL)
    *settled = HTS_TRUE;
  if (rules == NULL || *rules == '\0' || adr == NULL || destsize == 0)
    return NULL;
  /* engine pseudo-hosts: a "*=host" rule must not swallow them */
  if (adr[0] == '\0' || strcmp(adr, "primary") == 0 ||
      strcmp(adr, "file://") == 0)
    return NULL;
  proto = jump_protocol_const(adr);      /* after the scheme */
  host = jump_identification_const(adr); /* after the credentials */
  schemelen = (size_t) (proto - adr);
  if (schemelen >= sizeof(schemebuf))
    return NULL;
  memcpy(schemebuf, adr, schemelen);
  schemebuf[schemelen] = '\0';
  /* Match the host in the form the dedup layer uses: under the www. collapse a
     rule naming example.com covers www.example.com, and with the collapse off
     the rule matches the name as the link spells it. */
  hostsubj = collapse_www ? jump_normalized_const(host) : host;
  if (!hts_host_alias_subject(adr, hostsubj, fullsubj, sizeof(fullsubj)))
    return NULL;
  canon =
      hts_host_alias_match(rules, hostsubj, fullsubj, collapse_www, &canonlen);
  if (canon == NULL)
    return NULL;

  /* Follow the chain to its end, so the fold is a fixpoint: url_savename names
     the same link more than once, each time from the host it wrote last. */
  for (hop = 0;; hop++) {
    char BIGSTK canonbuf[HTS_URLMAXSIZE * 2];
    const char *next, *dhost, *chost, *cproto;
    size_t nextlen = 0, credlen, hostlen;

    /* The rule may name the scheme the canonical host speaks; when it does not,
       the scheme in effect stays, which is the previous hop's when a rule
       earlier in the chain named one. The link's credentials are kept. */
    if (canonlen >= sizeof(canonbuf))
      return NULL;
    memcpy(canonbuf, canon, canonlen);
    canonbuf[canonlen] = '\0';
    /* the link's credentials are the ones kept, so drop any the rule carries:
       they would otherwise be prepended to them again on every re-fold */
    cproto = jump_protocol_const(canonbuf);
    chost = jump_identification_const(cproto);
    hostlen = canonlen - (size_t) (chost - canonbuf);
    if (cproto != canonbuf) { /* the rule names a scheme */
      const size_t len = (size_t) (cproto - canonbuf);

      if (len >= sizeof(schemebuf))
        return NULL;
      memcpy(schemebuf, canonbuf, len);
      schemebuf[len] = '\0';
      schemelen = len;
    }
    credlen = (size_t) (host - proto);

    if (hostlen == 0)
      return NULL; /* a rule naming no host: leave the URL alone */
    if (schemelen >= destsize || credlen >= destsize - schemelen ||
        hostlen >= destsize - schemelen - credlen)
      return NULL; /* would not fit: leave the URL alone */
    memcpy(dest, schemebuf, schemelen);
    memcpy(dest + schemelen, proto, credlen);
    memcpy(dest + schemelen + credlen, chost, hostlen);
    dest[schemelen + credlen + hostlen] = '\0';

    dhost = jump_identification_const(dest);
    if (collapse_www)
      dhost = jump_normalized_const(dhost);
    if (!hts_host_alias_subject(dest, dhost, fullsubj, sizeof(fullsubj)))
      return NULL;
    next = hts_host_alias_match(rules, dhost, fullsubj, collapse_www, &nextlen);
    if (next == NULL)
      break; /* settled: no rule maps this host further */
    if (nextlen == canonlen && strncasecmp(next, canon, canonlen) == 0)
      break; /* a rule matching its own target */
    if (hop + 1 >= HTS_HOST_ALIAS_MAXHOPS) {
      if (settled != NULL)
        *settled = HTS_FALSE;
      return NULL;
    }
    canon = next;
    canonlen = nextlen;
  }
  /* a host that maps to itself: keep the callers' fast path */
  return strcasecmp(dest, adr) != 0 ? dest : NULL;
}

/* see htscore.h */
const char *hts_host_alias(const char *rules, const char *adr,
                           hts_boolean collapse_www, char *dest,
                           size_t destsize) {
  return hts_host_alias_resolve(rules, adr, collapse_www, dest, destsize, NULL);
}

/* see htscore.h */
hts_boolean hts_host_alias_collapse_www(httrackp *opt) {
  return opt->urlhack && !opt->no_www_dedup ? HTS_TRUE : HTS_FALSE;
}

/* see htscore.h */
const char *hts_host_alias_fold(httrackp *opt, lien_adrfil *af) {
  char BIGSTK canon[HTS_URLMAXSIZE * 2];
  const char *const alias =
      hts_host_alias(hts_host_alias_rules(opt), af->adr,
                     hts_host_alias_collapse_www(opt), canon, sizeof(canon));

  if (alias != NULL)
    strcpybuff(af->adr, alias);
  return af->adr;
}

/* see htscore.h */
hts_boolean hts_host_same_alias(const char *rules, const char *adra,
                                const char *adrb, hts_boolean collapse_www) {
  char BIGSTK ca[HTS_URLMAXSIZE * 2], cb[HTS_URLMAXSIZE * 2];
  const char *a, *b;

  if (rules == NULL || *rules == '\0' || adra == NULL || adrb == NULL)
    return HTS_FALSE;
  a = hts_host_alias(rules, adra, collapse_www, ca, sizeof(ca));
  b = hts_host_alias(rules, adrb, collapse_www, cb, sizeof(cb));
  if (a == NULL && b == NULL)
    return HTS_FALSE; /* neither is aliased: the caller's own test */
  return strcasecmp(jump_identification_const(a != NULL ? a : adra),
                    jump_identification_const(b != NULL ? b : adrb)) == 0
             ? HTS_TRUE
             : HTS_FALSE;
}

/* HTS_TRUE if TOKEN[0..LEN) is a host, optionally behind the scheme it speaks
   and with a trailing slash. GLOB keeps the filter metacharacters legal, which
   only the alias side of a rule may use. */
static hts_boolean hts_host_alias_token_ok(const char *token, size_t len,
                                           hts_boolean glob) {
  char BIGSTK buff[HTS_URLMAXSIZE * 2];
  const char *host;

  while (len > 0 && (*token == ' ' || *token == '\t')) {
    token++;
    len--;
  }
  while (len > 0 && (token[len - 1] == ' ' || token[len - 1] == '\t'))
    len--;
  if (len == 0 || len >= sizeof(buff))
    return HTS_FALSE;
  memcpy(buff, token, len);
  buff[len] = '\0';
  while (len > 0 && buff[len - 1] == '/')
    buff[--len] = '\0'; /* the matcher strips the whole run */
  /* an alias may glob its scheme; an unknown canonical one lands in the host */
  host = glob && strstr(buff, "://") != NULL ? strstr(buff, "://") + 3
                                             : jump_protocol_const(buff);
  /* a scheme with no host behind it, and the engine's own pseudo-host, name
     nothing the fold could write into an address */
  if (*host == '\0' || strcmp(host, "primary") == 0)
    return HTS_FALSE;
  /* --host-alias maps hosts: a path belongs to no part of an address */
  if (strchr(host, '/') != NULL)
    return HTS_FALSE;
  /* a control byte names no host, and '\n' would split the store's rule list */
  for (; *host != '\0'; host++) {
    if ((unsigned char) *host < ' ' || *host == '\x7f')
      return HTS_FALSE;
    if (strchr(glob ? "=# " : "=,*?();\\#@ ", *host) != NULL)
      return HTS_FALSE;
  }
  return HTS_TRUE;
}

/* see httrack-library.h */
HTSEXT_API hts_boolean hts_host_alias_rule_ok(const char *rule) {
  const char *const eq = rule != NULL ? strchr(rule, '=') : NULL;
  const char *pat;

  if (eq == NULL || eq == rule || eq[1] == '\0')
    return HTS_FALSE;
  if (!hts_host_alias_token_ok(eq + 1, strlen(eq + 1), HTS_FALSE))
    return HTS_FALSE;
  for (pat = rule; pat < eq;) {
    const char *const sep = memchr(pat, ',', (size_t) (eq - pat));
    const char *const end = sep != NULL ? sep : eq;

    if (!hts_host_alias_token_ok(pat, (size_t) (end - pat), HTS_TRUE))
      return HTS_FALSE;
    pat = end + 1;
  }
  return HTS_TRUE;
}

/* see htscore.h */
const char *hts_host_alias_looping(const char *rules, hts_boolean collapse_www,
                                   char *dest, size_t destsize) {
  const char *p;

  if (rules == NULL || destsize == 0)
    return NULL;
  for (p = rules; *p != '\0';) {
    const char *const line = p;
    const char *eol, *eq;

    while (*p != '\0' && *p != '\n')
      p++;
    eol = p;
    if (*p == '\n')
      p++;
    eq = memchr(line, '=', (size_t) (eol - line));
    if (eq == NULL || eq + 1 == eol)
      continue;
    {
      const size_t len = (size_t) (eol - eq - 1);
      char BIGSTK canon[HTS_URLMAXSIZE * 2], tmp[HTS_URLMAXSIZE * 2];
      hts_boolean settled;

      if (len >= sizeof(canon))
        continue;
      memcpy(canon, eq + 1, len);
      canon[len] = '\0';
      (void) hts_host_alias_resolve(rules, canon, collapse_www, tmp,
                                    sizeof(tmp), &settled);
      if (!settled) {
        if (strlen(canon) >= destsize)
          continue; /* cannot name it in full: keep looking */
        strlcpybuff(dest, canon, destsize);
        return dest;
      }
    }
  }
  return NULL;
}

#define endwith(a) ( (len >= (sizeof(a)-1)) ? ( strncmp(dest, a+len-(sizeof(a)-1), sizeof(a)-1) == 0 ) : 0 );
HTSEXT_API char *adr_normalized_sized(const char *source, char *dest,
                                      size_t destsize) {
  /* not yet too aggressive (no com<->net<->org checkings) */
  strlcpybuff(dest, jump_normalized_const(source), destsize);
  return dest;
}

// deprecated variant; kept for ABI compatibility. Bounds to the implicit
// contract the old callers relied on (an HTS_URLMAXSIZE*2 URL buffer).
HTSEXT_API char *adr_normalized(const char *source, char *dest) {
  return adr_normalized_sized(source, dest, HTS_URLMAXSIZE * 2);
}

#undef endwith

// find port (:80) or NULL if not found
// can handle IPV6 addresses
HTSEXT_API const char *jump_toport_const(const char *source) {
  const char *a, *trytofind;

  a = jump_identification_const(source);
  // last ] of the authority (http://[3ffe:b80:1234::1]:80/foo.html), bounded
  // from a: source's own scheme "//" would end the scan before it
  trytofind = strrchr_limit(a, ']', strchr(a, '/'));
  a = strchr((trytofind) ? trytofind : a, ':');
  return a;
}

HTSEXT_API DECLARE_NON_CONST_VERSION(jump_toport)

// strrchr, but not too far
const char *strrchr_limit(const char *s, char c, const char *limit) {
  if (limit == NULL) {
    const char *p = strrchr(s, c);

    return p ? (p + 1) : NULL;
  } else {
    const char *a = NULL, *p;

    for(;;) {
      p = strchr((a) ? a : s, c);
      if ((p >= limit) || (p == NULL))
        return a;
      a = p + 1;
    }
  }
}

// retourner adr sans ftp://
const char *jump_protocol_const(const char *source) {
  int p;

  // scheme
  // "Comparisons of scheme names MUST be case-insensitive" (RFC2616)
  if ((p = strfield(source, "http:")))
    source += p;
  else if ((p = strfield(source, "ftp:")))
    source += p;
  else if ((p = strfield(source, "https:")))
    source += p;
  else if ((p = strfield(source, "file:")))
    source += p;
  else if ((p = strfield(source, "socks5h:")))
    source += p;
  else if ((p = strfield(source, "socks5:")))
    source += p;
  else if ((p = strfield(source, "connect:")))
    source += p;
  // net_path
  if (strncmp(source, "//", 2) == 0)
    source += 2;
  return source;
}

DECLARE_NON_CONST_VERSION(jump_protocol)

hts_boolean hts_is_control_free_sized(const char *str, size_t len) {
  size_t i;

  for (i = 0; i < len; i++) {
    if ((unsigned char) str[i] < ' ')
      return HTS_FALSE;
  }
  return HTS_TRUE;
}

hts_boolean hts_is_control_free(const char *str) {
  return hts_is_control_free_sized(str, strlen(str));
}

hts_boolean hts_host_is_ipv4(const char *host, size_t len) {
  size_t i;
  int dots = 0;

  for (i = 0; i < len; i++) {
    if (host[i] == '.')
      dots++;
    else if (host[i] < '0' || host[i] > '9')
      return HTS_FALSE;
  }
  return dots > 0 ? HTS_TRUE : HTS_FALSE;
}

hts_boolean hts_proxy_is_socks(const char *name) {
  if (name == NULL)
    return HTS_FALSE;
  return (strfield(name, "socks5h:") || strfield(name, "socks5:")) ? HTS_TRUE
                                                                   : HTS_FALSE;
}

hts_boolean hts_proxy_is_connect(const char *name) {
  if (name == NULL)
    return HTS_FALSE;
  return strfield(name, "connect:") ? HTS_TRUE : HTS_FALSE;
}

// default proxy port for a -P argument, keyed on the scheme
static int proxy_default_port(const char *arg) {
  return hts_proxy_is_socks(arg) ? 1080 : 8080;
}

// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
static int parse_proxy_port(const char *a, const char *arg) {
  int port;

  if (!hts_parse_url_port(a, &port))
    return proxy_default_port(arg);
  return port;
}

void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
  const char *authority = strstr(arg, "://");
  const char *a;
  size_t namelen;

  if (name_size == 0)
    return;
  // scan back to the port ':' (or userinfo '@'), never past the authority (a
  // scheme's own colon is not a port separator) nor into an IPv6 literal (']'
  // stops it); inspect a[-1] from one-past-end so no pointer underflows
  authority = (authority != NULL) ? authority + 3 : arg;
  a = arg + strlen(arg);
  while (a > authority && a[-1] != ':' && a[-1] != '@' && a[-1] != ']')
    a--;
  if (a > authority && a[-1] == ':') {
    *port = parse_proxy_port(a, arg);
    namelen = (size_t) (a - 1 - arg);
  } else {
    *port = proxy_default_port(arg);
    namelen = strlen(arg);
  }
  if (namelen >= name_size) // arg is user-controlled: truncate, don't overflow
    namelen = name_size - 1;
  memcpy(name, arg, namelen);
  name[namelen] = '\0';
}

// codage base 64 a vers b
void code64(unsigned char *a, int size_a, unsigned char *b, int crlf) {
  int i1 = 0, i2 = 0, i3 = 0, i4 = 0;
  int loop = 0;
  unsigned long int store;
  int n;
  const char _hts_base64[] =
    "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
  while(size_a-- > 0) {
    // 24 bits
    n = 1;
    store = *a++;
    if (size_a-- > 0) {
      n = 2;
      store <<= 8;
      store |= *a++;
    }
    if (size_a-- > 0) {
      n = 3;
      store <<= 8;
      store |= *a++;
    }
    if (n == 3) {
      i4 = store & 63;
      i3 = (store >> 6) & 63;
      i2 = (store >> 12) & 63;
      i1 = (store >> 18) & 63;
    } else if (n == 2) {
      store <<= 2;
      i3 = store & 63;
      i2 = (store >> 6) & 63;
      i1 = (store >> 12) & 63;
    } else {
      store <<= 4;
      i2 = store & 63;
      i1 = (store >> 6) & 63;
    }

    *b++ = _hts_base64[i1];
    *b++ = _hts_base64[i2];
    if (n >= 2)
      *b++ = _hts_base64[i3];
    else
      *b++ = '=';
    if (n >= 3)
      *b++ = _hts_base64[i4];
    else
      *b++ = '=';

    if (crlf && ((loop += 3) % 60) == 0) {
      *b++ = '\r';
      *b++ = '\n';
    }
  }
  *b++ = '\0';
}

void unescape_amp(char *s) {
  if (hts_unescapeEntities(s, s, strlen(s) + 1) != 0) {
    assertf(! "error escaping html entities");
  }
}

// remplacer %20 par ' ', etc..
// buffer MAX 1Ko
HTSEXT_API char *unescape_http(char *const catbuff, const size_t size, const char *const s) {
  size_t i, j;

  RUNTIME_TIME_CHECK_SIZE(size);

  for(i = 0, j = 0; s[i] != '\0' && j + 1 < size ; i++) {
    int h;
    if (s[i] == '%' && (h = hts_ehex(&s[i + 1])) >= 0) {
      catbuff[j++] = (char) h;
      i += 2;
    } else
      catbuff[j++] = s[i];
  }
  catbuff[j++] = '\0';
  return catbuff;
}

// unescape in URL/URI ONLY what has to be escaped, to form a standard URL/URI
// DOES NOT DECODE %25 (part of CHAR_DELIM)
// no_high & 1: decode high chars
// no_high & 2: decode space
HTSEXT_API char *unescape_http_unharm(char *const catbuff, const size_t size,
                                      const char *s,
                                      const hts_boolean no_high) {
  size_t i, j;

  RUNTIME_TIME_CHECK_SIZE(size);

  for(i = 0, j = 0; s[i] != '\0' && j + 1 < size ; i++) {
    if (s[i] == '%') {
      const int nchar = hts_ehex(&s[i + 1]);

      const int test = 
        ( CHAR_RESERVED(nchar) && nchar != '+' )        /* %2B => + (not in query!) */
        || CHAR_DELIM(nchar)
        || CHAR_UNWISE(nchar)
        || CHAR_LOW(nchar)    /* CHAR_SPECIAL */
        || ( CHAR_XXAVOID(nchar) && ( nchar != ' ' || ( no_high & 2) == 0 ) )
        || ( ( no_high & 1 ) && CHAR_HIG(nchar) )
        ;

      if (!test && nchar >= 0) {  /* can safely unescape */
        catbuff[j++] = (char) nchar;
        i += 2;
      } else {
        catbuff[j++] = '%';
      }
    } else {
      catbuff[j++] = s[i];
    }
  }
  catbuff[j++] = '\0';
  return catbuff;
}

// remplacer " par %xx etc..
// buffer MAX 1Ko
HTSEXT_API size_t escape_spc_url(const char *const src, 
                                 char *const dest, const size_t size) {
  return x_escape_http(src, dest, size, 2);
}

// smith / john -> smith%20%2f%20john
HTSEXT_API size_t escape_in_url(const char *const src, 
                                char *const dest, const size_t size) {
  return x_escape_http(src, dest, size, 1);
}

// smith / john -> smith%20/%20john
HTSEXT_API size_t escape_uri(const char *const src, 
                             char *const dest, const size_t size) {
  return x_escape_http(src, dest, size, 3);
}

HTSEXT_API size_t escape_uri_utf(const char *const src, 
                                 char *const dest, const size_t size) {
  return x_escape_http(src, dest, size, 30);
}

HTSEXT_API size_t escape_check_url(const char *const src, char *const dest,
                                   const size_t size) {
  return x_escape_http(src, dest, size, 0);
}

HTSEXT_API size_t escape_control_url(const char *const src, char *const dest,
                                     const size_t size) {
  return x_escape_http(src, dest, size, 4);
}

// same as escape_check_url, but returns char*
HTSEXT_API char *escape_check_url_addr(const char *const src, 
                                       char *const dest, const size_t size) {
  escape_check_url(src, dest, size);
  return dest;
}

// Same as above, but appending to "dest"
#undef DECLARE_APPEND_ESCAPE_VERSION
#define DECLARE_APPEND_ESCAPE_VERSION(NAME) \
HTSEXT_API size_t append_ ##NAME(const char *const src, char *const dest, const size_t size) { \
  const size_t len = strnlen(dest, size); \
  assertf(len < size); \
  return NAME(src, dest + len, size - len); \
}

DECLARE_APPEND_ESCAPE_VERSION(escape_in_url)
DECLARE_APPEND_ESCAPE_VERSION(escape_spc_url)
DECLARE_APPEND_ESCAPE_VERSION(escape_uri_utf)
DECLARE_APPEND_ESCAPE_VERSION(escape_check_url)
DECLARE_APPEND_ESCAPE_VERSION(escape_uri)

#undef DECLARE_APPEND_ESCAPE_VERSION

// In-place escaping: copy dest aside, then escape that copy back into dest.
typedef size_t (*escape_fn_t)(const char *src, char *dest, size_t size);

static size_t inplace_escape(char *const dest, const size_t size,
                             escape_fn_t escape) {
  char buffer[256];
  const size_t len = strnlen(dest, size);
  const int in_buffer = len + 1 < sizeof(buffer);
  char *src = in_buffer ? buffer : malloct(len + 1);
  size_t ret;
  assertf(src != NULL);
  assertf(len < size);
  memcpy(src, dest, len + 1);
  ret = escape(src, dest, size);
  if (!in_buffer) {
    freet(src);
  }
  return ret;
}

// Thin exported wrappers binding inplace_escape() to each escaper (ABI).
#undef DECLARE_INPLACE_ESCAPE_VERSION
#define DECLARE_INPLACE_ESCAPE_VERSION(NAME)                                   \
  HTSEXT_API size_t inplace_##NAME(char *const dest, const size_t size) {      \
    return inplace_escape(dest, size, NAME);                                   \
  }

DECLARE_INPLACE_ESCAPE_VERSION(escape_in_url)
DECLARE_INPLACE_ESCAPE_VERSION(escape_spc_url)
DECLARE_INPLACE_ESCAPE_VERSION(escape_uri_utf)
DECLARE_INPLACE_ESCAPE_VERSION(escape_check_url)
DECLARE_INPLACE_ESCAPE_VERSION(escape_uri)

#undef DECLARE_INPLACE_ESCAPE_VERSION


HTSEXT_API size_t make_content_id(const char *const adr, const char *const fil, 
                                  char *const dest, const size_t size) {
  char *a;
  size_t esc_size = escape_in_url(adr, dest, size);
  esc_size += escape_in_url(fil, dest + esc_size, size - esc_size);
  RUNTIME_TIME_CHECK_SIZE(size);
  for(a = dest ; (a = strchr(a, '%')) != NULL ; a++) {
    *a = 'X';
  }
  return esc_size;
}

// strip all control characters
HTSEXT_API void escape_remove_control(char *const s) {
  size_t i, j;
  for(i = 0, j = 0 ; s[i] != '\0' ; i++) {
    const unsigned char c = (unsigned  char) s[i];
    if (c >= 32) {
      if (i != j) {
        assertf(j < i);
        s[j] = s[i];
      }
      j++;
    }
  }
  // compaction left the original tail sitting past j
  s[j] = '\0';
}

#undef ADD_CHAR
#define ADD_CHAR(C) do {   \
      assertf(j < size);    \
      if (j + 1 == size) { \
        dest[j] = '\0';    \
        return size;       \
      }                    \
      dest[j++] = (C);     \
  } while(0)

/* Returns the number of characters written (not taking in account the terminating \0), or 'size' upon overflow. */
HTSEXT_API size_t x_escape_http(const char *const s, char *const dest, 
                                const size_t size, const int mode) {
  static const char hex[] = "0123456789abcdef";
  size_t i, j;

  RUNTIME_TIME_CHECK_SIZE(size);

  // Out-of-bound.
  // Previous character is supposed to be the terminating \0.
  if (size == 0) {
    return 0;
  }

  for(i = 0, j = 0 ; s[i] != '\0' ; i++) {
    const unsigned char c = (unsigned char) s[i];
    int test = 0;

    if (mode == 0)
      test = c == '"' || c == ' ' || CHAR_SPECIAL(c);
    else if (mode == 1)
      test = CHAR_RESERVED(c)
             || CHAR_DELIM(c)
             || CHAR_UNWISE(c)
             || CHAR_SPECIAL(c)
             || CHAR_XXAVOID(c)
             || CHAR_MARK(c);
    else if (mode == 2)
      test = c == ' ';       // n'escaper que espace
    else if (mode == 3)         // échapper que ce qui est nécessaire
      test = CHAR_SPECIAL(c)
             || CHAR_XXAVOID(c);
    else if (mode == 4) // C0 controls only, leaving high bytes (UTF-8)
      test = CHAR_LOW(c);
    else if (mode == 30)      // échapper que ce qui est nécessaire
      test = (c != '/' && CHAR_RESERVED(c))
        || CHAR_DELIM(c)
        || CHAR_UNWISE(c)
        || CHAR_SPECIAL(c)
        || CHAR_XXAVOID(c);

    if (!test) {
      ADD_CHAR(c);
    } else {
      ADD_CHAR('%');
      ADD_CHAR(hex[c / 16]);
      ADD_CHAR(hex[c % 16]);
    }
  }

  assertf(j < size);
  dest[j] = '\0';
  return j;
}

HTSEXT_API size_t escape_for_html_print(const char *const s, char *const dest, const size_t size) {
  size_t i, j;

  RUNTIME_TIME_CHECK_SIZE(size);

  for(i = 0, j = 0 ; s[i] != '\0' ; i++) {
    const unsigned char c = (unsigned char) s[i];
    if (c == '&') {
      ADD_CHAR('&');
      ADD_CHAR('a');
      ADD_CHAR('m');
      ADD_CHAR('p');
      ADD_CHAR(';');
    } else {
      ADD_CHAR(c);
    }
  }
  assertf(j < size);
  dest[j] = '\0';
  return j;
}

HTSEXT_API size_t escape_for_html_print_full(const char *const s, char *const dest, const size_t size) {
  static const char hex[] = "0123456789abcdef";
  size_t i, j;

  RUNTIME_TIME_CHECK_SIZE(size);

  for(i = 0, j = 0 ; s[i] != '\0' ; i++) {
    const unsigned char c = (unsigned char) s[i];
    if (c == '&') {
      ADD_CHAR('&');
      ADD_CHAR('a');
      ADD_CHAR('m');
      ADD_CHAR('p');
      ADD_CHAR(';');
    } else if (CHAR_HIG(c)) {
      ADD_CHAR('&');
      ADD_CHAR('#');
      ADD_CHAR('x');
      ADD_CHAR(hex[c / 16]);
      ADD_CHAR(hex[c % 16]);
      ADD_CHAR(';');
    } else {
      ADD_CHAR(c);
    }
  }
  assertf(j < size);
  dest[j] = '\0';
  return j;
}

#undef ADD_CHAR

// lower-case conversion into caller buffer (capacity catbuffsize)
char *convtolower(char *catbuff, size_t catbuffsize, const char *a) {
  strlcpybuff(catbuff, a, catbuffsize);
  hts_lowcase(catbuff);         // lower case
  return catbuff;
}

// conversion en minuscules
void hts_lowcase(char *s) {
  size_t i;

  for(i = 0; s[i] != '\0'; i++)
    if ((s[i] >= 'A') && (s[i] <= 'Z'))
      s[i] += ('a' - 'A');
}

// guess a local file's mime type (e.g. fil="toto.gif" -> s="image/gif")
// returns 1 if a type was written to s, 0 otherwise
hts_boolean guess_httptype_sized(httrackp *opt, char *s, size_t ssize,
                                 const char *fil) {
  return get_httptype_sized(opt, s, ssize, fil, 1);
}

// deprecated variant; kept for ABI compatibility. Bounds to the implicit
// contract the old callers relied on (a contenttype-sized buffer).
void guess_httptype(httrackp * opt, char *s, const char *fil) {
  (void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, 1);
}

// first match in a NUL-terminated {mime,ext} table. key selects the lookup
// column (0=mime, 1=ext); returns the other column, or NULL if no row matches
// (a "*" partner means the row carries no value).
static const char *hts_mime_lookup(const char *(*table)[2], int key,
                                   const char *needle) {
  int j;

  for (j = 0; strnotempty(table[j][1]); j++) {
    if (strfield2(table[j][key], needle) && table[j][!key][0] != '*')
      return table[j][!key];
  }
  return NULL;
}

// write the mime type for fil into s (capacity ssize)
// flag: 1 to always return a type (the "application/..." / octet-stream
// fallback) returns 1 if a type was written to s, 0 otherwise
HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
                                          const char *fil, hts_boolean flag) {
  // userdef overrides get_httptype (a rule with an empty value, e.g. "--assume
  // cgi=", matches but writes nothing: report it as "no type" like the old
  // code, whose callers tested strnotempty(s))
  if (get_userhttptype(opt, s, ssize, fil)) {
    return s[0] != '\0';
  }
  // regular tests
  if (ishtml(opt, fil) == 1) {
    strlcpybuff(s, "text/html", ssize);
    return 1;
  } else {
    /* Check html -> text/html */
    const char *a = hts_lastcharptr(fil);

    while ((a > fil) && (*a != '.') && (*a != '/'))
      a--;
    if (a >= fil && *a == '.' && strlen(a) < 32) {
      const char *mime;

      a++;
      mime = hts_mime_lookup(hts_mime, 1, a);
      if (mime == NULL)
        mime = hts_mime_lookup(hts_mime_modern, 1, a);
      if (mime != NULL) {
        strlcpybuff(s, mime, ssize);
        return 1;
      }

      if (flag) {
        snprintf(s, ssize, "application/%s", a);
        return 1;
      }
    } else {
      if (flag) {
        strlcpybuff(s, "application/octet-stream", ssize);
        return 1;
      }
    }
  }
  return 0;
}

// deprecated variant; kept for ABI compatibility. Bounds to the implicit
// contract the old callers relied on (a contenttype-sized buffer).
HTSEXT_API void get_httptype(httrackp *opt, char *s, const char *fil,
                             int flag) {
  (void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, flag);
}

// get type of fil (php)
// s: destination buffer, or NULL
hts_boolean get_userhttptype(httrackp *opt, char *s, size_t ssize,
                             const char *fil) {
  if (s != NULL) {
    assertf(ssize != 0);
    if (s)
      s[0] = '\0';
    if (fil == NULL || *fil == '\0')
      return HTS_FALSE;
#if 1
    if (StringLength(opt->mimedefs) > 0) {

      /* Check --assume foooo/foo/bar.cgi=text/html, then foo/bar.cgi=text/html, then bar.cgi=text/html */
      /* also: --assume baz,bar,foooo/foo/bar.cgi=text/html */
      /* start from path beginning */
      do {
        const char *next;
        const char *mimedefs = StringBuff(opt->mimedefs);       /* loop through mime definitions : \nfoo=bar\nzoo=baz\n.. */

        while(*mimedefs != '\0') {
          const char *segment = fil + 1;

          if (*mimedefs == '\n') {
            mimedefs++;
          }
          /* compare current segment with user's definition */
          do {
            int i;

            /* check current item */
            for(i = 0; mimedefs[i] != '\0'      /* end of all defs */
                && mimedefs[i] != ' '   /* next item in left list */
                && mimedefs[i] != '='   /* end of left list */
                && mimedefs[i] != '\n'  /* end of this def (?) */
                && mimedefs[i] == segment[i]    /* same item */
                ; i++) ;
            /* success */
            if ((mimedefs[i] == '=' || mimedefs[i] == ' ') &&
                segment[i] == '\0') {
              while(mimedefs[i] != 0 && mimedefs[i] != '\n'
                    && mimedefs[i] != '=')
                i++;
              if (mimedefs[i] == '=') {
                size_t len;

                i++;
                len = strcspn(&mimedefs[i], "\n");
                /* clip, don't abort: the consumers match whole strings far
                   shorter than any destination, so a value long enough to be
                   clipped already matched nothing (#1276) */
                if (len >= ssize) {
                  hts_log_print(opt, LOG_WARNING,
                                "--assume value for '%s' is %d bytes, clipped "
                                "to %d",
                                fil, (int) len, (int) (ssize - 1));
                  len = ssize - 1;
                }
                memcpy(s, &mimedefs[i], len);
                s[len] = '\0';
                return HTS_TRUE; /* SUCCESS! */
              }
            }
            /* next item in list */
            for(mimedefs += i;
                *mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '='
                && *mimedefs != ' '; mimedefs++) ;
            if (*mimedefs == ' ') {
              mimedefs++;
            }
          } while(*mimedefs != '\0' && *mimedefs != '\n' && *mimedefs != '=');
          /* next user-def */
          for(; *mimedefs != '\0' && *mimedefs != '\n'; mimedefs++) ;
        }
        /* shorten segment */
        next = strchr(fil + 1, '/');
        if (next == NULL) {
          /* ext tests */
          next = strchr(fil + 1, '.');
        }
        fil = next;
      } while(fil != NULL);
    }
#else
    if (*buffer) {
      char BIGSTK search[1024];
      char *detect;

      sprintf(search, "\n%s=", ext);    // php=text/html
      detect = strstr(*buffer, search);
      if (!detect) {
        sprintf(search, "\n%s\n", ext); // php\ncgi=text/html
        detect = strstr(*buffer, search);
      }
      if (detect) {
        detect = strchr(detect, '=');
        if (detect) {
          detect++;
          if (s) {
            char *a;

            a = strchr(detect, '\n');
            if (a) {
              strncatbuff(s, detect, (int) (a - detect));
            }
          }
          return 1;
        }
      }
    }
#endif
  }
  return HTS_FALSE;
}

// give the file extension for a mime type (e.g. "image/gif" -> "gif")
// returns 1 if an extension was found (and written to s), 0 otherwise
int give_mimext(char *s, size_t ssize, const char *st) {
  int ok = 0;
  const char *ext;

  st = hts_effective_mime(st); /* no declared type: derive an html ext */
  s[0] = '\0';
  ext = hts_mime_lookup(hts_mime, 0, st);
  if (ext == NULL)
    ext = hts_mime_lookup(hts_mime_modern, 0, st);
  if (ext != NULL) {
    strlcpybuff(s, ext, ssize);
    ok = 1;
  }
  // wrap "x" mimetypes, such as:
  // application/x-mp3
  // or
  // application/mp3
  if (!ok) {
    int p;
    const char *a = NULL;

    if ((p = strfield(st, "application/x-")))
      a = st + p;
    else if ((p = strfield(st, "application/")))
      a = st + p;
    if (a) {
      if ((int) strlen(a) >= 1) {
        if ((int) strlen(a) <= 4) {
          strlcpybuff(s, a, ssize);
          ok = 1;
        }
      }
    }
  }
  return ok;
}

// extension connue?..
//  0 : non
//  1 : oui
//  2 : html
HTSEXT_API int is_knowntype(httrackp * opt, const char *fil) {
  char catbuff[CATBUFF_SIZE];
  const char *ext;
  int j = 0;

  if (!fil)
    return 0;
  ext = get_ext(catbuff, sizeof(catbuff), fil);
  while(strnotempty(hts_mime[j][1])) {
    if (strfield2(hts_mime[j][1], ext)) {
      if (is_html_mime_type(hts_mime[j][0]))
        return 2;
      else
        return 1;
    }
    j++;
  }

  // Known by user?
  return (is_userknowntype(opt, fil));
}

// known type?..
//  0 : no
//  1 : yes
//  2 : html
// setdefs : set mime buffer:
//   file=(char*) "asp=text/html\nphp=text/html\n"
HTSEXT_API int is_userknowntype(httrackp * opt, const char *fil) {
  char BIGSTK mime[1024];

  if (!fil)
    return 0;
  if (!strnotempty(fil))
    return 0;
  mime[0] = '\0';
  get_userhttptype(opt, mime, sizeof(mime), fil);
  if (!strnotempty(mime))
    return 0;
  else if (is_html_mime_type(mime))
    return 2;
  else
    return 1;
}

// page dynamique?
// is_dyntype(get_ext("foo.asp"))
HTSEXT_API hts_boolean is_dyntype(const char *fil) {
  int j = 0;

  if (!fil)
    return 0;
  if (!strnotempty(fil))
    return 0;
  while(strnotempty(hts_ext_dynamic[j])) {
    if (strfield2(hts_ext_dynamic[j], fil)) {
      return 1;
    }
    j++;
  }
  return 0;
}

// types critiques qui ne doivent pas être changés car renvoyés par des serveurs qui ne
// connaissent pas le type
hts_boolean may_unknown(httrackp *opt, const char *st) {
  int j = 0;

  // types média
  if (may_be_hypertext_mime(opt, st, "")) {
    return 1;
  }
  while(strnotempty(hts_mime_keep[j])) {
    if (strfield2(hts_mime_keep[j], st)) {      // trouvé
      return 1;
    }
    j++;
  }
  return 0;
}

/* returns 1 if the mime/filename seems to be bogus because of badly recognized multiple extension
  ; such as "application/x-wais-source" for "httrack-3.42-1.el5.src.rpm"
  reported by Hippy Dave 08/2008 (3.43) */
int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename) {
  int j;

  for(j = 0; strnotempty(hts_mime_bogus_multiple[j]); j++) {
    if (strfield2(hts_mime_bogus_multiple[j], mime)) {  /* found mime type in suspicious list */
      char ext[64];

      if (give_mimext(ext, sizeof(ext),
                      mime)) { /* we have an extension for that */
        const size_t ext_size = strlen(ext);
        const char *file = strrchr(filename, '/');      /* fetch terminal filename */

        if (file != NULL) {
          int i;

          for(i = 0; file[i] != 0; i++) {
            if (i > 0 && file[i - 1] == '.'
                && strncasecmp(&file[i], ext, ext_size) == 0
                && (file[i + ext_size] == 0 || file[i + ext_size] == '.'
                    || file[i + ext_size] == '?')) {
              return 1;         /* is ambiguous */
            }
          }
        }
      }
      return 0;
    }
  }
  return 0;
}

/* filename extension should not be changed because potentially bogus ; replaces may_unknown() (3.43) */
int may_unknown2(httrackp * opt, const char *mime, const char *filename) {
  int ret = may_unknown(opt, mime);

  if (ret == 0) {
    ret = may_bogus_multiple(opt, mime, filename);
  }
  return ret;
}

// -- Utils fichiers

// pretty print for i/o
void fprintfio(FILE * fp, const char *buff, const char *prefix) {
  char nl = 1;

  while(*buff) {
    switch (*buff) {
    case 13:
      break;
    case 10:
      fprintf(fp, "\r\n");
      nl = 1;
      break;
    default:
      if (nl)
        fprintf(fp, "%s", prefix);
      nl = 0;
      fputc(*buff, fp);
    }
    buff++;
  }
}

/* Le fichier existe-t-il? (ou est-il accessible?) */
/* Note: NOT utf-8 */
/* Note: preserve errno */
int fexist(const char *s) {
  char catbuff[CATBUFF_SIZE];
  const int err = errno;
  struct stat st;

  memset(&st, 0, sizeof(st));
  if (stat(fconv(catbuff, sizeof(catbuff), s), &st) == 0) {
    if (S_ISREG(st.st_mode)) {
      return 1;
    } else {
      return 0;
    }
  }
  errno = err;
  return 0;
}

/* Le fichier existe-t-il? (ou est-il accessible?) */
/* Note: utf-8 */
/* Note: preserve errno */
int fexist_utf8(const char *s) {
  char catbuff[CATBUFF_SIZE];
  const int err = errno;
  STRUCT_STAT st;

  memset(&st, 0, sizeof(st));
  if (STAT(fconv(catbuff, sizeof(catbuff), s), &st) == 0) {
    if (S_ISREG(st.st_mode)) {
      return 1;
    } else {
      return 0;
    }
  }
  errno = err;
  return 0;
}

/* File size, -1 if absent */
/* Note: NOT utf-8 */
LLint fsize(const char *s) {
  STRUCT_STAT st;

  if (!strnotempty(s)) // empty name: error
    return -1;
  /* _stat64, not stat(): the latter is 32-bit on MSVC */
#ifdef _WIN32
  if (_stat64(s, &st) == 0 && S_ISREG(st.st_mode)) {
#else
  if (stat(s, &st) == 0 && S_ISREG(st.st_mode)) {
#endif
    return st.st_size;
  } else {
    return -1;
  }
}

/* File size, -1 if absent */
/* Note: utf-8 */
LLint fsize_utf8(const char *s) {
  STRUCT_STAT st;

  if (!strnotempty(s)) // empty name: error
    return -1;
  if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
    return st.st_size;
  } else {
    return -1;
  }
}

/* fseeko/ftello, never fseek/ftell: ftell returns long, which cannot carry an
   offset past 2GB on a 32-bit platform and fails with EOVERFLOW there. */
LLint fpsize(FILE *fp) {
  LLint oldpos, size;

  if (!fp)
    return -1;
  oldpos = ftello(fp);
  fseeko(fp, 0, SEEK_END);
  size = ftello(fp);
  fseeko(fp, oldpos, SEEK_SET);
  return size;
}

/* root dir, with ending / */
typedef struct {
  char path[1024 + 4];
  int init;
} hts_rootdir_strc;
HTSEXT_API const char *hts_rootdir(char *file) {
  static hts_rootdir_strc strc = { "", 0 };
  if (file) {
    if (!strc.init) {
      strc.path[0] = '\0';
      strc.init = 1;
      if (strnotempty(file)) {
        const size_t file_len = strlen(file);
        char *a;

        assertf(file_len < sizeof(strc.path));
        strcpybuff(strc.path, file);
        while((a = strrchr(strc.path, '\\')))
          *a = '/';
        if ((a = strrchr(strc.path, '/'))) {
          *(a + 1) = '\0';
        } else
          strc.path[0] = '\0';
      }
      if (!strnotempty(strc.path)) {
        if (getcwd(strc.path, sizeof(strc.path)) == NULL)
          strc.path[0] = '\0';
        else
          strcatbuff(strc.path, "/");
      }
    }
    return NULL;
  } else if (strc.init)
    return strc.path;
  else
    return "";
}

HTSEXT_API hts_stat_struct HTS_STAT;

//
// return  number of downloadable bytes, depending on rate limiter
// see engine_stats() routine, too
// this routine works quite well for big files and regular ones, but apparently the rate limiter has
// some problems with very small files (rate too high)
LLint check_downloadable_bytes(int rate) {
  if (rate > 0) {
    TStamp time_now;
    TStamp elapsed_useconds;
    LLint bytes_transferred_during_period;
    LLint left;

    // get the older timer
    int id_timer = (HTS_STAT.istat_idlasttimer + 1) % 2;

    time_now = mtime_local();
    elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
    bytes_transferred_during_period =
      (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);

    left = ((rate * elapsed_useconds) / 1000) - bytes_transferred_during_period;
    if (left <= 0)
      left = 0;

    return left;
  } else
    return TAILLE_BUFFER;
}

// Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
// returns: 
// >0 : data received
// == 0 : not yet data
// <0: error or no data: READ_ERROR, READ_EOF or READ_TIMEOUT
int hts_read(htsblk * r, char *buff, int size) {
  int retour;

  if (r->is_file) {
#if HTS_WIDE_DEBUG
    DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
#endif
    if (r->fp) {
      retour = (int) fread(buff, 1, size, r->fp);
      if (retour == 0)  // can happen with directories (!)
        retour = READ_ERROR;
    } else
      retour = READ_ERROR;
  } else {
#if HTS_WIDE_DEBUG
    DEBUG_W("recv(%d, %p, %d)\n" _(int) r->soc _(void *)buff _(int) size);

    if (r->soc == INVALID_SOCKET)
      printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
#endif
#if HTS_USEOPENSSL
    if (r->ssl) {
      retour = SSL_read(r->ssl_con, buff, size);
      if (retour <= 0) {
        int err_code = SSL_get_error(r->ssl_con, retour);

        if ((err_code == SSL_ERROR_WANT_READ)
            || (err_code == SSL_ERROR_WANT_WRITE)
          ) {
          retour = 0;           /* no data yet (ssl cache) */
        } else if (err_code == SSL_ERROR_ZERO_RETURN) {
          retour = READ_EOF;    /* completed */
        } else {
          retour = READ_ERROR;  /* eof or error */
        }
      }
    } else {
#endif
      retour = recv(r->soc, buff, size, 0);
      if (retour == 0) {
        retour = READ_EOF;
      } else if (retour < 0) {
        retour = READ_ERROR;
      }
    }
    if (retour > 0)             // compter flux entrant
      HTS_STAT.HTS_TOTAL_RECV += retour;
#if HTS_USEOPENSSL
  }
#endif
#if HTS_WIDE_DEBUG
  DEBUG_W("recv/read done (%d bytes)\n" _(int) retour);
#endif
  return retour;
}

// -- Gestion cache DNS --
// 'RX98

// Free a DNS cache record (coucal value handler).
static void hts_cache_value_free(coucal_opaque arg, coucal_value value) {
  void *record = value.ptr;

  (void) arg;
  freet(record);
}

// opt's DNS cache hashtable, created on first use. Records (t_dnscache*) are
// owned by the table and freed by hts_cache_value_free on coucal_delete.
coucal hts_cache(httrackp *opt) {
  assertf(opt != NULL);
  if (opt->state.dns_cache == NULL) {
    coucal cache = coucal_new(0);

    coucal_set_name(cache, "dns_cache");
    coucal_value_set_value_handler(cache, hts_cache_value_free, NULL);
    opt->state.dns_cache = cache;
  }
  assertf(opt->state.dns_cache != NULL);
  return opt->state.dns_cache;
}

/* First lifetime of a negative DNS answer. Unlocked because only the
   self-test writes it, from the single thread that then resolves. */
static int hts_dns_negative_ttl_ms = HTS_DNS_NEGATIVE_TTL_MS;

void hts_dns_set_negative_ttl_ms(int ms) {
  hts_dns_negative_ttl_ms = ms > 0 ? ms : 1; /* the doubling needs a floor */
}

int hts_dns_negative_failures(httrackp *opt, const char *host) {
  void *ptr;
  int failures = 0;

  hts_mutexlock(&opt->state.lock);
  if (coucal_read_pvoid(hts_cache(opt), host, &ptr))
    failures = ((const t_dnscache *) ptr)->failures;
  hts_mutexrelease(&opt->state.lock);
  return failures;
}

void hts_dns_test_move_clock(httrackp *opt, const char *host, TStamp ms) {
  void *ptr;

  hts_mutexlock(&opt->state.lock);
  if (coucal_read_pvoid(hts_cache(opt), host, &ptr)) {
    t_dnscache *const record = (t_dnscache *) ptr;

    if (record->expiry != 0) { /* a positive record carries no stamps */
      record->expiry -= ms;
      record->stored -= ms;
    }
  }
  hts_mutexrelease(&opt->state.lock);
}

/* Doubles per consecutive failure up to the ceiling: an outage is retried
   soon, a host that is simply gone costs a handful of lookups per crawl. */
int hts_dns_negative_wait_ms(int failures) {
  int wait = hts_dns_negative_ttl_ms;
  int i;

  for (i = 1; i < failures && wait < HTS_DNS_NEGATIVE_TTL_MAX_MS; i++)
    wait *= 2;
  return wait < HTS_DNS_NEGATIVE_TTL_MAX_MS ? wait
                                            : HTS_DNS_NEGATIVE_TTL_MAX_MS;
}

// MUST BE LOCKED (coucal is not internally serialized vs FTP/web threads)
// Look up iadr in the DNS cache, filling out[0..min(count,max)-1].
// Returns: -1 not tested, or a negative answer past its expiry; 0
// negative-cached (not in DNS); >0 address count.
static int hts_ghbn_all(coucal cache, const char *const iadr,
                        SOCaddr *const out, const int max) {
  void *ptr;

  assertf(out != NULL);
  assertf(iadr != NULL);
  if (*iadr == '\0') {
    return -1;
  }
  if (coucal_read_pvoid(cache, iadr, &ptr)) { // ok trouvé
    const t_dnscache *const record = (const t_dnscache *) ptr;
    int i;

    assertf(record->host_count <= HTS_MAXADDRNUM);
    /* An outage must not blacklist a host for the rest of the crawl, and the
       stamps are wall-clock: a clock now behind the store time expires the
       record too, or an NTP step delays the retry by the size of the step. */
    if (record->expiry != 0) {
      const TStamp now = mtime_local();

      if (now >= record->expiry || now < record->stored) {
        return -1;
      }
    }
    for (i = 0; i < record->host_count && i < max; i++) {
      assertf(record->host_length[i] <= sizeof(record->host_addr[i]));
      SOCaddr_copyaddr2(out[i], record->host_addr[i], record->host_length[i]);
    }
    return record->host_count;
  }
  return -1;
}

#if HTS_INET6 != 0
/* Active resolver backend; defaults to the libc resolver. The self-test
   reroutes it to script DNS answers in-process (see
   hts_dns_set_resolver_backend). */
static const hts_resolver_backend hts_resolver_libc = {getaddrinfo,
                                                       freeaddrinfo};
static const hts_resolver_backend *hts_resolver = &hts_resolver_libc;

void hts_dns_set_resolver_backend(const hts_resolver_backend *backend) {
  hts_resolver = (backend != NULL) ? backend : &hts_resolver_libc;
}

/* Debug/test hook: HTTRACK_DEBUG_RESOLVE="host:ip[,ip...]" pins the resolution
   of `host` to the listed addresses (curl --resolve style), so the connect
   fallback can be exercised deterministically (a dead address first, a live one
   next). Any other host resolves normally. Below: an addrinfo backend that owns
   its chain (its own freeaddrinfo), so a synthesized and a delegated result
   free the same way. */

/* Deep-copy a libc addrinfo chain into our own allocations. */
static struct addrinfo *resolver_dup_chain(const struct addrinfo *src) {
  struct addrinfo *head = NULL, *tail = NULL;

  for (; src != NULL; src = src->ai_next) {
    struct addrinfo *const ai = calloct(1, sizeof(*ai));

    ai->ai_family = src->ai_family;
    ai->ai_socktype = src->ai_socktype;
    ai->ai_protocol = src->ai_protocol;
    ai->ai_addrlen = src->ai_addrlen;
    ai->ai_addr = malloct(src->ai_addrlen);
    memcpy(ai->ai_addr, src->ai_addr, src->ai_addrlen);
    if (head == NULL)
      head = ai;
    else
      tail->ai_next = ai;
    tail = ai;
  }
  return head;
}

/* Build one addrinfo node from an IPv4/IPv6 literal, or NULL if it does not
   parse or is filtered out by want_family (AF_INET/AF_INET6/PF_UNSPEC). */
static struct addrinfo *resolver_make_ai(const char *ip, int want_family) {
  struct addrinfo *ai;

  if (strchr(ip, ':') != NULL) { // IPv6 literal
    struct sockaddr_in6 sa6;

    if (want_family != PF_UNSPEC && want_family != AF_INET6)
      return NULL;
    memset(&sa6, 0, sizeof(sa6));
    if (inet_pton(AF_INET6, ip, &sa6.sin6_addr) != 1)
      return NULL;
    sa6.sin6_family = AF_INET6;
    ai = calloct(1, sizeof(*ai));
    ai->ai_family = AF_INET6;
    ai->ai_addrlen = sizeof(sa6);
    ai->ai_addr = malloct(sizeof(sa6));
    memcpy(ai->ai_addr, &sa6, sizeof(sa6));
  } else { // IPv4 literal
    struct sockaddr_in sa;

    if (want_family != PF_UNSPEC && want_family != AF_INET)
      return NULL;
    memset(&sa, 0, sizeof(sa));
    if (inet_pton(AF_INET, ip, &sa.sin_addr) != 1)
      return NULL;
    sa.sin_family = AF_INET;
    ai = calloct(1, sizeof(*ai));
    ai->ai_family = AF_INET;
    ai->ai_addrlen = sizeof(sa);
    ai->ai_addr = malloct(sizeof(sa));
    memcpy(ai->ai_addr, &sa, sizeof(sa));
  }
  return ai;
}

static void HTS_RESOLVER_CALL override_freeaddrinfo(struct addrinfo *res) {
  while (res != NULL) {
    struct addrinfo *const next = res->ai_next;

    freet(res->ai_addr);
    freet(res);
    res = next;
  }
}

static int HTS_RESOLVER_CALL override_getaddrinfo(const char *node,
                                                  const char *service,
                                                  const struct addrinfo *hints,
                                                  struct addrinfo **res) {
  const char *const spec = getenv("HTTRACK_DEBUG_RESOLVE");
  const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
  const char *colon;

  *res = NULL;
  if (spec != NULL && node != NULL && (colon = strchr(spec, ':')) != NULL &&
      (size_t) (colon - spec) == strlen(node) &&
      strncmp(spec, node, colon - spec) == 0) {
    struct addrinfo *head = NULL, *tail = NULL;
    char buf[256];
    char *p;

    buf[0] = '\0';
    strncatbuff(buf, colon + 1, sizeof(buf) - 1);
    for (p = strtok(buf, ","); p != NULL; p = strtok(NULL, ",")) {
      struct addrinfo *const ai = resolver_make_ai(p, want);

      if (ai != NULL) {
        if (head == NULL)
          head = ai;
        else
          tail->ai_next = ai;
        tail = ai;
      }
    }
    if (head == NULL)
      return EAI_NONAME;
    *res = head;
    return 0;
  }

  /* not overridden: delegate to libc, copying into our owned format */
  {
    struct addrinfo *sys = NULL;
    int gerr = getaddrinfo(node, service, hints, &sys);

    if (gerr != 0)
      return gerr;
    *res = resolver_dup_chain(sys);
    freeaddrinfo(sys);
    return 0;
  }
}

static const hts_resolver_backend hts_resolver_override = {
    override_getaddrinfo, override_freeaddrinfo};

/* Install the env override once, unless a backend was already set (self-test).
 */
static void hts_resolver_check_env(void) {
  static int checked = 0;

  if (!checked) {
    checked = 1;
    if (hts_resolver == &hts_resolver_libc &&
        getenv("HTTRACK_DEBUG_RESOLVE") != NULL) {
      hts_resolver = &hts_resolver_override;
    }
  }
}
#endif

// Resolve hostname into up to max addresses (resolver/RFC 6724 order), no
// cache. Returns the count copied into out[0..count-1]; 0 = does not resolve.
/* On a zero count, *permanent tells the two failures apart: the resolver
   saying the name does not exist, or it being unable to answer at all. */
static int hts_dns_resolve_nocache_list_(const char *const hostname,
                                         SOCaddr *const out, const int max,
                                         const char **error,
                                         hts_boolean *permanent) {
  int count = 0;
  hts_boolean is_permanent = HTS_FALSE;

#if HTS_INET6==0
  /* IPv4 resolver */
  struct hostent *const hp = gethostbyname(hostname);

  if (hp == NULL && h_errno == HOST_NOT_FOUND)
    is_permanent = HTS_TRUE;
  if (hp != NULL) {
    char **h;

    for (h = hp->h_addr_list; count < max && h != NULL && *h != NULL; h++) {
      SOCaddr_clear(out[count]);
      SOCaddr_copyaddr2(out[count], *h, hp->h_length);
      if (SOCaddr_is_valid(out[count]))
        count++;
    }
  }
#else
  /* IPv6 resolver */
  struct addrinfo *res = NULL, *cur;
  struct addrinfo hints;
  int gerr;

  hts_resolver_check_env();
  memset(&hints, 0, sizeof(hints));
  if (IPV6_resolver == 1) // V4 only (for bogus V6 entries)
    hints.ai_family = PF_INET;
  else if (IPV6_resolver == 2) // V6 only (for testing V6 only)
    hints.ai_family = PF_INET6;
  else // V4 + V6
    hints.ai_family = PF_UNSPEC;
  hints.ai_socktype = SOCK_STREAM;
  hints.ai_protocol = IPPROTO_TCP;
  if ((gerr = hts_resolver->getaddrinfo(hostname, NULL, &hints, &res)) == 0) {
    for (cur = res; cur != NULL && count < max; cur = cur->ai_next) {
      if (cur->ai_addr != NULL && cur->ai_addrlen != 0) {
        SOCaddr_clear(out[count]);
        SOCaddr_copyaddr2(out[count], cur->ai_addr, cur->ai_addrlen);
        if (SOCaddr_is_valid(out[count]))
          count++;
      }
    }
  } else {
    if (error != NULL)
      *error = gai_strerror(gerr);
    /* EAI_NONAME answers about the name; every other code says the resolver
       could not answer at all. */
    if (gerr == EAI_NONAME)
      is_permanent = HTS_TRUE;
  }
  if (res) {
    hts_resolver->freeaddrinfo(res);
  }
#endif

  if (permanent != NULL)
    *permanent = is_permanent;
  return count;
}

// Strip [] around a literal IPv6 ([3ffe:b80:1234:1::1]) the resolver won't
// take, then resolve into a list. Returns the count.
static int hts_dns_resolve_nocache_list(const char *const hostname,
                                        SOCaddr *const out, const int max,
                                        const char **error,
                                        hts_boolean *permanent) {
  if (!strnotempty(hostname) || max <= 0) {
    if (permanent != NULL)
      *permanent = HTS_FALSE;
    return 0;
  }
  if ((hostname[0] == '[') && (hts_lastchar(hostname) == ']')) {
    size_t size = strlen(hostname);
    char *copy = malloct(size + 1);
    int count;

    assertf(copy != NULL);
    copy[0] = '\0';
    strncat(copy, hostname + 1, size - 2);
    count = hts_dns_resolve_nocache_list_(copy, out, max, error, permanent);
    freet(copy);
    return count;
  } else {
    return hts_dns_resolve_nocache_list_(hostname, out, max, error, permanent);
  }
}

HTSEXT_API SOCaddr *hts_dns_resolve_nocache2(const char *const hostname,
                                             SOCaddr *const addr,
                                             const char **error) {
  SOCaddr_clear(*addr);
  if (hts_dns_resolve_nocache_list(hostname, addr, 1, error, NULL) > 0) {
    return SOCaddr_is_valid(*addr) ? addr : NULL;
  }
  return NULL;
}

HTSEXT_API SOCaddr* hts_dns_resolve_nocache(const char *const hostname, SOCaddr *const addr) {
  return hts_dns_resolve_nocache2(hostname, addr, NULL);
}

HTSEXT_API int check_hostname_dns(const char *const hostname) {
  SOCaddr buffer;
  return hts_dns_resolve_nocache(hostname, &buffer) != NULL;
}

/* A resolve in flight. Refcounted: a timed-out resolve is abandoned, not
   cancelled, so the last of caller/worker to leave frees the job. */
typedef struct dns_resolve_job {
  htsmutex lock;
  int refcount;
  hts_boolean done;
  char *hostname;
  SOCaddr addr[HTS_MAXADDRNUM];
  int count;
  const char *error;
  hts_boolean permanent; /* a zero count that was about the name itself */
} dns_resolve_job;

/* Copy the first min(count, max) addresses of src into dest. */
static void dns_copy_addrs(SOCaddr *dest, SOCaddr *src, int count, int max) {
  int i;

  for (i = 0; i < count && i < max; i++)
    SOCaddr_copy_SOCaddr(dest[i], src[i]);
}

static void dns_job_release(dns_resolve_job *job) {
  hts_boolean last;

  hts_mutexlock(&job->lock);
  last = (--job->refcount == 0) ? HTS_TRUE : HTS_FALSE;
  hts_mutexrelease(&job->lock);
  if (last) {
    hts_mutexfree(&job->lock);
    freet(job->hostname);
    freet(job);
  }
}

/* Outlives a timed-out resolve, so it writes only the job: never opt (freed
   before the thread wait at exit) nor the DNS cache. */
static void dns_resolve_thread(void *arg) {
  dns_resolve_job *const job = (dns_resolve_job *) arg;
  SOCaddr resolved[HTS_MAXADDRNUM];
  const char *error = NULL;
  hts_boolean permanent = HTS_FALSE;
  const int count = hts_dns_resolve_nocache_list(
      job->hostname, resolved, HTS_MAXADDRNUM, &error, &permanent);

  hts_mutexlock(&job->lock);
  dns_copy_addrs(job->addr, resolved, count, HTS_MAXADDRNUM);
  job->count = count;
  job->error = error;
  job->permanent = permanent;
  job->done = HTS_TRUE; /* published last: gates the caller's read of addr[] */
  hts_mutexrelease(&job->lock);
  dns_job_release(job);
}

/* Resolve hostname on a worker thread, giving up after timeout seconds or as
   soon as *cancel (optional) is raised. Returns the address count, or -1 on
   timeout -- distinct from 0 ("does not resolve"), which is a real answer and
   gets negative-cached. */
static int hts_dns_resolve_nocache_list_bounded(
    const char *hostname, SOCaddr *const out, const int max, const int timeout,
    const volatile hts_boolean *cancel, const char **error,
    hts_boolean *permanent) {
  dns_resolve_job *job;
  TStamp deadline;
  int count = -1;
  int poll_ms = 1;

  /* no bound asked for (--timeout 0), and nobody to cut it short either */
  if (timeout <= 0 && cancel == NULL)
    return hts_dns_resolve_nocache_list(hostname, out, max, error, permanent);

  job = calloct(1, sizeof(*job));
  assertf(job != NULL);
  hts_mutexinit(&job->lock);
  job->hostname = strdupt(hostname);
  job->refcount = 2; /* this caller + the worker */
  if (hts_newthread(dns_resolve_thread, job) != 0) {
    job->refcount = 1; /* no worker: fall back to resolving inline */
    dns_job_release(job);
    return hts_dns_resolve_nocache_list(hostname, out, max, error, permanent);
  }

  /* timeout <= 0 got here only for a cancellable resolve: no deadline then */
  deadline = timeout > 0 ? mtime_local() + (TStamp) timeout * 1000 : 0;
  for (;;) {
    hts_boolean done;

    hts_mutexlock(&job->lock);
    done = job->done;
    if (done) {
      count = job->count;
      dns_copy_addrs(out, job->addr, count, max);
      if (error != NULL)
        *error = job->error;
      if (permanent != NULL)
        *permanent = job->permanent;
    }
    hts_mutexrelease(&job->lock);
    if (done || (cancel != NULL && *cancel) ||
        (deadline != 0 && mtime_local() >= deadline))
      break;
    Sleep(poll_ms);
    if (poll_ms < 50) /* short first polls keep a fast resolve fast */
      poll_ms *= 2;
  }
  dns_job_release(job);
  return count;
}

int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
                        const char **error) {
  assertf(opt != NULL);
  /* the mirror's stop flag cancels: a resolve behind a black hole otherwise
     holds a stopped crawl for the whole --timeout (#1073) */
  return hts_dns_resolve_all_bounded(opt, iadr, out, max, opt->timeout,
                                     &opt->state.stop, error);
}

int hts_dns_resolve_all_bounded(httrackp *opt, const char *iadr, SOCaddr *out,
                                int max, int timeout,
                                const volatile hts_boolean *cancel,
                                const char **error) {
  char BIGSTK host[HTS_URLMAXSIZE * 2];
  SOCaddr resolved[HTS_MAXADDRNUM];
  hts_boolean permanent = HTS_FALSE;
  coucal cache;
  int count, i;

  assertf(opt != NULL);
  assertf(out != NULL);
  if (!strnotempty(iadr) || max <= 0) {
    return 0;
  }

  /* cache key and resolver input: identification and any ":port" stripped */
  strcpybuff(host, jump_identification_const(iadr));
  {
    char *a;

    if ((a = jump_toport(host)))
      *a = '\0';
  }

  hts_mutexlock(&opt->state.lock);
  cache = hts_cache(opt);
#if HTS_INET6 != 0
  hts_resolver_check_env(); /* settle the backend before a worker reads it */
#endif
  count = hts_ghbn_all(cache, host, out, max);
  hts_mutexrelease(&opt->state.lock);
  if (count >= 0) { // cache hit (0 == negative-cached)
    return count;
  }

#if DEBUGDNS
  printf("resolving (not cached) %s\n", host);
#endif

  /* Resolve with no lock held: getaddrinfo can block for a long time, and
     state.lock also gates the stop request (#606). */
  count = hts_dns_resolve_nocache_list_bounded(
      host, resolved, HTS_MAXADDRNUM, timeout, cancel, error, &permanent);

#if HTS_WIDE_DEBUG
  DEBUG_W("gethostbyname done\n");
#endif

  if (count < 0) { /* timed out: no answer to cache, and none to report */
    if (error != NULL)
      *error = "host name resolution timed out";
    return 0;
  }

  hts_mutexlock(&opt->state.lock);
  { /* store the full list (coucal owns the record and dups the host key; a
       concurrent resolve of the same host replaces, and frees, this one) */
    t_dnscache *const record = malloct(sizeof(t_dnscache));
    void *previous;
    const int failures =
        (count == 0 && coucal_read_pvoid(cache, host, &previous))
            ? ((const t_dnscache *) previous)->failures + 1
            : 1;

    if (record != NULL) {
      memset(record, 0, sizeof(*record));
      record->host_count = count;
      record->failures = count == 0 ? failures : 0;
      /* The resolver saying the name does not exist stands for the crawl;
         it being unable to answer is only as good as the network it asked. */
      if (count == 0 && !permanent) {
        record->stored = mtime_local();
        record->expiry = record->stored + hts_dns_negative_wait_ms(failures);
      }
      for (i = 0; i < count; i++) {
        record->host_length[i] = SOCaddr_size(resolved[i]);
        assertf(record->host_length[i] <= sizeof(record->host_addr[i]));
        memcpy(record->host_addr[i], &SOCaddr_sockaddr(resolved[i]),
               record->host_length[i]);
      }
      coucal_add_pvoid(cache, host, record);
    }
  }
  hts_mutexrelease(&opt->state.lock);

  /* copy result to caller (cache store may have failed; result still valid) */
  dns_copy_addrs(out, resolved, count, max);
  return count;
}

SOCaddr *hts_dns_resolve2(httrackp *opt, const char *_iadr, SOCaddr *const addr,
                          const char **error) {
  SOCaddr_clear(*addr);
  if (hts_dns_resolve_all(opt, _iadr, addr, 1, error) > 0) {
    return SOCaddr_is_valid(*addr) ? addr : NULL;
  }
  return NULL;
}

// --- Tracage des mallocs() ---
#ifdef HTS_TRACE_MALLOC
#define htsLocker(A, N) do {} while(0)
static mlink trmalloc = { NULL, 0, 0, NULL };

static int trmalloc_id = 0;
static htsmutex *mallocMutex = NULL;

static void hts_meminit(void) {}
void *hts_malloc(size_t len) {
  void *adr;

  hts_meminit();
  htsLocker(mallocMutex, 1);
  assertf(len > 0);
  adr = hts_xmalloc(len, 0);
  htsLocker(mallocMutex, 0);
  return adr;
}
void *hts_calloc(size_t len, size_t len2) {
  void *adr;

  hts_meminit();
  assertf(len > 0);
  assertf(len2 > 0);
  htsLocker(mallocMutex, 1);
  adr = hts_xmalloc(len, len2);
  htsLocker(mallocMutex, 0);
  memset(adr, 0, len * len2);
  return adr;
}
void *hts_strdup(char *str) {
  size_t size = str ? strlen(str) : 0;
  char *adr = (char *) hts_malloc(size + 1);

  assertf(adr != NULL);
  strcpy(adr, str ? str : "");
  return adr;
}
void *hts_xmalloc(size_t len, size_t len2) {
  mlink *lnk = (mlink *) calloc(1, sizeof(mlink));

  assertf(lnk != NULL);
  assertf(len > 0);
  assertf(len2 >= 0);
  if (lnk) {
    void *r = NULL;
    int size, bsize = sizeof(t_htsboundary);

    if (len2)
      size = len * len2;
    else
      size = len;
    size += ((bsize - (size % bsize)) % bsize); /* check alignement */
    r = malloc(size + bsize * 2);
    assertf(r != NULL);
    if (r) {
      *((t_htsboundary *) ((char *) r))
        = *((t_htsboundary *) ((char *) r + size + bsize))
        = htsboundary;
      ((char *) r) += bsize;    /* boundary */
      lnk->adr = r;
      lnk->len = size;
      lnk->id = trmalloc_id++;
      lnk->next = trmalloc.next;
      trmalloc.next = lnk;
      return r;
    } else {
      free(lnk);
    }
  }
  return NULL;
}
void hts_free(void *adr) {
  mlink *lnk = &trmalloc;
  int bsize = sizeof(t_htsboundary);

  assertf(adr != NULL);
  if (!adr) {
    return;
  }
  htsLocker(mallocMutex, 1);
  while(lnk->next != NULL) {
    if (lnk->next->adr == adr) {
      mlink *blk_free = lnk->next;

      assertf(blk_free->id != -1);
      assertf(*((t_htsboundary *) ((char *) adr - bsize)) == htsboundary);
      assertf(*((t_htsboundary *) ((char *) adr + blk_free->len)) ==
              htsboundary);
      lnk->next = lnk->next->next;
      free((void *) blk_free);
      free((char *) adr - bsize);
      htsLocker(mallocMutex, 0);
      return;
    }
    lnk = lnk->next;
    assertf(lnk->next != NULL);
  }
  free(adr);
  htsLocker(mallocMutex, 0);
}
void *hts_realloc(void *adr, size_t len) {
  int bsize = sizeof(t_htsboundary);

  len += ((bsize - (len % bsize)) % bsize);     /* check alignement */
  if (adr != NULL) {
    mlink *lnk = &trmalloc;

    htsLocker(mallocMutex, 1);
    while(lnk->next != NULL) {
      if (lnk->next->adr == adr) {
        {
          mlink *blk_free = lnk->next;

          assertf(blk_free->id != -1);
          assertf(*((t_htsboundary *) ((char *) adr - bsize)) == htsboundary);
          assertf(*((t_htsboundary *) ((char *) adr + blk_free->len)) ==
                  htsboundary);
        }
        adr = realloc((char *) adr - bsize, len + bsize * 2);
        assertf(adr != NULL);
        lnk->next->adr = (char *) adr + bsize;
        lnk->next->len = len;
        *((t_htsboundary *) ((char *) adr))
          = *((t_htsboundary *) ((char *) adr + len + bsize))
          = htsboundary;
        htsLocker(mallocMutex, 0);
        return (char *) adr + bsize;
      }
      lnk = lnk->next;
      assertf(lnk->next != NULL);
    }
    htsLocker(mallocMutex, 0);
  }
  return hts_malloc(len);
}
mlink *hts_find(char *adr) {
  char *stkframe = (char *) &stkframe;
  mlink *lnk = &trmalloc;
  int bsize = sizeof(t_htsboundary);

  assertf(adr != NULL);
  if (!adr) {
    return NULL;
  }
  htsLocker(mallocMutex, 1);
  while(lnk->next != NULL) {
    if (adr >= lnk->next->adr && adr <= lnk->next->adr + lnk->next->len) {      /* found */
      htsLocker(mallocMutex, 0);
      return lnk->next;
    }
    lnk = lnk->next;
  }
  htsLocker(mallocMutex, 0);
  {
    int depl = (int) (adr - stkframe);

    if (depl < 0)
      depl = -depl;
    return NULL;
  }
}

// check the malloct() and calloct() trace stack
void hts_freeall(void) {
  int bsize = sizeof(t_htsboundary);

  while(trmalloc.next) {
#if MEMDEBUG
    printf("* block %d\t not released: at %d\t (%d\t bytes)\n",
           trmalloc.next->id, trmalloc.next->adr, trmalloc.next->len);
#endif
    if (trmalloc.next->id != -1) {
      free((char *) trmalloc.next->adr - bsize);
    }
  }
}
#endif

// -- divers //

// cut path and project name
// patch also initial path
void cut_path(char *fullpath, char *path, size_t path_size, char *pname,
              size_t pname_size) {
  path[0] = pname[0] = '\0';
  if (strnotempty(fullpath)) {
    if (!hts_striplastchar(fullpath, '/'))
      hts_striplastchar(fullpath, '\\');
    if (strlen(fullpath) > 1) {
      char *a;

      while((a = strchr(fullpath, '\\')))
        *a = '/';               // remplacer par /
      a = fullpath + strlen(fullpath) - 2;
      while((*a != '/') && (a > fullpath))
        a--;
      if (*a == '/')
        a++;
      strlcpybuff(pname, a, pname_size);
      strlncatbuff(path, fullpath, path_size, (size_t) (a - fullpath));
    }
  }
}

// -- Gestion protocole ftp --

#ifdef _WIN32
int ftp_available(void) {
  return 1;
}
#else
int ftp_available(void) {
  return 1; // ok!
}
#endif

static void hts_debug_log_print(const char *format, ...) HTS_PRINTF_FUN(1, 2);

static int hts_dgb_init = 0;
static FILE *hts_dgb_init_fp = NULL;
HTSEXT_API void hts_debug(int level) {
  hts_dgb_init = level;
  if (hts_dgb_init > 0) {
    hts_debug_log_print("hts_debug() called");
  }
}

static FILE *hts_dgb_(void) {
  if (hts_dgb_init_fp == NULL) {
    if ((hts_dgb_init & 0x80) == 0) {
      hts_dgb_init_fp = stderr;
    } else {
      hts_dgb_init_fp = FOPEN("hts-debug.txt", "wb");
      if (hts_dgb_init_fp != NULL) {
        fprintf(hts_dgb_init_fp, "* Creating file\r\n");
      }
    }
  }
  return hts_dgb_init_fp;
}

static void hts_debug_log_print(const char *format, ...) {
  if (hts_dgb_init > 0) {
    const int error = errno;
    FILE *const fp = hts_dgb_();
    va_list args;

    assertf(format != NULL);
    va_start(args, format);
    (void) vfprintf(fp, format, args);
    va_end(args);
    fputs("\n", fp);
    fflush(fp);
    errno = error;
  }
}

HTSEXT_API const char* hts_version(void) {
  return HTTRACK_VERSIONID;
}

static int ssl_vulnerable(const char *version) {
#ifdef _WIN32
  static const char *const match = "OpenSSL 1.0.1";
  const size_t match_len = strlen(match);
  if (version != NULL && strncmp(version, match, match_len) == 0) {
    // CVE-2014-0160
    // "OpenSSL 1.0.1g 7 Apr 2014"
    const char minor = version[match_len];
    return minor == ' ' || ( minor >= 'a' && minor <= 'f' );
  }
#endif
  return 0;
}

/* user abort callback */
htsErrorCallback htsCallbackErr = NULL;

HTSEXT_API void hts_set_error_callback(htsErrorCallback handler) {
  htsCallbackErr = handler;
}

HTSEXT_API htsErrorCallback hts_get_error_callback(void) {
  return htsCallbackErr;
}

static void default_coucal_asserthandler(void *arg, const char* exp, const char* file, int line) {
  abortf_(exp, file, line);
}

static int get_loglevel_from_coucal(coucal_loglevel level) {
  switch(level) {
  case coucal_log_critical:
    return LOG_PANIC;
    break;
  case coucal_log_warning:
    return LOG_WARNING;
    break;
  case coucal_log_info:
    return LOG_INFO;
    break;
  case coucal_log_debug:
    return LOG_DEBUG;
    break;
  case coucal_log_trace:
    return LOG_TRACE;
    break;
  default:
    return LOG_ERROR;
    break;
  }
}

/* log to default console */
static HTS_PRINTF_FUN(3, 0) void default_coucal_loghandler(
    void *arg, coucal_loglevel level, const char *format, va_list args) {

  /* informational chatter (hashtable stats on delete, etc.) only when
     debugging; keep warnings and critical errors always visible. */
  if (level > coucal_log_warning && hts_dgb_init <= 0) {
    return;
  }
  if (level <= coucal_log_warning) {
    fprintf(stderr, "** warning: ");
  }
  vfprintf(stderr, format, args);
  fprintf(stderr, "\n");
}

/* log to project log */
static HTS_PRINTF_FUN(3, 0) void htsopt_coucal_loghandler(void *arg,
                                                          coucal_loglevel level,
                                                          const char *format,
                                                          va_list args) {
  httrackp *const opt = (httrackp*) arg;
  if (opt != NULL && opt->log != NULL) {
    hts_log_vprint(opt, get_loglevel_from_coucal(level), 
      format, args);
  } else {
    default_coucal_loghandler(NULL, level, format, args);
  }
}

/* attach hashtable logger to project log */
void hts_set_hash_handler(coucal hashtable, httrackp *opt) {
  /* Init hashtable default assertion handler. */
  coucal_set_assert_handler(hashtable,
    htsopt_coucal_loghandler, 
    default_coucal_asserthandler,
    opt);
}

static int hts_init_ok = 0;
HTSEXT_API int hts_init(void) {
  const char *dbg_env;

  /* */
  if (hts_init_ok)
    return 1;
  hts_init_ok = 1;

  /* enable debugging ? */
  dbg_env = getenv("HTS_LOG");
  if (dbg_env != NULL && *dbg_env != 0) {
    int level = 0;

    if (sscanf(dbg_env, "%d", &level) == 1) {
      hts_debug(level);
    }
  }

  hts_debug_log_print("entering hts_init()");   /* debug */

  /* Init hashtable default assertion handler. */
  coucal_set_global_assert_handler(default_coucal_loghandler, 
    default_coucal_asserthandler);

  /* Init threads (lazy init) */
  htsthread_init();

  /* Ensure external modules are loaded */
  hts_debug_log_print("calling htspe_init()");  /* debug */
  htspe_init();                 /* module load (lazy) */

  /* MD5 Auto-test */
  {
    char digest[32 + 2];
    const char *atest = "MD5 Checksum Autotest";

    digest[0] = '\0';
    domd5mem(atest, strlen(atest), digest, 1);  /* a42ec44369da07ace5ec1d660ba4a69a */
    if (strcmp(digest, "a42ec44369da07ace5ec1d660ba4a69a") != 0) {
      int fatal_broken_md5 = 0;

      assertf(fatal_broken_md5);
    }
  }

  hts_debug_log_print("initializing SSL");      /* debug */
#if HTS_USEOPENSSL
  /*
     Initialize the OpensSSL library
   */
  if (!openssl_ctx) {
    const char *version;
    const SSL_METHOD *method;

    /* OpenSSL >= 1.1.0 / LibreSSL >= 2.7.0 auto-init and provide the generic
       methods. The legacy init and SSLv23/SSLeay calls (deprecated since 1.1.0,
       likely gone in 4.0) are kept only for older OpenSSL. */
#if OPENSSL_VERSION_NUMBER < 0x10100000L \
  || (defined(LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x2070000fL)
    SSL_load_error_strings();
    SSL_library_init();
    version = SSLeay_version(SSLEAY_VERSION);
    method = SSLv23_client_method();
#else
    version = OpenSSL_version(OPENSSL_VERSION);
    method = TLS_client_method();
#endif

    // Check CVE-2014-0160.
    if (ssl_vulnerable(version)) {
      fprintf(stderr, "OpenSSL version == '%s'\n", version);
      abortLog("unable to initialize TLS: OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)");
      assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
    }

    openssl_ctx = SSL_CTX_new(method);
    if (!openssl_ctx) {
      fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
      abortLog("unable to initialize TLS: SSL_CTX_new()");
      assertf("unable to initialize TLS" == NULL);
    }
    /* Pin a TLS floor (no SSLv3/TLS1.0/1.1); no cert verify, by design. */
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
    SSL_CTX_set_min_proto_version(openssl_ctx, TLS1_2_VERSION);
#else
    SSL_CTX_set_options(openssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 |
                                         SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1);
#endif
  }
#endif

  hts_debug_log_print("ending hts_init()");     /* debug */
  return 1;
}

/* will not free thread env. */
HTSEXT_API int hts_uninit(void) {
  /* hts_init() is a lazy initializer, with limited a allocation (one or two mutexes) ;
     we won't free anything here as the .h semantic was never being very clear */
  return 1;
}

HTSEXT_API int hts_uninit_module(void) {
  if (!hts_init_ok)
    return 1;
  htsthread_uninit();
  htspe_uninit();
  hts_init_ok = 0;
  return 1;
}

// legacy. do not use
HTSEXT_API hts_boolean hts_log(httrackp *opt, const char *prefix,
                               const char *msg) {
  if (opt->log != NULL) {
    fspc(opt, opt->log, prefix);
    fprintf(opt->log, "%s" LF, msg);
    return 0;
  }
  return 1;                     /* Error */
}

static void (*hts_log_print_callback)(httrackp * opt, int type, const char *format, va_list args) = NULL;

HTSEXT_API void hts_set_log_vprint_callback(void (*callback)(httrackp * opt,
                                            int type, const char *format, va_list args)) {
  hts_log_print_callback = callback;
}

HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_list args) {
  assertf(format != NULL);
  if (hts_log_print_callback != NULL) {
    va_list args_copy;
    va_copy(args_copy, args);
    /* the copy, so the vfprintf() below still has an unread list */
    hts_log_print_callback(opt, type, format, args_copy);
    va_end(args_copy);
  }
  if (opt != NULL && opt->log != NULL) {
    const int save_errno = errno;
    const char *s_type = "unknown";
    const int level = type & 0xff;

    // Check log level
    if (opt->debug < level) {
      return;
    }

    switch (level) {
    case LOG_TRACE:
      s_type = "trace";
      break;
    case LOG_DEBUG:
      s_type = "debug";
      break;
    case LOG_INFO:
    case LOG_NOTICE: /* not a warning: counted in the footer's messages */
      s_type = "info";
      break;
    case LOG_WARNING:
      s_type = "warning";
      break;
    case LOG_ERROR:
      s_type = "error";
      break;
    case LOG_PANIC:
      s_type = "panic";
      break;
    }
    fspc(opt, opt->log, s_type);
    (void) vfprintf(opt->log, format, args);
    if ((type & LOG_ERRNO) != 0) {
      fprintf(opt->log, ": %s", strerror(save_errno));
    }
    fputs(LF, opt->log);
    if (opt->flush) {
      fflush(opt->log);
    }
    errno = save_errno;
  }
}

HTSEXT_API void hts_log_print(httrackp * opt, int type, const char *format, ...) {
  va_list args;
  assertf(format != NULL);
  va_start(args, format);
  hts_log_vprint(opt, type, format, args);
  va_end(args);
}

HTSEXT_API void set_wrappers(httrackp * opt) {  // LEGACY
}

HTSEXT_API int plug_wrapper(httrackp * opt, const char *moduleName,
                            const char *argv) {
  void *handle = openFunctionLib(moduleName);

  if (handle != NULL) {
    t_hts_plug plug = (t_hts_plug) getFunctionPtr(handle, "hts_plug");
    t_hts_unplug unplug = (t_hts_unplug) getFunctionPtr(handle, "hts_unplug");

    if (plug != NULL) {
      int ret = plug(opt, argv);

      if (hts_dgb_init > 0 && opt->log != NULL) {
        hts_debug_log_print("plugged module '%s' (return code=%d)", moduleName,
                            ret);
      }
      if (ret == 1) {           /* Success! */
        opt->libHandles.handles =
          (htslibhandle *) realloct(opt->libHandles.handles,
                                    (opt->libHandles.count +
                                     1) * sizeof(htslibhandle));
        opt->libHandles.handles[opt->libHandles.count].handle = handle;
        opt->libHandles.handles[opt->libHandles.count].moduleName =
          strdupt(moduleName);
        opt->libHandles.count++;
        return 1;
      } else {
        hts_debug_log_print
          ("* note: error while running entry point 'hts_plug' in %s",
           moduleName);
        if (unplug)
          unplug(opt);
      }
    } else {
      int last_errno = errno;

      hts_debug_log_print("* note: can't find entry point 'hts_plug' in %s: %s",
                          moduleName, strerror(last_errno));
    }
    closeFunctionLib(handle);
    return 0;
  } else {
    int last_errno = errno;

    hts_debug_log_print("* note: can't load %s: %s", moduleName,
                        strerror(last_errno));
  }
  return -1;
}

static void unplug_wrappers(httrackp * opt) {
  if (opt->libHandles.handles != NULL) {
    int i;

    for(i = 0; i < opt->libHandles.count; i++) {
      if (opt->libHandles.handles[i].handle != NULL) {
        /* hts_unplug(), the dll exit point (finalizer) */
        t_hts_unplug unplug =
          (t_hts_unplug) getFunctionPtr(opt->libHandles.handles[i].handle,
                                        "hts_unplug");
        if (unplug != NULL)
          unplug(opt);
        closeFunctionLib(opt->libHandles.handles[i].handle);
        opt->libHandles.handles[i].handle = NULL;
      }
      if (opt->libHandles.handles[i].moduleName != NULL) {
        freet(opt->libHandles.handles[i].moduleName);
        opt->libHandles.handles[i].moduleName = NULL;
      }
    }
    freet(opt->libHandles.handles);
    opt->libHandles.handles = NULL;
    opt->libHandles.count = 0;
  }
}

int multipleStringMatch(const char *s, const char *match) {
  int ret = 0;
  String name = STRING_EMPTY;

  if (match == NULL || s == NULL || *s == 0)
    return 0;
  for(; *match != 0; match++) {
    StringClear(name);
    for(; *match != 0 && *match != '\n'; match++) {
      StringAddchar(name, *match);
    }
    if (StringLength(name) > 0 && strstr(s, StringBuff(name)) != NULL) {
      ret = 1;
      break;
    }
  }
  StringFree(name);
  return ret;
}

HTSEXT_API httrackp *hts_create_opt(void) {
  static const char *defaultModules[] = {"httrack-plugin", NULL};
  httrackp *opt = malloc(sizeof(httrackp));

  /* default options */
  memset(opt, 0, sizeof(httrackp));
  opt->size_httrackp = sizeof(httrackp);

  /* mutexes */
  hts_mutexinit(&opt->state.lock);

  /* custom wrappers */
  opt->libHandles.count = 0;

  /* default settings */

  opt->wizard = HTS_WIZARD_AUTO; // wizard automatique
  opt->quiet = HTS_FALSE;
  //
  opt->travel = HTS_TRAVEL_SAME_ADDRESS; // même adresse
  opt->depth = 9999;            // mirror total par défaut
  opt->extdepth = 0;            // mais pas à l'extérieur
  opt->seeker = HTS_SEEKER_DOWN;       // down
  opt->urlmode = HTS_URLMODE_RELATIVE; // relatif par défaut
  opt->no_type_change = HTS_FALSE;
  opt->debug = LOG_NOTICE;      // small log
  opt->getmode = HTS_GETMODE_HTML | HTS_GETMODE_NONHTML;
  opt->maxsite = -1;            // taille max site (aucune)
  opt->maxfile_nonhtml = -1;    // taille max fichier non html
  opt->maxfile_html = -1;       // idem pour html
  opt->maxsoc = 4;              // nbre socket max
  opt->fragment = -1;           // pas de fragmentation
  opt->nearlink = HTS_FALSE;
  opt->makeindex = HTS_TRUE;
  opt->kindex = HTS_FALSE;
  opt->delete_old = HTS_TRUE;
  opt->background_on_suspend = HTS_TRUE;
  opt->makestat = HTS_FALSE;
  opt->maketrack = HTS_FALSE;
  opt->timeout = 120;           // timeout par défaut (2 minutes)
  opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
  opt->shell = HTS_FALSE;
  opt->proxy.active = 0;        // pas de proxy
  opt->user_agent_send = HTS_TRUE;
  StringCopy(opt->user_agent, HTS_DEFAULT_USER_AGENT);
  StringCopy(opt->referer, "");
  StringCopy(opt->from, "");
  opt->savename_83 = HTS_SAVENAME_83_LONG; // long names by default
  opt->savename_type = 0;       // avec structure originale
  opt->savename_delayed =
      HTS_SAVENAME_DELAYED_HARD; // always delay the type check (default)
  opt->delayed_cached = HTS_TRUE;
  opt->mimehtml = HTS_FALSE;
  opt->parsejava = HTSPARSE_DEFAULT;    // parser classes
  opt->hostcontrol = 0;         // PAS de control host pour timeout et traffic jammer
  opt->retry = 2;               // 2 retry par défaut
  opt->errpage = HTS_TRUE;
  // d'erreur (404 etc.)
  opt->check_type = HTS_TRUE;
  // considéré comme html
  opt->all_in_cache = HTS_FALSE;
  opt->robots = HTS_ROBOTS_ALWAYS; // traiter les robots.txt
  opt->external = HTS_FALSE;
  opt->passprivacy = HTS_FALSE;
  opt->includequery = HTS_TRUE;
  opt->mirror_first_page = HTS_FALSE;
  opt->accept_cookie = HTS_TRUE;
  opt->cookie = NULL;
  opt->http10 = HTS_FALSE;
  opt->nokeepalive = HTS_FALSE;
  opt->nocompression = HTS_FALSE;
  opt->tolerant = HTS_FALSE;
  opt->parseall = HTS_TRUE;
  opt->parsedebug = HTS_FALSE;
  opt->norecatch = HTS_FALSE;
  opt->verbosedisplay = HTS_VERBOSE_NONE; // no text animation
  opt->sizehack = HTS_FALSE;
  opt->urlhack = HTS_TRUE;
  opt->no_www_dedup = HTS_FALSE;
  opt->no_slash_dedup = HTS_FALSE;
  opt->no_query_dedup = HTS_FALSE;
  StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
  StringCopy(opt->strip_query, "");
  StringCopy(opt->host_alias, "");
  StringCopy(opt->cookies_file, "");
  StringCopy(opt->warc_file, "");
  StringCopy(opt->sitemap_url, "");
  opt->warc_max_size = 0; /* no rotation unless --warc-max-size sets it */
  opt->changes = HTS_FALSE;
  opt->changes_state = NULL;
  opt->single_file = HTS_FALSE;
  opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
  opt->singlefile_state = NULL;
  opt->links_unqueued = HTS_FALSE;
  StringCopy(opt->why_url, "");
  opt->pause_min_ms = 0;
  opt->pause_max_ms = 0;
  opt->ftp_proxy = HTS_TRUE;
  opt->convert_utf8 = HTS_TRUE;
  StringCopy(opt->filelist, "");
  StringCopy(opt->lang_iso, "en, *");
  StringCopy(opt->accept,
    "text/html,image/png,image/jpeg,image/pjpeg,image/x-xbitmap,image/svg+xml,image/gif;q=0.9,*/*;q=0.1");
  StringCopy(opt->headers, "");
  StringCopy(opt->mimedefs, "\n");      // aucun filtre mime (\n IMPORTANT)
  StringClear(opt->mod_blacklist);
  //
  opt->log = stdout;
  opt->errlog = stderr;
  opt->flush = HTS_TRUE;
  opt->keyboard = HTS_FALSE;
  //
  StringCopy(opt->path_html, "");
  StringCopy(opt->path_html_utf8, "");
  StringCopy(opt->path_log, "");
  StringCopy(opt->path_bin, "");
  //
  opt->maxlink = 100000;        // 100,000 liens max par défaut
  opt->maxfilter = 200;         // 200 filtres max par défaut
  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
                                // PARAMETRABLE POUR L'INSTANT --
  opt->maxtime = -1;            // temps max en secondes
  opt->maxrate = 100000;        // taux maxi
  opt->maxconn = 5.0;           // nombre connexions/s
  opt->waittime = -1;           // wait until.. hh*3600+mm*60+ss
  //
  opt->exec = "";
  opt->is_update = HTS_FALSE;
  opt->dir_topindex = HTS_FALSE;
  //
  opt->bypass_limits = HTS_FALSE;
  opt->state.stop = 0;          // stopper
  opt->state.exit_xh = 0;       // abort
  //
  opt->state.is_ended = 0;

  /* Alocated buffers */

  opt->callbacks_fun =
    (t_hts_htmlcheck_callbacks *) malloct(sizeof(t_hts_htmlcheck_callbacks));
  memset(opt->callbacks_fun, 0, sizeof(t_hts_htmlcheck_callbacks));

  /* Preload callbacks : java and flash parser, and the automatic user-defined callback */

  {
    int i;

    for(i = 0; defaultModules[i] != NULL; i++) {
      int ret = plug_wrapper(opt, defaultModules[i], defaultModules[i]);

      if (ret == 0) {           /* Module aborted initialization */
        /* Ignored. */
      }
    }
  }

  return opt;
}

HTSEXT_API size_t hts_sizeof_opt(void) {
  return sizeof(httrackp);
}

HTSEXT_API void hts_free_opt(httrackp * opt) {
  if (opt != NULL) {
    /* An FTP worker reads opt for its whole run, and not every caller drains
       its threads the way httrack.c does before getting here. */
    ftp_stop_workers();

    /* Alocated callbacks */

    if (opt->callbacks_fun != NULL) {
      int i;
      t_hts_htmlcheck_callbacks_item *items =
        (t_hts_htmlcheck_callbacks_item *) opt->callbacks_fun;
      const int size =
        (int) sizeof(t_hts_htmlcheck_callbacks) /
        sizeof(t_hts_htmlcheck_callbacks_item);
      assertf(sizeof(t_hts_htmlcheck_callbacks_item) * size ==
              sizeof(t_hts_htmlcheck_callbacks));

      /* Free all linked lists */
      for(i = 0; i < size; i++) {
        t_hts_callbackarg *carg, *next_carg;

        for(carg = items[i].carg;
            carg != NULL && (next_carg = carg->prev.carg, carg != NULL);
            carg = next_carg) {
          hts_free(carg);
        }
      }

      freet(opt->callbacks_fun);
      opt->callbacks_fun = NULL;
    }

    /* Close library handles */
    unplug_wrappers(opt);

    /* Cache */
    if (opt->state.dns_cache != NULL) {
      coucal root;

      hts_mutexlock(&opt->state.lock);
      root = opt->state.dns_cache;
      opt->state.dns_cache = NULL;
      hts_mutexrelease(&opt->state.lock);

      coucal_delete(&root); // frees records via hts_cache_value_free
    }

    /* URLs a front end queued but the engine never got to inject */
    hts_addurl_free(hts_addurl_take(opt));

    /* Cancel chain */
    if (opt->state.cancel != NULL) {
      htsoptstatecancel *cancel;

      for(cancel = opt->state.cancel; cancel != NULL;) {
        htsoptstatecancel *next = cancel->next;

        if (cancel->url != NULL) {
          freet(cancel->url);
        }
        freet(cancel);
        cancel = next;
      }
      opt->state.cancel = NULL;
    }

    /* Free strings */

    StringFree(opt->proxy.name);
    StringFree(opt->proxy.bindhost);

    StringFree(opt->savename_userdef);
    StringFree(opt->user_agent);
    StringFree(opt->referer);
    StringFree(opt->from);
    StringFree(opt->lang_iso);
    StringFree(opt->accept);
    StringFree(opt->headers);
    StringFree(opt->sys_com);
    StringFree(opt->mimedefs);
    StringFree(opt->filelist);
    StringFree(opt->urllist);
    StringFree(opt->footer);
    StringFree(opt->mod_blacklist);
    StringFree(opt->strip_query);
    StringFree(opt->host_alias);
    StringFree(opt->cookies_file);
    StringFree(opt->why_url);
    StringFree(opt->warc_file);
    StringFree(opt->sitemap_url);
    hts_sitemap_free(opt); /* backstop: httpmirror's early-return paths */
    singlefile_free(opt);

    hts_changes_free_opt(opt);

    StringFree(opt->path_html);
    StringFree(opt->path_html_utf8);
    StringFree(opt->path_log);
    StringFree(opt->path_bin);

    /* mutexes */
    hts_mutexfree(&opt->state.lock);

    /* Free structure */
    free(opt);
  }
}

// TEMPORARY - PUT THIS STRUCTURE INSIDE httrackp !
const hts_stat_struct* hts_get_stats(httrackp * opt) {
  if (opt == NULL) {
    return NULL;
  }

  HTS_STAT.stat_nsocket = 0;
  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
  HTS_STAT.nbk = 0;
  HTS_STAT.nb = 0;

  return &HTS_STAT;
}

// defaut wrappers
static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}

static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
  return 1;
}
static int __cdecl htsdefault_chopt(t_hts_callbackarg * carg, httrackp * opt) {
  return 1;
}
static int __cdecl htsdefault_end(t_hts_callbackarg * carg, httrackp * opt) {
  return 1;
}
static int __cdecl htsdefault_preprocesshtml(t_hts_callbackarg * carg,
                                             httrackp * opt, char **html,
                                             int *len, const char *url_adresse,
                                             const char *url_fichier) {
  return 1;
}
static int __cdecl htsdefault_postprocesshtml(t_hts_callbackarg * carg,
                                              httrackp * opt, char **html,
                                              int *len, const char *url_adresse,
                                              const char *url_fichier) {
  return 1;
}
static int __cdecl htsdefault_checkhtml(t_hts_callbackarg * carg,
                                        httrackp * opt, char *html, int len,
                                        const char *url_adresse,
                                        const char *url_fichier) {
  return 1;
}
static int __cdecl htsdefault_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * back, int back_max, int back_index, int lien_n, int lien_tot, int stat_time, hts_stat_struct * stats) {        // appelé à chaque boucle de HTTrack
  return 1;
}
static const char *__cdecl htsdefault_query(t_hts_callbackarg * carg,
                                            httrackp * opt,
                                            const char *question) {
  return "";
}
static const char *__cdecl htsdefault_query2(t_hts_callbackarg * carg,
                                             httrackp * opt,
                                             const char *question) {
  return "";
}
static const char *__cdecl htsdefault_query3(t_hts_callbackarg * carg,
                                             httrackp * opt,
                                             const char *question) {
  return "";
}
static int __cdecl htsdefault_check(t_hts_callbackarg * carg, httrackp * opt,
                                    const char *adr, const char *fil,
                                    int status) {
  return -1;
}
static int __cdecl htsdefault_check_mime(t_hts_callbackarg * carg,
                                         httrackp * opt, const char *adr,
                                         const char *fil, const char *mime,
                                         int status) {
  return -1;
}
static void __cdecl htsdefault_pause(t_hts_callbackarg * carg, httrackp * opt,
                                     const char *lockfile) {
  while(fexist(lockfile)) {
    Sleep(1000);
  }
}
static void __cdecl htsdefault_filesave(t_hts_callbackarg * carg,
                                        httrackp * opt, const char *file) {
}
static void __cdecl htsdefault_filesave2(t_hts_callbackarg * carg,
                                         httrackp * opt, const char *adr,
                                         const char *file, const char *sav,
                                         int is_new, int is_modified,
                                         int not_updated) {
}
static int __cdecl htsdefault_linkdetected(t_hts_callbackarg * carg,
                                           httrackp * opt, char *link) {
  return 1;
}
static int __cdecl htsdefault_linkdetected2(t_hts_callbackarg * carg,
                                            httrackp * opt, char *link,
                                            const char *start_tag) {
  return 1;
}
static int __cdecl htsdefault_xfrstatus(t_hts_callbackarg * carg,
                                        httrackp * opt, lien_back * back) {
  return 1;
}
static int __cdecl htsdefault_savename(t_hts_callbackarg * carg, httrackp * opt,
                                       const char *adr_complete,
                                       const char *fil_complete,
                                       const char *referer_adr,
                                       const char *referer_fil, char *save) {
  return 1;
}
static int __cdecl htsdefault_sendhead(t_hts_callbackarg * carg, httrackp * opt,
                                       char *buff, const char *adr,
                                       const char *fil, const char *referer_adr,
                                       const char *referer_fil,
                                       htsblk * outgoing) {
  return 1;
}
static int __cdecl htsdefault_receivehead(t_hts_callbackarg * carg,
                                          httrackp * opt, char *buff,
                                          const char *adr, const char *fil,
                                          const char *referer_adr,
                                          const char *referer_fil,
                                          htsblk * incoming) {
  return 1;
}
static int __cdecl htsdefault_detect(t_hts_callbackarg * carg, httrackp * opt,
                                     htsmoduleStruct * str) {
  return 0;
}
static int __cdecl htsdefault_parse(t_hts_callbackarg * carg, httrackp * opt,
                                    htsmoduleStruct * str) {
  return 0;
}

/* Default internal dummy callbacks */
const t_hts_htmlcheck_callbacks default_callbacks = {
  {htsdefault_init, NULL},
  {htsdefault_uninit, NULL},
  {htsdefault_start, NULL},
  {htsdefault_end, NULL},
  {htsdefault_chopt, NULL},
  {htsdefault_preprocesshtml, NULL},
  {htsdefault_postprocesshtml, NULL},
  {htsdefault_checkhtml, NULL},
  {htsdefault_query, NULL},
  {htsdefault_query2, NULL},
  {htsdefault_query3, NULL},
  {htsdefault_loop, NULL},
  {htsdefault_check, NULL},
  {htsdefault_check_mime, NULL},
  {htsdefault_pause, NULL},
  {htsdefault_filesave, NULL},
  {htsdefault_filesave2, NULL},
  {htsdefault_linkdetected, NULL},
  {htsdefault_linkdetected2, NULL},
  {htsdefault_xfrstatus, NULL},
  {htsdefault_savename, NULL},
  {htsdefault_sendhead, NULL},
  {htsdefault_receivehead, NULL},
  {htsdefault_detect, NULL},
  {htsdefault_parse, NULL}
};

#define CALLBACK_OP(CB, NAME, OPERATION, S, FUN) do {   \
  if (strcmp(NAME, S) == 0) {                           \
    OPERATION(t_hts_htmlcheck_ ##FUN, (CB)->FUN.fun);   \
  }                                                     \
} while(0)

#define DISPATCH_CALLBACK(CB, NAME, OPERATION) do { \
  CALLBACK_OP(CB, NAME, OPERATION, "init", init); \
  CALLBACK_OP(CB, NAME, OPERATION, "free", uninit); \
  CALLBACK_OP(CB, NAME, OPERATION, "start", start); \
  CALLBACK_OP(CB, NAME, OPERATION, "end", end); \
  CALLBACK_OP(CB, NAME, OPERATION, "change-options", chopt); \
  CALLBACK_OP(CB, NAME, OPERATION, "preprocess-html", preprocess); \
  CALLBACK_OP(CB, NAME, OPERATION, "postprocess-html", postprocess); \
  CALLBACK_OP(CB, NAME, OPERATION, "check-html", check_html); \
  CALLBACK_OP(CB, NAME, OPERATION, "query", query); \
  CALLBACK_OP(CB, NAME, OPERATION, "query2", query2); \
  CALLBACK_OP(CB, NAME, OPERATION, "query3", query3); \
  CALLBACK_OP(CB, NAME, OPERATION, "loop", loop); \
  CALLBACK_OP(CB, NAME, OPERATION, "check-link", check_link); \
  CALLBACK_OP(CB, NAME, OPERATION, "check-mime", check_mime); \
  CALLBACK_OP(CB, NAME, OPERATION, "pause", pause); \
  CALLBACK_OP(CB, NAME, OPERATION, "save-file", filesave); \
  CALLBACK_OP(CB, NAME, OPERATION, "save-file2", filesave2); \
  CALLBACK_OP(CB, NAME, OPERATION, "link-detected", linkdetected); \
  CALLBACK_OP(CB, NAME, OPERATION, "link-detected2", linkdetected2); \
  CALLBACK_OP(CB, NAME, OPERATION, "transfer-status", xfrstatus); \
  CALLBACK_OP(CB, NAME, OPERATION, "save-name", savename); \
  CALLBACK_OP(CB, NAME, OPERATION, "send-header", sendhead); \
  CALLBACK_OP(CB, NAME, OPERATION, "receive-header", receivehead); \
} while(0)

int hts_set_callback(t_hts_htmlcheck_callbacks * callbacks, const char *name,
                     void *function) {
  int error = 1;
#define CALLBACK_OPERATION(TYPE, FUNCTION) do { \
    FUNCTION = (TYPE) function;                 \
    error = 0;                                  \
  } while(0)
  DISPATCH_CALLBACK(callbacks, name, CALLBACK_OPERATION);
#undef CALLBACK_OPERATION
  return error;
}

void *hts_get_callback(t_hts_htmlcheck_callbacks * callbacks, const char *name) {
#define CALLBACK_OPERATION(TYPE, FUNCTION) do { \
    return (void*) FUNCTION;                    \
  } while(0)
  DISPATCH_CALLBACK(callbacks, name, CALLBACK_OPERATION);
#undef CALLBACK_OPERATION
  return NULL;
}

// end defaut wrappers

/* libc stubs */

HTSEXT_API char *hts_strdup(const char *str) {
  return strdup(str);
}

HTSEXT_API void *hts_malloc(size_t size) {
  return malloc(size);
}

HTSEXT_API void *hts_realloc(void *const data, const size_t size) {
  return realloc(data, size);
}

HTSEXT_API void hts_free(void *data) {
  free(data);
}

/* Dummy functions */
HTSEXT_API int hts_resetvar(void) {
  return 0;
}

#ifdef _WIN32

typedef struct dirent dirent;

DIR *opendir(const char *name) {
  WIN32_FILE_ATTRIBUTE_DATA st;
  DIR *dir;
  size_t len;
  int i;
  LPWSTR wname;

  if (name == NULL || *name == '\0') {
    errno = ENOENT;
    return NULL;
  }
  // Wide \\?\ path: no MAX_PATH cap, no CP_ACP mis-decode (#133,#630).
  wname = hts_pathToUCS2(name);
  if (wname == NULL ||
      !GetFileAttributesExW(wname, GetFileExInfoStandard, &st) ||
      (st.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) {
    freet(wname);
    errno = ENOENT;
    return NULL;
  }
  freet(wname);
  dir = calloc(sizeof(DIR), 1);
  if (dir == NULL) {
    errno = ENOMEM;
    return NULL;
  }
  len = strlen(name);
  dir->h = INVALID_HANDLE_VALUE;
  dir->name = malloc(len + 2 + 1);
  strcpy(dir->name, name);
  for(i = 0; dir->name[i] != '\0'; i++) {
    if (dir->name[i] == '/') {
      dir->name[i] = '\\';
    }
  }
  strcat(dir->name, "\\*");
  return dir;
}

struct dirent *readdir(DIR * dir) {
  WIN32_FIND_DATAW find;

  if (dir->h == INVALID_HANDLE_VALUE) {
    // \\?\-prefix so a long/non-ASCII directory enumerates instead of ENOENT.
    LPWSTR wname = hts_pathToUCS2(dir->name);

    dir->h =
        wname != NULL ? FindFirstFileW(wname, &find) : INVALID_HANDLE_VALUE;
    freet(wname);
  } else {
    if (!FindNextFileW(dir->h, &find)) {
      FindClose(dir->h);
      dir->h = INVALID_HANDLE_VALUE;
    }
  }
  if (dir->h != INVALID_HANDLE_VALUE) {
    char *u = hts_convertUCS2StringToUTF8(find.cFileName, -1);

    dir->entry.d_name[0] = 0;
    if (u != NULL) {
      strncat(dir->entry.d_name, u, HTS_DIRENT_SIZE - 1);
      freet(u);
    }
    return &dir->entry;
  }
  errno = ENOENT;
  return NULL;
}

int closedir(DIR * dir) {
  if (dir != NULL) {
    if (dir->h != INVALID_HANDLE_VALUE) {
      CloseHandle(dir->h);
    }
    if (dir->name != NULL) {
      free(dir->name);
    }
    free(dir);
    return 0;
  }
  errno = EBADF;
  return -1;
}

// UTF-8 aware FILE API

static void copyWchar(LPWSTR dest, const char *src) {
  int i;

  for(i = 0; src[i]; i++) {
    dest[i] = src[i];
  }
  dest[i] = '\0';
}

/* UTF-8 path -> UCS-2 for the _w* file APIs. At/above HTS_WIN_LONGPATH_MIN,
   \\?\-prefix it via GetFullPathNameW to clear MAX_PATH (#133); else unchanged.
   Any prefixing failure falls back to the plain converted path. */
#define HTS_WIN_LONGPATH_MIN 240 /* stay clear of MAX_PATH (260) */

LPWSTR hts_pathToUCS2(const char *path) {
  LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);

  if (wpath == NULL) {
    return NULL;
  }
  const size_t len = wcslen(wpath);
  // Already "\\?\" or "\\.\": don't re-prefix.
  const int verbatim = len >= 4 && wpath[0] == L'\\' && wpath[1] == L'\\' &&
                       (wpath[2] == L'?' || wpath[2] == L'.') &&
                       wpath[3] == L'\\';

  if (len < HTS_WIN_LONGPATH_MIN || verbatim) {
    return wpath;
  }

  const DWORD need = GetFullPathNameW(wpath, 0, NULL, NULL); /* incl NUL */
  LPWSTR full = need != 0 ? malloct((size_t) need * sizeof(WCHAR)) : NULL;

  if (full == NULL) {
    return wpath; /* fall back to the plain path */
  }
  const DWORD written = GetFullPathNameW(wpath, need, full, NULL);

  if (written == 0 || written >= need || full[0] == L'\0') {
    freet(full);
    return wpath;
  }

  const int isUNC = full[0] == L'\\' && full[1] == L'\\';
  // UNC "\\srv\share" -> "\\?\UNC\srv\share": the prefix subsumes the "\\".
  const WCHAR *const pfx = isUNC ? L"\\\\?\\UNC\\" : L"\\\\?\\";
  const WCHAR *const body = isUNC ? full + 2 : full;
  const size_t pfxLen = wcslen(pfx), bodyLen = wcslen(body);
  LPWSTR out = malloct((pfxLen + bodyLen + 1) * sizeof(WCHAR));

  if (out == NULL) {
    freet(full);
    return wpath;
  }
  memcpybuff(out, pfx, pfxLen * sizeof(WCHAR));
  memcpybuff(out + pfxLen, body, (bodyLen + 1) * sizeof(WCHAR));
  freet(full);
  freet(wpath);
  return out;
}

FILE *hts_fopen_utf8(const char *path, const char *mode) {
  WCHAR wmode[32];
  LPWSTR wpath = hts_pathToUCS2(path);

  assertf(strlen(mode) < sizeof(wmode) / sizeof(WCHAR));
  copyWchar(wmode, mode);
  if (wpath != NULL) {
    FILE *const fp = _wfopen(wpath, wmode);

    free(wpath);
    return fp;
  } else {
    // Fallback on conversion error.
    return fopen(path, mode);
  }
}

int hts_stat_utf8(const char *path, STRUCT_STAT * buf) {
  LPWSTR wpath = hts_pathToUCS2(path);

  if (wpath != NULL) {
    const int result = _wstat64(wpath, buf);

    free(wpath);
    return result;
  } else {
    // Fallback on conversion error.
    return _stat64(path, buf);
  }
}

int hts_unlink_utf8(const char *path) {
  LPWSTR wpath = hts_pathToUCS2(path);

  if (wpath != NULL) {
    const int result = _wunlink(wpath);

    free(wpath);
    return result;
  } else {
    // Fallback on conversion error.
    return _unlink(path);
  }
}

int hts_rename_utf8(const char *oldpath, const char *newpath) {
  LPWSTR woldpath = hts_pathToUCS2(oldpath);
  LPWSTR wnewpath = hts_pathToUCS2(newpath);
  if (woldpath != NULL && wnewpath != NULL) {
    const int result = _wrename(woldpath, wnewpath);
    /* Save errno: callers key off it (#779) and free() may clobber it. */
    const int err = errno;

    free(woldpath);
    free(wnewpath);
    errno = err;
    return result;
  } else {
    if (woldpath != NULL)
      free(woldpath);
    if (wnewpath != NULL)
      free(wnewpath);
    // Fallback on conversion error.
    return rename(oldpath, newpath);
  }
}

int hts_rmdir_utf8(const char *path) {
  LPWSTR wpath = hts_pathToUCS2(path);

  if (wpath != NULL) {
    const int result = _wrmdir(wpath);

    free(wpath);
    return result;
  } else {
    // Fallback on conversion error.
    return _rmdir(path);
  }
}

int hts_mkdir_utf8(const char *path) {
  LPWSTR wpath = hts_pathToUCS2(path);

  if (wpath != NULL) {
    const int result = _wmkdir(wpath);

    free(wpath);
    return result;
  } else {
    // Fallback on conversion error.
    return _mkdir(path);
  }
}

HTSEXT_API int hts_utime_utf8(const char *path, const STRUCT_UTIMBUF * times) {
  STRUCT_UTIMBUF mtimes = *times;
  LPWSTR wpath = hts_pathToUCS2(path);

  if (wpath != NULL) {
    const int result = _wutime(wpath, &mtimes);

    free(wpath);
    return result;
  } else {
    // Fallback on conversion error.
    return _utime(path, &mtimes);
  }
}

#endif

// Fin

Coverage Report

Created: 2026-09-01 06:54