/src/httrack/src/htscore.h

Source
/* ------------------------------------------------------------ */
/*
HTTrack Website Copier, Offline Browser for Windows and Unix
Copyright (C) 1998 Xavier Roche and other contributors

SPDX-License-Identifier: GPL-3.0-or-later

This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.

This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
GNU General Public License for more details.

You should have received a copy of the GNU General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.

Ethical use: we kindly ask that you NOT use this software to harvest email
addresses or to collect any other private information about people. Doing so
would dishonor our work and waste the many hours we have spent on it.

Please visit our Website: http://www.httrack.com
*/

/* ------------------------------------------------------------ */
/* File: Main file .h                                           */
/* Author: Xavier Roche                                         */
/* ------------------------------------------------------------ */

/* Core engine declarations. Not an installed header, but part of the de-facto
   API surface: external consumers (e.g. httrack-android) read these structs and
   constants and call functions declared here. */
#ifndef HTS_CORE_DEFH
#define HTS_CORE_DEFH

#include "htsglobal.h"

/* specific definitions */
#include "htsbase.h"
/* Includes and definitions */
#include <limits.h>
#include <sys/types.h>
#include <sys/stat.h>
#ifdef _WIN32
#include <conio.h>
#include <direct.h>
#else
#ifndef _WIN32
#include <unistd.h>
#endif
#endif
/* END specific definitions */

/* Forward definitions */
#ifndef HTS_DEF_FWSTRUCT_lien_url
#define HTS_DEF_FWSTRUCT_lien_url
typedef struct lien_url lien_url;
#endif
#ifndef HTS_DEF_FWSTRUCT_lien_back
#define HTS_DEF_FWSTRUCT_lien_back
typedef struct lien_back lien_back;
#endif
#ifndef HTS_DEF_FWSTRUCT_struct_back
#define HTS_DEF_FWSTRUCT_struct_back
typedef struct struct_back struct_back;
#endif
#ifndef HTS_DEF_FWSTRUCT_cache_back
#define HTS_DEF_FWSTRUCT_cache_back
typedef struct cache_back cache_back;
#endif
#ifndef HTS_DEF_FWSTRUCT_hash_struct
#define HTS_DEF_FWSTRUCT_hash_struct
typedef struct hash_struct hash_struct;
#endif
#ifndef HTS_DEF_FWSTRUCT_filecreate_params
#define HTS_DEF_FWSTRUCT_filecreate_params
typedef struct filecreate_params filecreate_params;
#endif

// Include htslib.h for all types
#include "htslib.h"

// options
#include "htsopt.h"

// HTTrack engine sub-headers

// main entry point
#include "htscoremain.h"

// core routines
#include "htscore.h"

// misc tools for httrack.c
#include "htstools.h"

// command-line help
#include "htshelp.h"

// build the on-disk save filename
#include "htsname.h"

// FTP support
#include "htsftp.h"

// URL interception
#include "htscatchurl.h"

// robots.txt handling
#include "htsrobots.h"

// link-acceptance rules
#include "htswizard.h"

// regexp/filter routines
#include "htsfilters.h"

// download backing (the back[] slot ring)
#include "htsback.h"

// cache handling
#include "htscache.h"

// hashing
#include "htshash.h"
#include "coucal.h"

#include "htsdefines.h"

#include "hts-indextmpl.h"

/** A remote URL split into host and path, each a fixed inline buffer
    (HTS_URLMAXSIZE*2 bytes, NUL-terminated). */
#ifndef HTS_DEF_FWSTRUCT_lien_adrfil
#define HTS_DEF_FWSTRUCT_lien_adrfil
typedef struct lien_adrfil lien_adrfil;
#endif
struct lien_adrfil {
  char adr[HTS_URLMAXSIZE * 2]; /**< host (address) */
  char fil[HTS_URLMAXSIZE * 2]; /**< remote file path */
};

/** A remote URL plus the local on-disk path it is saved to. */
#ifndef HTS_DEF_FWSTRUCT_lien_adrfilsave
#define HTS_DEF_FWSTRUCT_lien_adrfilsave
typedef struct lien_adrfilsave lien_adrfilsave;
#endif
struct lien_adrfilsave {
  lien_adrfil af;
  char save[HTS_URLMAXSIZE * 2]; /**< local save path (with directory) */
};

/** Per-slot connect-fallback bookkeeping (parallel to struct_back.lnk).
    Tracks which resolved address the slot is currently connecting to so a
    stuck connect can be retried against the next one. */
typedef struct hts_connect_fallback {
  int addr_index;       /**< candidate being connected (0-based) */
  int addr_count;       /**< resolved addresses; -1 = not yet probed */
} hts_connect_fallback;

/** The download-slot ring: the set of concurrent transfers in flight.
    Allocated/owned by the engine; consumers (status callbacks, the loop)
    read it but do not resize or free it. */
#ifndef HTS_DEF_FWSTRUCT_struct_back
#define HTS_DEF_FWSTRUCT_struct_back
typedef struct struct_back struct_back;
#endif
struct struct_back {
  lien_back *lnk;         /**< slot array, valid indices [0..count-1]
                               (count+1 entries allocated); a slot is
                               active iff lnk[i].status != STATUS_FREE.
                               See struct lien_back in htsopt.h and the
                               STATUS_* codes in htsbasenet.h. */
  int count;              /**< number of usable slots (back_max) */
  coucal ready;           /**< index of slots whose transfer completed */
  LLint ready_size_bytes; /**< total bytes buffered in completed slots */
  hts_connect_fallback *connect_fallback; /**< per-slot, count+1 entries */
  hts_fetch_meta *fetch; /**< per-slot Sec-Fetch-* values, count+1 entries */
  hts_boolean bot_challenge_noted; /**< the bot-protection note is already in
                                        this mirror's log (#1732) */
  TStamp retry_after_until; /**< Retry-After: no new launch before this local
                                 ms timestamp; 0 holds nothing */
};

/** Open handle to the mirror cache (the read-from-old / write-to-new state
    used to resume and to avoid re-fetching unchanged files). Engine-owned. */
#ifndef HTS_DEF_FWSTRUCT_cache_back
#define HTS_DEF_FWSTRUCT_cache_back
typedef struct cache_back cache_back;
#endif
struct cache_back {
  int version; /**< cache-file format version being read */
  /* */
  int type;
  int ro;                   /**< read-only: no new cache is written */

  FILE *lst;                /**< file list, used for purge */
  FILE *txt;                /**< human-readable file list (info) */
  char lastmodified[256];
  // HASH
  coucal hashtable;
  // HASH for tests (naming subsystem)
  coucal cached_tests;
  /** URLs whose previous entry this run already carried forward (#1421), so a
      retry of the same link does not store it again. */
  coucal kept;
  /* optional log files */
  FILE *log;
  FILE *errlog;
  /* read-ahead cursors into the old cache */
  int ptr_ant;
  int ptr_last;
  /* ZIP-backed cache backend (newer format) */
  void *zipInput;
  void *zipOutput;
  hts_boolean
      zipWriteFailed; /**< a cache write failed; stop touching the stream */
  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
};

#ifndef HTS_DEF_FWSTRUCT_hash_struct
#define HTS_DEF_FWSTRUCT_hash_struct
typedef struct hash_struct hash_struct;
#endif
/** Lookup indexes over the link heap: map save-name / URL back to a link, so a
    URL seen twice resolves to one entry. The coucal tables index into liens;
    they do not own the links. */
struct hash_struct {
  /* points at the engine's link array (opt->liens); not owned */
  const lien_url *const*const*liens;
  /* save-name -> link index (case-insensitive: keys lowercased) */
  coucal sav;
  /* address+path -> link index */
  coucal adrfil;
  /* former address+path -> link index (renamed/moved entries) */
  coucal former_adrfil;
  /* effective urlhack sub-flags: www.==host / // collapse / query-arg sort */
  hts_boolean norm_host;
  hts_boolean norm_slash;
  hts_boolean norm_query;
  /* query-strip keys (not owned); set from opt->strip_query at hash_init */
  const char *strip_query;
  /* host-alias rules (not owned); set from opt->host_alias at hash_init */
  const char *host_alias;
  /* scratch: the hash key is a host plus a path, the compare one a path */
  char normfil[HTS_URLMAXSIZE * 4];
  char normfil2[HTS_URLMAXSIZE * 2];
  char catbuff[CATBUFF_SIZE];
};

#ifndef HTS_DEF_FWSTRUCT_filecreate_params
#define HTS_DEF_FWSTRUCT_filecreate_params
typedef struct filecreate_params filecreate_params;
#endif
/** Parameters threaded through file-creation callbacks (filenote). */
struct filecreate_params {
  FILE *lst; /**< open file list to append created paths to */
  char path[HTS_URLMAXSIZE * 2];
};

/* Convenience accessors over the link heap; assume `opt` (and where used,
   `ptr`/`parent_relative`/`module_base`) are in scope. heap(N) is the Nth link;
   heap_top_index() is the last recorded link's index. */
#define heap(N)            (opt->liens[N])

#define heap_top_index()   (opt->lien_tot - 1)

#define heap_top()         (heap(heap_top_index()))

#define urladr()           (heap(ptr)->adr)

#define urlfil()           (heap(ptr)->fil)

#define savename()         (heap(ptr)->sav)

#define parenturladr()     (heap(heap(ptr)->precedent)->adr)

#define parenturlfil()     (heap(heap(ptr)->precedent)->fil)

#define parentsavename()   (heap(heap(ptr)->precedent)->sav)

/* A module specifier takes the script's own base even in a .js document. */
#define relative_to_parent() (parent_relative && !module_base)

#define relativeurladr() ((!relative_to_parent()) ? urladr() : parenturladr())

#define relativeurlfil() ((!relative_to_parent()) ? urlfil() : parenturlfil())

#define relativesavename()                                                     \
  ((!relative_to_parent()) ? savename() : parentsavename())

/* Library-internal helpers (engine-only, HTS_INTERNAL_BYTECODE). */
#ifdef HTS_INTERNAL_BYTECODE

/* True if a new cache is being written (plain or zip backend). */
HTS_STATIC int cache_writable(cache_back * cache) {
  return (cache != NULL && cache->zipOutput != NULL);
}

/* True if an old cache is available to read (plain or zip backend). */
HTS_STATIC int cache_readable(cache_back * cache) {
  return (cache != NULL && cache->zipInput != NULL);
}

#endif

// Functions

/* Library-internal only (engine TUs). */
#ifdef HTS_INTERNAL_BYTECODE

char *hts_cancel_file_pop(httrackp * opt);

/* Detach the pending hts_addurl() list and give the caller ownership; NULL
   when none is pending. Free it with hts_addurl_free(). */
char **hts_addurl_take(httrackp *opt);

/* Free a list taken with hts_addurl_take(), strings included. */
void hts_addurl_free(char **url);

/* Detach the queued rules. Returns NULL if none. The caller frees the list
   with hts_addurl_free(). */
char **hts_addfilter_take(httrackp *opt);

/* Takes the list hts_setfilters() queued, like hts_addfilter_take(). */
char **hts_setfilters_take(httrackp *opt);

#endif

/* Has the -#L cap left fewer than two free slots? One parser step records
   robots.txt and then the link itself, so both must still fit. */
hts_boolean hts_maxlinks_no_room(const httrackp *opt);

/* Record a link on the heap. All strings are copied (caller keeps ownership).
   Returns 1 on success, 0 if the link limit (opt->maxlink) is reached. */
int hts_record_link(httrackp * opt,
                    const char *address, const char *file, const char *save,
                    const char *ref_address, const char *ref_file,
                    const char *codebase);

/* Index of the most recently recorded link. */
size_t hts_record_link_latest(httrackp *opt);

/* Mark link at index lpos as not to be processed (sets pass2 = -1). */
void hts_invalidate_link(httrackp * opt, int lpos);

/* Allocates the link heap if absent, and returns HTS_FALSE without memory. */
hts_boolean hts_record_init(httrackp *opt);

void hts_record_free(httrackp *opt);

/* Load the -%L file at path into *buff, NUL-terminated after its *size bytes,
   to be freed with freet(). Returns NULL, or why it could not be read, and then
   *buff is NULL. The reason may be written in errbuf, of HTS_STRERROR_SIZE
   bytes. */
const char *hts_urllist_load(const char *path, char **buff, size_t *size,
                             char *errbuf);

/* The size of the line buffer hts_urllist_next() fills. */
#define HTS_URLLIST_LINE_SIZE (HTS_URLMAXSIZE * 2)

typedef enum hts_urllist_status {
  HTS_URLLIST_END,  /* no line is left */
  HTS_URLLIST_LINE, /* line holds the next line */
  HTS_URLLIST_LONG, /* the line is longer than HTS_URLMAXSIZE bytes */
  HTS_URLLIST_NUL   /* the line holds a NUL byte */
} hts_urllist_status;

/* Read the next -%L line of buff at *pos, and move *pos past it. Returns how
   the line ended. line holds the head of a LONG or NUL line, which the crawl
   drops. */
hts_urllist_status hts_urllist_next(const char *buff, size_t size, size_t *pos,
                                    char line[HTS_URLLIST_LINE_SIZE]);

/* Run the mirror for the given start URL(s) under opt. Top-level engine entry.
   Returns 1 once the crawl loop ran, 0 or -1 for a bailout. *completed says
   whether it reached the end: false for a bailout, and for any stop that cut it
   short, however it arrived (state.stop or exit_xh). */
int httpmirror(char *url1, httrackp *opt, hts_boolean *completed);

/* Start the stats clock HTS_STAT.stat_timestart (seconds) and the --max-time
   clock hts_mirror_elapsed_ms(). */
void hts_mirror_clock_start(void);

/* Return the time since hts_mirror_clock_start(), in monotonic ms. */
TStamp hts_mirror_elapsed_ms(void);

/* Write len bytes of adr to local path s. url_adr/url_fil (may be NULL) name
   the source URL for logging/notification. */
int filesave(httrackp * opt, const char *adr, int len, const char *s,
             const char *url_adr /* = NULL */ ,
             const char *url_fil /* = NULL */ );

char *hts_cancel_file_pop(httrackp * opt);

int check_fatal_io_errno(void);

int engine_stats(void);

void host_ban(httrackp * opt, int ptr, struct_back * sback, const char *host);

/* Open local file s for writing (filecreate, truncate) or appending
   (fileappend), creating parent directories as needed. Return an open FILE*
   the caller must fclose(), or NULL on failure. */
FILE *filecreate(filenote_strc * strct, const char *s);

FILE *fileappend(filenote_strc * strct, const char *s);

/* Create an empty file, return 1 on success, 0 on failure. */
int filecreateempty(filenote_strc * strct, const char *filename);

int filenote(filenote_strc * strct, const char *s, filecreate_params * params);

/* Copy into dest (destsize bytes) the form under which the local path s is
   listed in new.lst: forward slashes, with the mirror `root` stripped when s
   sits under it. Also the change report's key, so the two must not drift. */
void hts_savename_listed(const char *root, const char *s, char *dest,
                         size_t destsize);

void file_notify(httrackp * opt, const char *adr, const char *fil,
                 const char *save, int create, int modify, int wasupdated);

void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
                 const char *adr, const char *fil);

/* usercommand_expand() verdicts; only OK may be run. */
#define USERCOMMAND_EXPAND_OK 1
#define USERCOMMAND_EXPAND_TOOLONG 0
#define USERCOMMAND_EXPAND_REFUSED (-1)
/* Rewrite a -V template's $0. On POSIX "file" is ignored: usercommand_exe()
   passes it to the shell as $1, so the rewrite does not depend on it. */
int usercommand_expand(char *dest, size_t size, const char *cmd,
                       const char *file, const char **why);
/* Run a -V template against a save name; returns the shell's exit status, or
   -1 when the command was not run at all. */
int usercommand_exe(httrackp *opt, const char *cmd, const char *file);
#ifndef _WIN32
/* Room for "sh", "-c", the command, the shell's $0, its $1 and the NULL. */
#define USERCOMMAND_ARGV_MAX 6
size_t usercommand_argv(char *argv[USERCOMMAND_ARGV_MAX], const char *command,
                        const char *file);
#endif

// Finish the makeindex index.html (footer + refresh meta), run usercommand.
// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
                          FILE **makeindex_fp, int makeindex_links,
                          const char *makeindex_firstlink,
                          const char *template_footer, const char *adr,
                          const char *fil);

/* Re-encode 'size' bytes of UCS2 (UTF-16, BMP only) as UTF-8. 'swap' selects
   little-endian input, 'size' must be even. Returns a NUL-terminated buffer to
   release with freet(), its length (NUL excluded) in *outsize, or NULL when the
   result does not fit memory. */
char *hts_ucs2_to_utf8(const unsigned char *src, size_t size, hts_boolean swap,
                       size_t *outsize);

// Flush ht_buff[0..ht_len] to save on disk; *fp closed+NULLed on write.
// Precondition: ht_len>0.
void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                          FILE **fp, const char *ht_buff, size_t ht_len,
                          const char *adr, const char *fil, const char *save);

/* Longest suffix a rule builder appends after the path (the wizard's
   "*[file]"), NUL included; asserted against in htswizard.c. */
#define HTS_FILTER_SUFFIX_MAX 8

/* Per-slot capacity of the filters array; filters_init() strides the slots by
   it. Holds the longest rule any builder emits, because truncating one changes
   what it matches. */
#define HTS_FILTER_SLOT_SIZE                                                   \
  (1 + (HTS_URLMAXSIZE * 2 - 1) + 1 + (HTS_URLMAXSIZE * 2 - 1) +               \
   HTS_FILTER_SUFFIX_MAX)

/* Largest -#F, and the largest filter list that may still grow by
   HTS_FILTERSINC, because the list's byte size must fit an int. */
#define HTS_FILTERS_MAX (INT_MAX / HTS_FILTER_SLOT_SIZE - HTS_FILTERSINC - 2)

int filters_init(char ***ptrfilters, int maxfilter, int filterinc);

/* Binds this crawl's filter array to `opt`, and empties the wizard block with
   it: opt->wizard_filters indexes into that array and outlives no crawl. */
void filters_bind(httrackp *opt, char ***ptrfilters, int *filptr);

/* Longest rule a slot holds and the matcher still looks at. */
#define HTS_FILTER_MAXLEN                                                      \
  (HTS_FILTER_SLOT_SIZE - 1 < STRJOKER_MAXLEN ? HTS_FILTER_SLOT_SIZE - 1       \
                                              : STRJOKER_MAXLEN)

/* Stores `pattern` at `pos`, shifting the rules from there up a slot. The one
   door into the array: a rule past HTS_FILTER_MAXLEN, or one that finds the
   array full, warns and returns HTS_FALSE, never stored dead (#1270). */
hts_boolean filters_insert(httrackp *opt, int pos, const char *pattern);

/* Remove the n rules starting at pos, shifting the rest down. */
void filters_remove(httrackp *opt, int pos, int n);

/* Can a filter array of maxfilter rules grow by HTS_FILTERSINC? */
hts_boolean filters_may_grow(int maxfilter);

/* Makes room for n more rules. Returns HTS_FALSE, having logged it, only when
   memory runs out. At the HTS_FILTERS_MAX cap it returns HTS_TRUE and
   filters_insert() drops what does not fit. */
hts_boolean filters_grow(httrackp *opt, int n);

/* Same as filters_grow(), but aborts when memory runs out. */
void filters_make_room(httrackp *opt, int n);

/* Returns the fa_strjoker_dual() verdict of the nfil filters on adr+fil, in
   both forms the wizard tests, and sets *depth when depth is not NULL. */
int filters_match_url(char **filters, int nfil, const char *adr,
                      const char *fil, int *depth);

/* Apply the queued list and rules to the user's rules, then drop the queued
   links after ptr that the change makes them refuse. Crawl thread only, as it
   alone reads the filters. */
void hts_apply_live_filters(httrackp *opt, struct_back *sback, int ptr);

int fspc(httrackp * opt, FILE * fp, const char *type);
/* Count one log event into opt's counters that hts_get_stats() publishes,
   whether or not a log file is open. "panic" counts as an error. */
void fspc_count(httrackp *opt, const char *type);

/* Like fil_normalized(), but first drops query keys in STRIP (comma-separated,
   "*" = all); STRIP NULL/empty behaves exactly like fil_normalized(). */
char *fil_normalized_filtered(const char *source, char *dest,
                              const char *strip);

/* As fil_normalized_filtered(), but DO_SLASH/DO_QUERY gate the // collapse and
   the query-argument sort independently (the urlhack sub-flags). */
char *fil_normalized_filtered_ex(const char *source, char *dest,
                                 const char *strip, int do_slash, int do_query);

/* For URL ADR/FIL, return (in DEST) the comma keylist to strip from the
   '\n'-separated "[pattern=]keys" RULES (patterns matched on host/path via
   strjoker, last wins); NULL if none match. Feeds fil_normalized_filtered(). */
const char *hts_query_strip_keys(const char *rules, const char *adr,
                                 const char *fil, char *dest, size_t destsize);

/* Is the key ARG[0..KEYLEN) in the comma list STRIP? Tokens are space-trimmed
   and case-sensitive, and "*" matches any key. */
hts_boolean hts_query_key_stripped(const char *arg, size_t keylen,
                                   const char *strip);

/* Return the canonical host (length *CANONLEN) of the last RULES entry whose
   aliases match HOST or FULL ("scheme://host"), or NULL. COLLAPSE_WWW ignores
   a "www." prefix on both sides. */
const char *hts_host_alias_match(const char *rules, const char *host,
                                 const char *full, hts_boolean collapse_www,
                                 size_t *canonlen);

/* The --host-alias rules of OPT, or NULL when the option was not given. */
const char *hts_host_alias_rules(httrackp *opt);

/* ADR with its host replaced by the canonical one, from the '\n'-separated
   "alias[,alias...]=canonical" RULES (aliases matched with strjoker, last
   wins). NOWWW matches the host with its www. prefix collapsed, as -%u keys it.
   Chains are followed to their end, so the result is a fixpoint. Scheme and
   credentials are kept. NULL when no rule applies, the mapping is a no-op, the
   rules loop, or the result would not fit DEST. */
const char *hts_host_alias(const char *rules, const char *adr,
                           hts_boolean collapse_www, char *dest,
                           size_t destsize);

/* Resolve like hts_host_alias, and set *SETTLED, when SETTLED is not NULL, to
   whether the chain ended within the hop limit. Exposed for the -#test=spandiff
   test. */
const char *hts_host_alias_resolve(const char *rules, const char *adr,
                                   hts_boolean collapse_www, char *dest,
                                   size_t destsize, hts_boolean *settled);

/* HTS_TRUE when OPT's url hacks collapse www.host onto host. */
hts_boolean hts_host_alias_collapse_www(httrackp *opt);

/* Rewrite AF's host to its --host-alias canonical form, in place, and return
   it. Idempotent; returns af->adr unchanged when no rule applies. */
const char *hts_host_alias_fold(httrackp *opt, lien_adrfil *af);

/* HTS_TRUE if ADRA and ADRB name the same host once RULES are applied. */
hts_boolean hts_host_same_alias(const char *rules, const char *adra,
                                const char *adrb, hts_boolean collapse_www);

/* First canonical host in RULES whose chain never ends ("a=b" plus "b=a"),
   copied into DEST, or NULL if every chain settles. Those hosts are left
   unaliased, so the engine reports them rather than picking a hop. */
const char *hts_host_alias_looping(const char *rules, hts_boolean collapse_www,
                                   char *dest, size_t destsize);

/* Read a whole file into a freshly malloc'd, NUL-terminated buffer; the caller
   owns it and must release it with freet(). Return NULL on missing/unreadable
   file (readfile_or substitutes defaultdata instead). The byte content is NOT
   transcoded except readfile_utf8, which expects a UTF-8 path. readfile2
   reports the byte size (excluding the NUL) via *size when non-NULL. */
char *readfile(const char *fil);

char *readfile2(const char *fil, LLint * size);

/* readfile2() refusing a body too large for the int indexes downstream, on the
   same stat that sizes the read: a caller checking the size itself would leave
   a window for the file to grow. NULL when it does not fit. "fil" is UTF-8,
   as a mirror path is. */
char *readfile2_inmem(const char *fil, LLint *size);

char *readfile_utf8(const char *fil);

char *readfile2_utf8(const char *fil, LLint *size);

char *readfile_or(const char *fil, const char *defaultdata);

/* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
   Not thread-safe; call from the single crawl loop. */

/* True if a connecting slot should give up on the current address and try the
   next one: a fallback address remains (addr_index+1 < addr_count) and the
   candidate has been connecting for at least its deadline, min(timeout, an
   internal cap). elapsed/timeout in seconds. Exposed for the -#D self-test. */
int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
                              int timeout);

/* How many new sockets may be opened now, honoring maxsoc and the maxconn rate
   limit (>=0). _strict ignores reserved-slot headroom; the plain form leaves
   room for naming tests and stops at 0 when the stack is nearly full. */
int back_pluggable_sockets(struct_back * sback, httrackp * opt);

int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);

/* One engine-loop tick: refresh the transfer stats and run the loop callback
   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);

/* Wait until a test socket can be plugged, pumping transfers, stats and the
   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
                                      cache_back *cache, int ptr);

/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
   timestamp seed so it is stable within one gap and rerolls per launch. */
int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);

/* Withhold every new launch for `seconds`, clipped to opt->max_retry_after.
   A hold already running further out is kept. */
void back_set_retry_after(struct_back *sback, httrackp *opt, int seconds);

/* Schedule more links from the heap into free slots. Returns the number queued,
   or <=0 if none could be added (no free slot / paused / stopped). */
int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
              int ptr, int numero_passe);

/* Count of links already finished (in background or served from cache). */
int backlinks_done(const struct_back *sback, lien_url **liens, int lien_tot,
                   int ptr);

/* Like back_fill, but a no-op (returns -1) when in-memory buffered data already
   exceeds opt->maxcache. */
int back_fillmax(struct_back * sback, httrackp * opt, cache_back * cache,
                 int ptr, int numero_passe);

/* Interactive prompt: continue an interrupted mirror? Returns nonzero to go on.
 */
int ask_continue(httrackp * opt);

/* Number of decimal digits in n. */
int nombre_digit(int n);

// Polling
#if HTS_POLL
int check_flot(T_SOC s);

int check_stdin(void);
#endif
/* Nonzero if the socket has a pending error. */
int check_sockerror(T_SOC s);

/* external modules: register a link discovered by a parser plugin. */
int htsAddLink(htsmoduleStruct * str, char *link);

/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
#define HTS_HTMLESCAPE_MAXEXP 5      /* escape_for_html_print: '&'->"&amp;" */
#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */

#endif

Coverage Report

Created: 2026-10-28 06:49