/*

 * Process debugging functions.

 *

 * Copyright 2000-2019 Willy Tarreau <willy@haproxy.org>.

 *

 * This program is free software; you can redistribute it and/or

 * modify it under the terms of the GNU General Public License

 * as published by the Free Software Foundation; either version

 * 2 of the License, or (at your option) any later version.

 *

 */

#include <errno.h>

#include <fcntl.h>

#include <signal.h>

#include <time.h>

#include <stdio.h>

#include <stdlib.h>

#include <syslog.h>

#include <sys/resource.h>

#include <sys/stat.h>

#include <sys/types.h>

#include <sys/utsname.h>

#include <sys/wait.h>

#include <unistd.h>

#ifdef USE_EPOLL

#include <sys/epoll.h>

#endif

#include <haproxy/api.h>

#include <haproxy/applet.h>

#include <haproxy/buf.h>

#include <haproxy/cfgparse.h>

#include <haproxy/cli.h>

#include <haproxy/clock.h>

#include <haproxy/debug.h>

#include <haproxy/fd.h>

#include <haproxy/global.h>

#include <haproxy/hlua.h>

#include <haproxy/http_ana.h>

#include <haproxy/limits.h>

#if defined(USE_LINUX_CAP)

#include <haproxy/linuxcap.h>

#endif

#include <haproxy/log.h>

#include <haproxy/net_helper.h>

#include <haproxy/sc_strm.h>

#include <haproxy/proxy.h>

#include <haproxy/stconn.h>

#include <haproxy/task.h>

#include <haproxy/thread.h>

#include <haproxy/time.h>

#include <haproxy/tools.h>

#include <haproxy/trace.h>

#include <haproxy/version.h>

#include <import/ist.h>

/* The dump state is made of:

 *   - num_thread on the lowest 15 bits

 *   - a SYNC flag on bit 15 (waiting for sync start)

 *   - number of participating threads on bits 16-30

 * Initiating a dump consists in setting it to SYNC and incrementing the

 * num_thread part when entering the function. The first thread periodically

 * recounts active threads and compares it to the ready ones, and clears SYNC

 * and sets the number of participants to the value found, which serves as a

 * start signal. A thread finished dumping looks up the TID of the next active

 * thread after it and writes it in the lowest part. If there's none, it sets

 * the thread counter to the number of participants and resets that part,

 * which serves as an end-of-dump signal. All threads decrement the num_thread

 * part. Then all threads wait for the value to reach zero. Only used when

 * USE_THREAD_DUMP is set.

 */

#define THREAD_DUMP_TMASK     0x00007FFFU

#define THREAD_DUMP_FSYNC     0x00008000U

#define THREAD_DUMP_PMASK     0x7FFF0000U

/* Description of a component with name, version, path, build options etc. E.g.

 * one of them is haproxy. Others might be some clearly identified shared libs.

 * They're intentionally self-contained and to be placed into an array to make

 * it easier to find them in a core. The important fields (name and version)

 * are locally allocated, other ones are dynamic.

 */

struct post_mortem_component {

  char name[32];           // symbolic short name

  char version[32];        // exact version

  char *toolchain;         // compiler and version (e.g. gcc-11.4.0)

  char *toolchain_opts;    // optims, arch-specific options (e.g. CFLAGS)

  char *build_settings;    // build options (e.g. USE_*, TARGET, etc)

  char *path;              // path if known.

};

/* This is a collection of information that are centralized to help with core

 * dump analysis. It must be used with a public variable and gather elements

 * as much as possible without dereferences so that even when identified in a

 * core dump it's possible to get the most out of it even if the core file is

 * not much exploitable. It's aligned to 256 so that it's easy to spot, given

 * that being that large it will not change its size much.

 */

struct post_mortem {

  /* platform-specific information */

  char post_mortem_magic[32];     // "POST-MORTEM STARTS HERE+7654321\0"

  struct {

    struct utsname utsname; // OS name+ver+arch+hostname

    char hw_vendor[64];     // hardware/hypervisor vendor when known

    char hw_family[64];     // hardware/hypervisor product family when known

    char hw_model[64];      // hardware/hypervisor product/model when known

    char brd_vendor[64];    // mainboard vendor when known

    char brd_model[64];     // mainboard model when known

    char soc_vendor[64];    // SoC/CPU vendor from cpuinfo

    char soc_model[64];     // SoC model when known and relevant

    char cpu_model[64];     // CPU model when different from SoC

    char virt_techno[16];   // when provided by cpuid

    char cont_techno[16];   // empty, "no", "yes", "docker" or others

  } platform;

  /* process-specific information */

  struct {

    pid_t pid;

    uid_t boot_uid;

    gid_t boot_gid;

    uid_t run_uid;

    gid_t run_gid;

#if defined(USE_LINUX_CAP)

    struct {

      // initial process capabilities

      struct __user_cap_data_struct boot[_LINUX_CAPABILITY_U32S_3];

      int err_boot; // errno, if capget() syscall fails at boot

      // runtime process capabilities

      struct __user_cap_data_struct run[_LINUX_CAPABILITY_U32S_3];

      int err_run; // errno, if capget() syscall fails at runtime

    } caps;

#endif

    struct rlimit boot_lim_fd;  // RLIMIT_NOFILE at startup

    struct rlimit boot_lim_ram; // RLIMIT_DATA at startup

    struct rlimit run_lim_fd;  // RLIMIT_NOFILE just before enter in polling loop

    struct rlimit run_lim_ram; // RLIMIT_DATA just before enter in polling loop

    char **argv;

    unsigned char argc;

  } process;

#if defined(HA_HAVE_DUMP_LIBS)

  /* information about dynamic shared libraries involved */

  char *libs;                      // dump of one addr / path per line, or NULL

#endif

  struct tgroup_info *tgroup_info; // pointer to ha_tgroup_info

  struct thread_info *thread_info; // pointer to ha_thread_info

  struct tgroup_ctx  *tgroup_ctx;  // pointer to ha_tgroup_ctx

  struct thread_ctx  *thread_ctx;  // pointer to ha_thread_ctx

  struct list *pools;              // pointer to the head of the pools list

  struct proxy **proxies;          // pointer to the head of the proxies list

  struct global *global;           // pointer to the struct global

  struct fdtab **fdtab;            // pointer to the fdtab array

  struct activity *activity;       // pointer to the activity[] per-thread array

  /* info about identified distinct components (executable, shared libs, etc).

   * These can be all listed at once in gdb using:

   *    p *post_mortem.components@post_mortem.nb_components

   */

  uint nb_components;              // # of components below

  struct post_mortem_component *components; // NULL or array

} post_mortem ALIGNED(256) HA_SECTION("_post_mortem") = { };

unsigned int debug_commands_issued = 0;

unsigned int warn_blocked_issued = 0;

unsigned int debug_enable_counters = (DEBUG_COUNTERS >= 2);

/* dumps a backtrace of the current thread that is appended to buffer <buf>.

 * Lines are prefixed with the string <prefix> which may be empty (used for

 * indenting). It is recommended to use this at a function's tail so that

 * the function does not appear in the call stack. The <dump> argument

 * indicates what dump state to start from, and should usually be zero. It

 * may be among the following values:

 *   - 0: search usual callers before step 1, or directly jump to 2

 *   - 1: skip usual callers before step 2

 *   - 2: dump until polling loop, scheduler, or main() (excluded)

 *   - 3: end

 *   - 4-7: like 0 but stops *after* main.

 */

void ha_dump_backtrace(struct buffer *buf, const char *prefix, int dump)

{

  sigset_t new_mask, old_mask;

  struct buffer bak;

  char pfx2[100];

  void *callers[100];

  int j, nptrs;

  const void *addr;

  /* make sure we don't re-enter from debug coming from other threads,

   * as some libc's backtrace() are not re-entrant. We'll block these

   * sensitive signals while possibly dumping a backtrace.

   */

  sigemptyset(&new_mask);

#ifdef WDTSIG

  sigaddset(&new_mask, WDTSIG);

#endif

#ifdef DEBUGSIG

  sigaddset(&new_mask, DEBUGSIG);

#endif

  ha_sigmask(SIG_BLOCK, &new_mask, &old_mask);

  nptrs = my_backtrace(callers, sizeof(callers)/sizeof(*callers));

  if (!nptrs)

    goto leave;

  if (snprintf(pfx2, sizeof(pfx2), "%s| ", prefix) > sizeof(pfx2))

    pfx2[0] = 0;

  /* The call backtrace_symbols_fd(callers, nptrs, STDOUT_FILENO would

   * produce similar output to the following:

   */

  chunk_appendf(buf, "%scall trace(%d):\n", prefix, nptrs);

  for (j = 0; (j < nptrs || (dump & 3) < 2); j++) {

    if (j == nptrs && !(dump & 3)) {

      /* we failed to spot the starting point of the

       * dump, let's start over dumping everything we

       * have.

       */

      dump += 2;

      j = 0;

    }

    bak = *buf;

    dump_addr_and_bytes(buf, pfx2, callers[j], -8);

    addr = resolve_sym_name(buf, ": ", callers[j]);

#if defined(__i386__) || defined(__x86_64__)

    /* Try to decode a relative call (0xe8 + 32-bit signed ofs) */

    if (may_access(callers[j] - 5) && may_access(callers[j] - 1) &&

        *((uchar*)(callers[j] - 5)) == 0xe8) {

      int ofs = *((int *)(callers[j] - 4));

      const void *addr2 = callers[j] + ofs;

      resolve_sym_name(buf, " > ", addr2);

    }

#elif defined(__aarch64__)

    /* Try to decode a relative call (0x9X + 26-bit signed ofs) */

    if (may_access(callers[j] - 4) && may_access(callers[j] - 1) &&

        (*((int*)(callers[j] - 4)) & 0xFC000000) == 0x94000000) {

      int ofs = (*((int *)(callers[j] - 4)) << 6) >> 4; // 26-bit signed immed*4

      const void *addr2 = callers[j] - 4 + ofs;

      resolve_sym_name(buf, " > ", addr2);

    }

#endif

    if ((dump & 3) == 0) {

      /* dump not started, will start *after* ha_thread_dump_one(),

       * ha_panic and ha_backtrace_to_stderr

       */

      if (addr == ha_panic ||

          addr == ha_backtrace_to_stderr || addr == ha_thread_dump_one)

        dump++;

      *buf = bak;

      continue;

    }

    if ((dump & 3) == 1) {

      /* starting */

      if (addr == ha_panic ||

          addr == ha_backtrace_to_stderr || addr == ha_thread_dump_one) {

        *buf = bak;

        continue;

      }

      dump++;

    }

    if ((dump & 3) == 2) {

      /* still dumping */

      if (dump == 6) {

        /* we only stop *after* main and we must send the LF */

        if (addr == main) {

          j = nptrs;

          dump++;

        }

      }

      else if (addr == run_poll_loop || addr == main || addr == run_tasks_from_lists) {

        dump++;

        *buf = bak;

        break;

      }

    }

    /* OK, line dumped */

    chunk_appendf(buf, "\n");

  }

 leave:

  /* unblock temporarily blocked signals */

  ha_sigmask(SIG_SETMASK, &old_mask, NULL);

}

/* dump a backtrace of current thread's stack to stderr. */

void ha_backtrace_to_stderr(void)

{

  char area[8192];

  struct buffer b = b_make(area, sizeof(area), 0, 0);

  ha_dump_backtrace(&b, "  ", 4);

  if (b.data)

    DISGUISE(write(2, b.area, b.data));

}

/* Dumps some known information about the current thread into its dump buffer,

 * and optionally extra info when it's considered safe to do so. The dump will

 * be appended to the buffer, so the caller is responsible for preliminary

 * initializing it. The <is_caller> argument will indicate if the thread is the

 * one requesting the dump (e.g. watchdog, panic etc), in order to display a

 * star ('*') in front of the thread to indicate the requesting one. Any stuck

 * thread is also prefixed with a '>'. The caller is responsible for atomically

 * setting up the thread's dump buffer to point to a valid buffer with enough

 * room. Output will be truncated if it does not fit. When the dump is complete

 * the dump buffer will have bit 0 set to 1 to tell the caller it's done, and

 * the caller will then change that value to indicate it's done once the

 * contents are collected.

 */

void ha_thread_dump_one(struct buffer *buf, int is_caller)

{

  unsigned long long p = th_ctx->prev_cpu_time;

  unsigned long long n = now_cpu_time();

  int stuck = !!(th_ctx->flags & TH_FL_STUCK);

  /* keep a copy of the dump pointer for post-mortem analysis */

  HA_ATOMIC_STORE(&th_ctx->last_dump_buffer, buf);

  chunk_appendf(buf,

                "%c%cThread %-2u: id=0x%llx act=%d glob=%d wq=%d rq=%d tl=%d tlsz=%d rqsz=%d\n"

                "     %2u/%-2u   stuck=%d prof=%d",

                (is_caller) ? '*' : ' ', stuck ? '>' : ' ', tid + 1,

          ha_get_pthread_id(tid),

          thread_has_tasks(),

                !eb_is_empty(&th_ctx->rqueue_shared),

                !eb_is_empty(&th_ctx->timers),

                !eb_is_empty(&th_ctx->rqueue),

                !(LIST_ISEMPTY(&th_ctx->tasklets[TL_URGENT]) &&

      LIST_ISEMPTY(&th_ctx->tasklets[TL_NORMAL]) &&

      LIST_ISEMPTY(&th_ctx->tasklets[TL_BULK]) &&

      MT_LIST_ISEMPTY(&th_ctx->shared_tasklet_list)),

                th_ctx->tasks_in_list,

                th_ctx->rq_total,

          ti->tgid, ti->ltid + 1,

                stuck,

                !!(th_ctx->flags & TH_FL_TASK_PROFILING));

#if defined(USE_THREAD)

  chunk_appendf(buf,

                " harmless=%d isolated=%d",

                !!(_HA_ATOMIC_LOAD(&tg_ctx->threads_harmless) & ti->ltid_bit),

          isolated_thread == tid);

#endif

  chunk_appendf(buf, "\n");

  chunk_appendf(buf, "             cpu_ns: poll=%llu now=%llu diff=%llu\n", p, n, n-p);

  /* this is the end of what we can dump from outside the current thread */

  chunk_appendf(buf, "             curr_task=");

  ha_task_dump(buf, th_ctx->current, "             ");

#if defined(USE_THREAD) && ((DEBUG_THREAD > 0) || defined(DEBUG_FULL))

  /* List the lock history */

  if (th_ctx->lock_history) {

    int lkh, lkl, lbl;

    int done;

    chunk_appendf(buf, "             lock_hist:");

    for (lkl = 7; lkl >= 0; lkl--) {

      lkh = (th_ctx->lock_history >> (lkl * 8)) & 0xff;

      if (!lkh)

        continue;

      chunk_appendf(buf, " %c:%s",

              "URSW"[lkh & 3], lock_label((lkh >> 2) - 1));

    }

    /* now rescan the list to only show those that remain */

    done = 0;

    for (lbl = 0; lbl < LOCK_LABELS; lbl++) {

      /* find the latest occurrence of each label */

      for (lkl = 0; lkl < 8; lkl++) {

        lkh = (th_ctx->lock_history >> (lkl * 8)) & 0xff;

        if (!lkh)

          continue;

        if ((lkh >> 2) == lbl)

          break;

      }

      if (lkl == 8) // not found

        continue;

      if ((lkh & 3) == _LK_UN)

        continue;

      if (!done)

        chunk_appendf(buf, " locked:");

      chunk_appendf(buf, " %s(%c)",

              lock_label((lkh >> 2) - 1),

              "URSW"[lkh & 3]);

      done++;

    }

    chunk_appendf(buf, "\n");

  }

#endif

  if (!(HA_ATOMIC_LOAD(&tg_ctx->threads_idle) & ti->ltid_bit)) {

    /* only dump the stack of active threads */

#ifdef USE_LUA

    if (th_ctx->current &&

        th_ctx->current->process == process_stream && th_ctx->current->context) {

      const struct stream *s = (const struct stream *)th_ctx->current->context;

      struct hlua *hlua = NULL;

      if (s) {

        if (s->hlua[0] && HLUA_IS_BUSY(s->hlua[0]))

          hlua = s->hlua[0];

        else if (s->hlua[1] && HLUA_IS_BUSY(s->hlua[1]))

          hlua = s->hlua[1];

      }

      if (hlua) {

        mark_tainted(TAINTED_LUA_STUCK);

        if (hlua->state_id == 0)

          mark_tainted(TAINTED_LUA_STUCK_SHARED);

      }

    }

#endif

    if (HA_ATOMIC_LOAD(&pool_trim_in_progress))

      mark_tainted(TAINTED_MEM_TRIMMING_STUCK);

    ha_dump_backtrace(buf, "             ", 0);

  }

 leave:

  return;

}

/* Triggers a thread dump from thread <thr>, either directly if it's the

 * current thread or if thread dump signals are not implemented, or by sending

 * a signal if it's a remote one and the feature is supported. The buffer <buf>

 * will get the dump appended, and the caller is responsible for making sure

 * there is enough room otherwise some contents will be truncated. The function

 * waits for the called thread to fill the buffer before returning (or cancelling

 * by reporting NULL). It does not release the called thread yet, unless it's the

 * current one, which in this case is always available. It returns a pointer to

 * the buffer used if the dump was done, otherwise NULL. When the dump starts, it

 * marks the current thread as dumping, which will only be released via a failure

 * (returns NULL) or via a call to ha_dump_thread_done().

 */

struct buffer *ha_thread_dump_fill(struct buffer *buf, int thr)

{

#ifdef USE_THREAD_DUMP

  /* silence bogus warning in gcc 11 without threads */

  ASSUME(0 <= thr && thr < MAX_THREADS);

  if (thr != tid) {

    struct buffer *old = NULL;

    /* try to impose our dump buffer and to reserve the target thread's

     * next dump for us.

     */

    do {

      if (old)

        ha_thread_relax();

      old = NULL;

    } while (!HA_ATOMIC_CAS(&ha_thread_ctx[thr].thread_dump_buffer, &old, buf));

    /* asking the remote thread to dump itself allows to get more details

     * including a backtrace.

     */

    ha_tkill(thr, DEBUGSIG);

    /* now wait for the dump to be done (or cancelled) */

    while (1) {

      buf = HA_ATOMIC_LOAD(&ha_thread_ctx[thr].thread_dump_buffer);

      if ((ulong)buf & 0x1)

        break;

      if (!buf)

        return buf;

      ha_thread_relax();

    }

  }

  else

    ha_thread_dump_one(buf, 1);

#else /* !USE_THREAD_DUMP below, we're on the target thread */

  /* when thread-dump is not supported, we can only dump our own thread */

  if (thr != tid)

    return NULL;

  /* the buffer might not be valid in case of a panic, since we

   * have to allocate it ourselves in this case.

   */

  if ((ulong)buf == 0x2UL)

    buf = get_trash_chunk();

  HA_ATOMIC_STORE(&th_ctx->thread_dump_buffer, buf);

  ha_thread_dump_one(buf, 1);

#endif

  return (struct buffer *)((ulong)buf & ~0x1UL);

}

/* Indicates to the called thread that the dumped data are collected by

 * clearing the thread_dump_buffer pointer. It waits for the dump to be

 * completed if it was not the case, and can also leave if the pointer

 * is already NULL (e.g. if a thread has aborted).

 */

void ha_thread_dump_done(int thr)

{

  struct buffer *old;

  /* silence bogus warning in gcc 11 without threads */

  ASSUME(0 <= thr && thr < MAX_THREADS);

  /* now wait for the dump to be done or cancelled, and release it */

  do {

    if (thr == tid)

      break;

    old = HA_ATOMIC_LOAD(&ha_thread_ctx[thr].thread_dump_buffer);

    if (!((ulong)old & 0x1)) {

      if (!old)

        break;

      ha_thread_relax();

      continue;

    }

  } while (!HA_ATOMIC_CAS(&ha_thread_ctx[thr].thread_dump_buffer, &old, NULL));

}

/* dumps into the buffer some information related to task <task> (which may

 * either be a task or a tasklet, and prepend each line except the first one

 * with <pfx>. The buffer is only appended and the first output starts by the

 * pointer itself. The caller is responsible for making sure the task is not

 * going to vanish during the dump.

 */

void ha_task_dump(struct buffer *buf, const struct task *task, const char *pfx)

{

  const struct stream *s = NULL;

  const struct appctx __maybe_unused *appctx = NULL;

  struct hlua __maybe_unused *hlua = NULL;

  const struct stconn *sc;

  if (!task) {

    chunk_appendf(buf, "0\n");

    return;

  }

  if (TASK_IS_TASKLET(task))

    chunk_appendf(buf,

                  "%p (tasklet) calls=%u\n",

                  task,

                  task->calls);

  else

    chunk_appendf(buf,

                  "%p (task) calls=%u last=%llu%s\n",

                  task,

                  task->calls,

                  task->wake_date ? (unsigned long long)(now_mono_time() - task->wake_date) : 0,

                  task->wake_date ? " ns ago" : "");

  chunk_appendf(buf, "%s  fct=%p(", pfx, task->process);

  resolve_sym_name(buf, NULL, task->process);

  chunk_appendf(buf,") ctx=%p", task->context);

  if (task->process == task_run_applet && (appctx = task->context))

    chunk_appendf(buf, "(%s)\n", appctx->applet->name);

  else

    chunk_appendf(buf, "\n");

  if (task->process == process_stream && task->context)

    s = (struct stream *)task->context;

  else if (task->process == task_run_applet && task->context && (sc = appctx_sc((struct appctx *)task->context)))

    s = sc_strm(sc);

  else if (task->process == sc_conn_io_cb && task->context)

    s = sc_strm(((struct stconn *)task->context));

  if (s) {

    chunk_appendf(buf, "%sstream=", pfx);

    strm_dump_to_buffer(buf, s, pfx, HA_ATOMIC_LOAD(&global.anon_key));

  }

#ifdef USE_LUA

  hlua = NULL;

  if (s && ((s->hlua[0] && HLUA_IS_BUSY(s->hlua[0])) ||

      (s->hlua[1] && HLUA_IS_BUSY(s->hlua[1])))) {

    hlua = (s->hlua[0] && HLUA_IS_BUSY(s->hlua[0])) ? s->hlua[0] : s->hlua[1];

    chunk_appendf(buf, "%sCurrent executing Lua from a stream analyser -- ", pfx);

  }

  else if (task->process == hlua_process_task && (hlua = task->context)) {

    chunk_appendf(buf, "%sCurrent executing a Lua task -- ", pfx);

  }

  else if (task->process == task_run_applet && (appctx = task->context) &&

     (appctx->applet->fct == hlua_applet_tcp_fct)) {

    chunk_appendf(buf, "%sCurrent executing a Lua TCP service -- ", pfx);

  }

  else if (task->process == task_run_applet && (appctx = task->context) &&

     (appctx->applet->fct == hlua_applet_http_fct)) {

    chunk_appendf(buf, "%sCurrent executing a Lua HTTP service -- ", pfx);

  }

  if (hlua && hlua->T) {

    chunk_appendf(buf, "stack traceback:\n    ");

    append_prefixed_str(buf, hlua_traceback(hlua->T, "\n    "), pfx, '\n', 0);

  }

  /* we may need to terminate the current line */

  if (*b_peek(buf, b_data(buf)-1) != '\n')

    b_putchr(buf, '\n');

#endif

}

/* This function dumps all profiling settings. It returns 0 if the output

 * buffer is full and it needs to be called again, otherwise non-zero.

 * Note: to not statify this one, it's hard to spot in backtraces!

 */

int cli_io_handler_show_threads(struct appctx *appctx)

{

  int *thr = appctx->svcctx;

  if (!thr)

    thr = applet_reserve_svcctx(appctx, sizeof(*thr));

  do {

    chunk_reset(&trash);

    if (ha_thread_dump_fill(&trash, *thr)) {

      ha_thread_dump_done(*thr);

      if (applet_putchk(appctx, &trash) == -1) {

        /* failed, try again */

        return 0;

      }

    }

    (*thr)++;

  } while (*thr < global.nbthread);

  return 1;

}

#if defined(HA_HAVE_DUMP_LIBS)

/* parse a "show libs" command. It returns 1 if it emits anything otherwise zero. */

static int debug_parse_cli_show_libs(char **args, char *payload, struct appctx *appctx, void *private)

{

  if (!cli_has_level(appctx, ACCESS_LVL_OPER))

    return 1;

  chunk_reset(&trash);

  if (dump_libs(&trash, 1))

    return cli_msg(appctx, LOG_INFO, trash.area);

  else

    return 0;

}

#endif

/* parse a "show dev" command. It returns 1 if it emits anything otherwise zero. */

static int debug_parse_cli_show_dev(char **args, char *payload, struct appctx *appctx, void *private)

{

  const char **build_opt;

  char *err = NULL;

  int i;

  if (*args[2])

    return cli_err(appctx, "This command takes no argument.\n");

  chunk_reset(&trash);

  chunk_appendf(&trash, "HAProxy version %s\n", haproxy_version);

  chunk_appendf(&trash, "Features\n  %s\n", build_features);

  chunk_appendf(&trash, "Build options\n");

  for (build_opt = NULL; (build_opt = hap_get_next_build_opt(build_opt)); )

    if (append_prefixed_str(&trash, *build_opt, "  ", '\n', 0) == 0)

      chunk_strcat(&trash, "\n");

  chunk_appendf(&trash, "Platform info\n");

  if (*post_mortem.platform.hw_vendor)

    chunk_appendf(&trash, "  machine vendor: %s\n", post_mortem.platform.hw_vendor);

  if (*post_mortem.platform.hw_family)

    chunk_appendf(&trash, "  machine family: %s\n", post_mortem.platform.hw_family);

  if (*post_mortem.platform.hw_model)

    chunk_appendf(&trash, "  machine model: %s\n", post_mortem.platform.hw_model);

  if (*post_mortem.platform.brd_vendor)

    chunk_appendf(&trash, "  board vendor: %s\n", post_mortem.platform.brd_vendor);

  if (*post_mortem.platform.brd_model)

    chunk_appendf(&trash, "  board model: %s\n", post_mortem.platform.brd_model);

  if (*post_mortem.platform.soc_vendor)

    chunk_appendf(&trash, "  soc vendor: %s\n", post_mortem.platform.soc_vendor);

  if (*post_mortem.platform.soc_model)

    chunk_appendf(&trash, "  soc model: %s\n", post_mortem.platform.soc_model);

  if (*post_mortem.platform.cpu_model)

    chunk_appendf(&trash, "  cpu model: %s\n", post_mortem.platform.cpu_model);

  if (*post_mortem.platform.virt_techno)

    chunk_appendf(&trash, "  virtual machine: %s\n", post_mortem.platform.virt_techno);

  if (*post_mortem.platform.cont_techno)

    chunk_appendf(&trash, "  container: %s\n", post_mortem.platform.cont_techno);

  if (*post_mortem.platform.utsname.sysname)

    chunk_appendf(&trash, "  OS name: %s\n", post_mortem.platform.utsname.sysname);

  if (*post_mortem.platform.utsname.release)

    chunk_appendf(&trash, "  OS release: %s\n", post_mortem.platform.utsname.release);

  if (*post_mortem.platform.utsname.version)

    chunk_appendf(&trash, "  OS version: %s\n", post_mortem.platform.utsname.version);

  if (*post_mortem.platform.utsname.machine)

    chunk_appendf(&trash, "  OS architecture: %s\n", post_mortem.platform.utsname.machine);

  if (*post_mortem.platform.utsname.nodename)

    chunk_appendf(&trash, "  node name: %s\n", HA_ANON_CLI(post_mortem.platform.utsname.nodename));

  chunk_appendf(&trash, "Process info\n");

  chunk_appendf(&trash, "  pid: %d\n", post_mortem.process.pid);

  chunk_appendf(&trash, "  cmdline: ");

  for (i = 0; i < post_mortem.process.argc; i++)

    chunk_appendf(&trash, "%s ", post_mortem.process.argv[i]);

  chunk_appendf(&trash, "\n");

#if defined(USE_LINUX_CAP)

  /* let's dump saved in feed_post_mortem() initial capabilities sets */

  if(!post_mortem.process.caps.err_boot) {

    chunk_appendf(&trash, "  boot capabilities:\n");

    chunk_appendf(&trash, "  \tCapEff: 0x%016llx\n",

            CAPS_TO_ULLONG(post_mortem.process.caps.boot[0].effective,

               post_mortem.process.caps.boot[1].effective));

    chunk_appendf(&trash, "  \tCapPrm: 0x%016llx\n",

            CAPS_TO_ULLONG(post_mortem.process.caps.boot[0].permitted,

               post_mortem.process.caps.boot[1].permitted));

    chunk_appendf(&trash, "  \tCapInh: 0x%016llx\n",

            CAPS_TO_ULLONG(post_mortem.process.caps.boot[0].inheritable,

               post_mortem.process.caps.boot[1].inheritable));

  } else

    chunk_appendf(&trash, "  capget() failed at boot with: %s.\n",

            errname(post_mortem.process.caps.err_boot, &err));

  /* let's print actual capabilities sets, could be useful in order to compare */

  if (!post_mortem.process.caps.err_run) {

    chunk_appendf(&trash, "  runtime capabilities:\n");

    chunk_appendf(&trash, "  \tCapEff: 0x%016llx\n",

            CAPS_TO_ULLONG(post_mortem.process.caps.run[0].effective,

               post_mortem.process.caps.run[1].effective));

    chunk_appendf(&trash, "  \tCapPrm: 0x%016llx\n",

            CAPS_TO_ULLONG(post_mortem.process.caps.run[0].permitted,

               post_mortem.process.caps.run[1].permitted));

    chunk_appendf(&trash, "  \tCapInh: 0x%016llx\n",

            CAPS_TO_ULLONG(post_mortem.process.caps.run[0].inheritable,

               post_mortem.process.caps.run[1].inheritable));

  } else

    chunk_appendf(&trash, "  capget() failed at runtime with: %s.\n",

            errname(post_mortem.process.caps.err_run, &err));

#endif

  chunk_appendf(&trash, "  %-22s  %-11s  %-11s \n", "identity:", "-boot-", "-runtime-");

  chunk_appendf(&trash, "  %-22s  %-11d  %-11d \n", "    uid:", post_mortem.process.boot_uid,

                                                          post_mortem.process.run_uid);

  chunk_appendf(&trash, "  %-22s  %-11d  %-11d \n", "    gid:", post_mortem.process.boot_gid,

                                                          post_mortem.process.run_gid);

  chunk_appendf(&trash, "  %-22s  %-11s  %-11s \n", "limits:", "-boot-", "-runtime-");

  chunk_appendf(&trash, "  %-22s  %-11s  %-11s \n", "    fd limit (soft):",

    LIM2A(normalize_rlim((ulong)post_mortem.process.boot_lim_fd.rlim_cur), "unlimited"),

    LIM2A(normalize_rlim((ulong)post_mortem.process.run_lim_fd.rlim_cur), "unlimited"));

  chunk_appendf(&trash, "  %-22s  %-11s  %-11s \n", "    fd limit (hard):",

    LIM2A(normalize_rlim((ulong)post_mortem.process.boot_lim_fd.rlim_max), "unlimited"),

    LIM2A(normalize_rlim((ulong)post_mortem.process.run_lim_fd.rlim_max), "unlimited"));

  chunk_appendf(&trash, "  %-22s  %-11s  %-11s \n", "    ram limit (soft):",

    LIM2A(normalize_rlim((ulong)post_mortem.process.boot_lim_ram.rlim_cur), "unlimited"),

    LIM2A(normalize_rlim((ulong)post_mortem.process.run_lim_ram.rlim_cur), "unlimited"));

  chunk_appendf(&trash, "  %-22s  %-11s  %-11s \n", "    ram limit (hard):",

    LIM2A(normalize_rlim((ulong)post_mortem.process.boot_lim_ram.rlim_max), "unlimited"),

    LIM2A(normalize_rlim((ulong)post_mortem.process.run_lim_ram.rlim_max), "unlimited"));

  ha_free(&err);

  return cli_msg(appctx, LOG_INFO, trash.area);

}

/* Dumps a state of all threads into the trash and on fd #2, then aborts. */

void ha_panic()

{

  struct buffer *buf;

  unsigned int thr;

  if (mark_tainted(TAINTED_PANIC) & TAINTED_PANIC) {

    /* a panic dump is already in progress, let's not disturb it,

     * we'll be called via signal DEBUGSIG. By returning we may be

     * able to leave a current signal handler (e.g. WDT) so that

     * this will ensure more reliable signal delivery.

     */

    return;

  }

  chunk_printf(&trash, "Thread %u is about to kill the process.\n", tid + 1);

  DISGUISE(write(2, trash.area, trash.data));

  for (thr = 0; thr < global.nbthread; thr++) {

    if (thr == tid)

      buf = get_trash_chunk();

    else

      buf = (void *)0x2UL; // let the target thread allocate it

    buf = ha_thread_dump_fill(buf, thr);

    if (!buf)

      continue;

    DISGUISE(write(2, buf->area, buf->data));

    /* restore the thread's dump pointer for easier post-mortem analysis */

    ha_thread_dump_done(thr);

  }

#ifdef USE_LUA

  if (get_tainted() & TAINTED_LUA_STUCK_SHARED && global.nbthread > 1) {

    chunk_printf(&trash,

           "### Note: at least one thread was stuck in a Lua context loaded using the\n"

           "          'lua-load' directive, which is known for causing heavy contention\n"

           "          when used with threads. Please consider using 'lua-load-per-thread'\n"

           "          instead if your code is safe to run in parallel on multiple threads.\n");

    DISGUISE(write(2, trash.area, trash.data));

  }

  else if (get_tainted() & TAINTED_LUA_STUCK) {

    chunk_printf(&trash,

           "### Note: at least one thread was stuck in a Lua context in a way that suggests\n"

           "          heavy processing inside a dependency or a long loop that can't yield.\n"

           "          Please make sure any external code you may rely on is safe for use in\n"

           "          an event-driven engine.\n");

    DISGUISE(write(2, trash.area, trash.data));

  }

#endif

  if (get_tainted() & TAINTED_MEM_TRIMMING_STUCK) {

    chunk_printf(&trash,

           "### Note: one thread was found stuck under malloc_trim(), which can run for a\n"

           "          very long time on large memory systems. You way want to disable this\n"

           "          memory reclaiming feature by setting 'no-memory-trimming' in the\n"

           "          'global' section of your configuration to avoid this in the future.\n");

    DISGUISE(write(2, trash.area, trash.data));

  }

  chunk_printf(&trash,

               "\n"

               "Hint: when reporting this bug to developers, please check if a core file was\n"

               "      produced, open it with 'gdb', issue 't a a bt full', check that the\n"

               "      output does not contain sensitive data, then join it with the bug report.\n"

               "      For more info, please see https://github.com/haproxy/haproxy/issues/2374\n");

  DISGUISE(write(2, trash.area, trash.data));

  for (;;)

    abort();

}

/* Dumps a state of the current thread on fd #2 and returns. It takes a great

 * care about not using any global state variable so as to gracefully recover.

 * It is designed to be called exclusively from the watchdog signal handler,

 * and takes care of not touching thread_dump_buffer so as not to interfere

 * with any other parallel dump that could have been started.

 */

void ha_stuck_warning(void)

{

  char msg_buf[8192];

  struct buffer buf;

  ullong n, p;

  if (mark_tainted(TAINTED_WARN_BLOCKED_TRAFFIC) & TAINTED_PANIC) {

    /* a panic dump is already in progress, let's not disturb it,

     * we'll be called via signal DEBUGSIG. By returning we may be

     * able to leave a current signal handler (e.g. WDT) so that

     * this will ensure more reliable signal delivery.

     */

    return;

  }

  HA_ATOMIC_INC(&warn_blocked_issued);

  buf = b_make(msg_buf, sizeof(msg_buf), 0, 0);

  p = HA_ATOMIC_LOAD(&th_ctx->prev_cpu_time);

  n = now_cpu_time();

  chunk_appendf(&buf,

         "\nWARNING! thread %u has stopped processing traffic for %llu milliseconds\n"

         "    with %d streams currently blocked, prevented from making any progress.\n"

         "    While this may occasionally happen with inefficient configurations\n"

         "    involving excess of regular expressions, map_reg, or heavy Lua processing,\n"

         "    this must remain exceptional because the system's stability is now at risk.\n"

         "    Timers in logs may be reported incorrectly, spurious timeouts may happen,\n"

         "    some incoming connections may silently be dropped, health checks may\n"

         "    randomly fail, and accesses to the CLI may block the whole process. The\n"

         "    blocking delay before emitting this warning may be adjusted via the global\n"

         "    'warn-blocked-traffic-after' directive. Please check the trace below for\n"

         "    any clues about configuration elements that need to be corrected:\n\n",

         tid + 1, (n - p) / 1000000ULL,

         HA_ATOMIC_LOAD(&ha_thread_ctx[tid].stream_cnt));

  ha_thread_dump_one(&buf, 1);

#ifdef USE_LUA

  if (get_tainted() & TAINTED_LUA_STUCK_SHARED && global.nbthread > 1) {

    chunk_appendf(&buf,

           "### Note: at least one thread was stuck in a Lua context loaded using the\n"

           "          'lua-load' directive, which is known for causing heavy contention\n"

           "          when used with threads. Please consider using 'lua-load-per-thread'\n"

           "          instead if your code is safe to run in parallel on multiple threads.\n");

  }

  else if (get_tainted() & TAINTED_LUA_STUCK) {

    chunk_appendf(&buf,

           "### Note: at least one thread was stuck in a Lua context in a way that suggests\n"

           "          heavy processing inside a dependency or a long loop that can't yield.\n"

           "          Please make sure any external code you may rely on is safe for use in\n"

           "          an event-driven engine.\n");

  }

#endif

  if (get_tainted() & TAINTED_MEM_TRIMMING_STUCK) {

    chunk_appendf(&buf,

           "### Note: one thread was found stuck under malloc_trim(), which can run for a\n"

           "          very long time on large memory systems. You way want to disable this\n"

           "          memory reclaiming feature by setting 'no-memory-trimming' in the\n"

           "          'global' section of your configuration to avoid this in the future.\n");

  }

  chunk_appendf(&buf, " => Trying to gracefully recover now.\n");

  /* Note: it's important to dump the whole buffer at once to avoid

   * interleaved outputs from multiple threads dumping in parallel.

   */

  DISGUISE(write(2, buf.area, buf.data));

}

/* Complain with message <msg> on stderr. If <counter> is not NULL, it is

 * atomically incremented, and the message is only printed when the counter

 * was zero, so that the message is only printed once. <taint> is only checked

 * on bit 1, and will taint the process either for a bug (2) or warn (0).

 */

void complain(int *counter, const char *msg, int taint)

{

  if (counter && _HA_ATOMIC_FETCH_ADD(counter, 1))

    return;

  DISGUISE(write(2, msg, strlen(msg)));

  if (taint & 2)

    mark_tainted(TAINTED_BUG);

  else

    mark_tainted(TAINTED_WARN);

}

/* parse a "debug dev exit" command. It always returns 1, though it should never return. */

static int debug_parse_cli_exit(char **args, char *payload, struct appctx *appctx, void *private)

{

  int code = atoi(args[3]);

  if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))

    return 1;

  _HA_ATOMIC_INC(&debug_commands_issued);

  exit(code);

  return 1;

}

/* parse a "debug dev bug" command. It always returns 1, though it should never return.

 * Note: we make sure not to make the function static so that it appears in the trace.

 */

int debug_parse_cli_bug(char **args, char *payload, struct appctx *appctx, void *private)

{

  if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))

    return 1;

  _HA_ATOMIC_INC(&debug_commands_issued);

  BUG_ON(one > zero, "This was triggered on purpose from the CLI 'debug dev bug' command.");

  return 1;

}

/* parse a "debug dev warn" command. It always returns 1.

 * Note: we make sure not to make the function static so that it appears in the trace.

 */

int debug_parse_cli_warn(char **args, char *payload, struct appctx *appctx, void *private)

{

  if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))

    return 1;

  _HA_ATOMIC_INC(&debug_commands_issued);

  WARN_ON(one > zero, "This was triggered on purpose from the CLI 'debug dev warn' command.");

  return 1;

}

/* parse a "debug dev check" command. It always returns 1.

 * Note: we make sure not to make the function static so that it appears in the trace.

 */

int debug_parse_cli_check(char **args, char *payload, struct appctx *appctx, void *private)

{

  if (!cli_has_level(appctx, ACCESS_LVL_ADMIN))

    return 1;

  _HA_ATOMIC_INC(&debug_commands_issued);

  CHECK_IF(one > zero, "This was triggered on purpose from the CLI 'debug dev check' command.");