/***************************************************************************

 *                                  _   _ ____  _

 *  Project                     ___| | | |  _ \| |

 *                             / __| | | | |_) | |

 *                            | (__| |_| |  _ <| |___

 *                             \___|\___/|_| \_\_____|

 *

 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.

 *

 * This software is licensed as described in the file COPYING, which

 * you should have received as part of this distribution. The terms

 * are also available at https://curl.se/docs/copyright.html.

 *

 * You may opt to use, copy, modify, merge, publish, distribute and/or sell

 * copies of the Software, and permit persons to whom the Software is

 * furnished to do so, under the terms of the COPYING file.

 *

 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY

 * KIND, either express or implied.

 *

 * SPDX-License-Identifier: curl

 *

 ***************************************************************************/

#include "curl_setup.h"

#include "urldata.h"

#include "vquic/vquic.h"

#include "vtls/vtls.h"

#include "curl_trc.h"

#if !defined(CURL_DISABLE_HTTP) && defined(USE_HTTP3)

#ifdef HAVE_NETINET_UDP_H

#include <netinet/udp.h>

#endif

#ifdef HAVE_NETINET_IP_H

#include <netinet/ip.h>

#endif

#ifdef USE_NGHTTP3

#include <nghttp3/nghttp3.h>

#endif

#if defined(USE_APPLE_FAST_UDP) && defined(__APPLE__)

#include <sys/syscall.h>

#if defined(SYS_recvmsg_x) && defined(SYS_sendmsg_x)

#define HAVE_APPLE_MSG_X

#endif

#endif

#include "bufq.h"

#include "curlx/dynbuf.h"

#include "curlx/win32-fopen.h"

#include "cfilters.h"

#include "vdns/cf-dns.h"

#include "vquic/cf-ngtcp2.h"

#include "vquic/cf-ngtcp2-cmn.h"

#include "vquic/cf-ngtcp2-proxy.h"

#include "vquic/cf-quiche.h"

#include "multiif.h"

#include "progress.h"

#include "rand.h"

#include "vquic/vquic_int.h"

#include "curlx/strerr.h"

#include "curlx/strparse.h"

#define NW_CHUNK_SIZE     (64 * 1024)

#define NW_SEND_CHUNKS    1

#ifdef HAVE_APPLE_MSG_X

/* this is `struct msghdr` with an additional field at the end */

struct msghdr_x {

  void *msg_name;           /* optional address */

  socklen_t msg_namelen;    /* size of address */

  struct iovec *msg_iov;    /* scatter/gather array */

  int msg_iovlen;           /* # elements in msg_iov */

  void *msg_control;        /* ancillary data, see below */

  socklen_t msg_controllen; /* ancillary data buffer len */

  int msg_flags;            /* flags on received message */

  size_t msg_datalen;       /* byte length of buffer in msg_iov */

};

#endif

#ifdef CURLVERBOSE

#ifdef HAVE_APPLE_MSG_X

#define VQUIC_SEND_METHOD   "sendmsg_x"

#elif defined(HAVE_SENDMSG)

#define VQUIC_SEND_METHOD   "sendmsg"

#else

#define VQUIC_SEND_METHOD   "send"

#endif

#endif

int Curl_vquic_init(void)

{

#if defined(USE_NGTCP2) && defined(OPENSSL_QUIC_API2)

  if(ngtcp2_crypto_ossl_init())

    return 0;

#endif

  return 1;

}

void Curl_vquic_cleanup(void)

{

#if defined(USE_NGTCP2) && defined(OPENSSL_QUIC_API2) && \

  (NGTCP2_VERSION_NUM >= 0x011800)

  ngtcp2_crypto_ossl_free();

#endif

}

void Curl_quic_ver(char *p, size_t len)

{

#if defined(USE_NGTCP2) && defined(USE_NGHTTP3)

  Curl_ngtcp2_ver(p, len);

#elif defined(USE_QUICHE)

  Curl_quiche_ver(p, len);

#endif

}

CURLcode Curl_vquic_ctx_init(struct Curl_easy *data,

                             struct cf_quic_ctx *qctx)

{

  Curl_bufq_init2(&qctx->sendbuf, NW_CHUNK_SIZE, NW_SEND_CHUNKS,

                  BUFQ_OPT_SOFT_LIMIT);

#if defined(__linux__) && defined(UDP_SEGMENT) && defined(HAVE_SENDMSG)

  qctx->no_gso = FALSE;

#elif defined(HAVE_APPLE_MSG_X)

  qctx->no_gso = FALSE;

#else

  qctx->no_gso = TRUE;

#endif

#ifdef DEBUGBUILD

  {

    const char *p = getenv("CURL_DBG_QUIC_WBLOCK");

    if(p) {

      curl_off_t l;

      if(!curlx_str_number(&p, &l, 100))

        qctx->wblock_percent = (int)l;

    }

  }

#endif

  Curl_vquic_ctx_set_time(qctx, Curl_pgrs_now(data));

  return CURLE_OK;

}

void Curl_vquic_ctx_free(struct cf_quic_ctx *qctx)

{

  Curl_bufq_free(&qctx->sendbuf);

}

void Curl_vquic_ctx_set_time(struct cf_quic_ctx *qctx,

                             const struct curltime *pnow)

{

  qctx->last_op = *pnow;

}

void Curl_vquic_ctx_update_time(struct cf_quic_ctx *qctx,

                                const struct curltime *pnow)

{

  qctx->last_op = *pnow;

}

#ifdef HAVE_APPLE_MSG_X

static CURLcode do_sendmsg(struct Curl_cfilter *cf,

                           struct Curl_easy *data,

                           struct cf_quic_ctx *qctx,

                           const uint8_t *pkt, size_t pktlen, size_t gsolen,

                           size_t *psent)

{

#define MSG_X_SNUM  64

  struct iovec msg_iov[MSG_X_SNUM];

  struct msghdr_x mmsg[MSG_X_SNUM];

  char errstr[STRERROR_LEN];

  size_t n, i = 0, sent = 0;

  int rc;

  CURLcode result = CURLE_OK;

  VERBOSE(size_t calls = 0);

  *psent = 0;

  if(!pktlen)

    return CURLE_OK;

  if(!gsolen || (pktlen < gsolen))

    gsolen = pktlen;

  n = (pktlen + gsolen - 1) / gsolen;

  while(i < n) {

    size_t j, batch = CURLMIN(n - i, MSG_X_SNUM), pkts_sent = 0;

    for(j = 0; j < batch; ++j) {

      const size_t offset = (i + j) * gsolen;

      msg_iov[j].iov_base = CURL_UNCONST(pkt + offset);

      msg_iov[j].iov_len = CURLMIN(gsolen, pktlen - offset);

      memset(&mmsg[j], 0, sizeof(mmsg[j]));

      mmsg[j].msg_iov = &msg_iov[j];

      mmsg[j].msg_iovlen = 1;

      mmsg[j].msg_datalen = msg_iov[j].iov_len;

    }

#if defined(CURL_HAVE_DIAG) && defined(__APPLE__)

#pragma GCC diagnostic push

#pragma GCC diagnostic ignored "-Wdeprecated-declarations"

#endif

    while((rc = syscall(SYS_sendmsg_x, qctx->sockfd, &mmsg, batch, 0)) == -1 &&

          (SOCKERRNO == SOCKEINTR))

      ;

#if defined(CURL_HAVE_DIAG) && defined(__APPLE__)

#pragma GCC diagnostic pop

#endif

    if(rc < 0) {

      if(SOCK_EAGAIN(SOCKERRNO)) {

        CURL_TRC_CF(data, cf, "egress, sendmsg_x -> EAGAIN");

        result = sent ? CURLE_OK : CURLE_AGAIN;

        goto out;

      }

      if(SOCKERRNO != SOCKEMSGSIZE) {

        curlx_strerror(SOCKERRNO, errstr, sizeof(errstr));

        failf(data, "QUIC: sendmsg_x() returned %d (errno=%d; %s)",

              rc, SOCKERRNO, errstr);

        result = CURLE_SEND_ERROR;

        goto out;

      }

      /* Error was SOCKEMSGSIZE. Network stack does not accept the packet

       * length(s). This might be a PMTUD. Just drop it into the void

       * and fall through to the success handling. */

      pkts_sent = batch;

    }

    else

      pkts_sent = CURLMIN((size_t)rc, batch);

    VERBOSE(++calls);

    if(!pkts_sent) {  /* no packets of the current batch were sent */

      result = sent ? CURLE_OK : CURLE_AGAIN;

      goto out;

    }

    i += pkts_sent; /* Some have been sent */

    for(j = 0; j < pkts_sent; ++j)

      sent += msg_iov[j].iov_len;

    if(pkts_sent < batch)

      goto out; /* but not all of them */

  }

out:

  *psent = sent;

  if(sent || result)

    CURL_TRC_CF(data, cf,

                "vquic_sendmsg_x(len=%zu, gso=%zu, packets=%zu, "

                "calls=%zu) -> %d",

                sent, gsolen, n, calls, (int)result);

  return result;

}

#else /* HAVE_APPLE_MSG_X */

static CURLcode send_packet_no_gso(struct Curl_cfilter *cf,

                                   struct Curl_easy *data,

                                   struct cf_quic_ctx *qctx,

                                   const uint8_t *pkt, size_t pktlen,

                                   size_t gsolen, size_t *psent);

static CURLcode do_sendmsg(struct Curl_cfilter *cf,

                           struct Curl_easy *data,

                           struct cf_quic_ctx *qctx,

                           const uint8_t *pkt, size_t pktlen, size_t gsolen,

                           size_t *psent)

{

  CURLcode result = CURLE_OK;

#ifdef HAVE_SENDMSG

  struct iovec msg_iov;

  struct msghdr msg = { 0 };

  ssize_t rv;

#if defined(__linux__) && defined(UDP_SEGMENT)

  uint8_t msg_ctrl[32];

  struct cmsghdr *cm;

#endif

  *psent = 0;

  msg_iov.iov_base = (uint8_t *)CURL_UNCONST(pkt);

  msg_iov.iov_len = pktlen;

  msg.msg_iov = &msg_iov;

  msg.msg_iovlen = 1;

#if defined(__linux__) && defined(UDP_SEGMENT)

  if(pktlen > gsolen) {

    /* Only set this, when we need it. macOS, for example,

     * does not seem to like a msg_control of length 0. */

    memset(msg_ctrl, 0, sizeof(msg_ctrl));

    msg.msg_control = msg_ctrl;

    DEBUGASSERT(sizeof(msg_ctrl) >= CMSG_SPACE(sizeof(int)));

    msg.msg_controllen = CMSG_SPACE(sizeof(int));

    cm = CMSG_FIRSTHDR(&msg);

    cm->cmsg_level = SOL_UDP;

    cm->cmsg_type = UDP_SEGMENT;

    cm->cmsg_len = CMSG_LEN(sizeof(uint16_t));

    *(uint16_t *)(void *)CMSG_DATA(cm) = gsolen & 0xffff;

  }

#endif

  while((rv = sendmsg(qctx->sockfd, &msg, 0)) == -1 && SOCKERRNO == SOCKEINTR)

    ;

  if(!curlx_sztouz(rv, psent)) {

    int sockerr = SOCKERRNO;

    if(SOCK_EAGAIN(sockerr))

      return CURLE_AGAIN;

    switch(sockerr) {

    case SOCKEMSGSIZE:

      /* UDP datagram is too large; caused by PMTUD. Let it be lost. */

      *psent = pktlen;

      break;

    case EIO:

      if(pktlen > gsolen) {

        /* GSO failure */

        infof(data, "sendmsg() returned %zd (errno %d); disable GSO", rv,

              sockerr);

        qctx->no_gso = TRUE;

        return send_packet_no_gso(cf, data, qctx, pkt, pktlen, gsolen, psent);

      }

      FALLTHROUGH();

    default:

      failf(data, "sendmsg() returned %zd (errno %d)", rv, sockerr);

      result = CURLE_SEND_ERROR;

      goto out;

    }

  }

  else if(pktlen != *psent) {

    failf(data, "sendmsg() sent only %zu/%zu bytes", *psent, pktlen);

    result = CURLE_SEND_ERROR;

    goto out;

  }

#else

  ssize_t rv;

  (void)gsolen;

  *psent = 0;

  while((rv = swrite(qctx->sockfd, pkt, pktlen)) == -1 &&

        SOCKERRNO == SOCKEINTR)

    ;

  if(!curlx_sztouz(rv, psent)) {

    if(SOCK_EAGAIN(SOCKERRNO)) {

      result = CURLE_AGAIN;

      goto out;

    }

    else {

      if(SOCKERRNO != SOCKEMSGSIZE) {

        failf(data, "send() returned %zd (errno %d)", rv, SOCKERRNO);

        result = CURLE_SEND_ERROR;

        goto out;

      }

      /* UDP datagram is too large; caused by PMTUD. Let it be lost. */

      *psent = pktlen;

    }

  }

#endif

  (void)cf;

out:

  CURL_TRC_CF(data, cf,

              "vquic_%s(len=%zu, gso=%zu, calls=1) -> %d, sent=%zu",

              VQUIC_SEND_METHOD, pktlen, gsolen, (int)result, *psent);

  return result;

}

#endif /* !HAVE_APPLE_MSG_X */

static CURLcode send_packet_no_gso(struct Curl_cfilter *cf,

                                   struct Curl_easy *data,

                                   struct cf_quic_ctx *qctx,

                                   const uint8_t *pkt, size_t pktlen,

                                   size_t gsolen, size_t *psent)

{

  const uint8_t *p, *end = pkt + pktlen;

  size_t sent, len;

  CURLcode result = CURLE_OK;

  VERBOSE(size_t calls = 0);

  *psent = 0;

  for(p = pkt; p < end; p += gsolen) {

    len = CURLMIN(gsolen, (size_t)(end - p));

    result = do_sendmsg(cf, data, qctx, p, len, len, &sent);

    if(result)

      goto out;

    *psent += sent;

    VERBOSE(++calls);

  }

out:

  CURL_TRC_CF(data, cf,

              "vquic_%s(len=%zu, gso=%zu, calls=%zu) -> %d, sent=%zu",

              VQUIC_SEND_METHOD, pktlen, gsolen, calls, (int)result, *psent);

  return result;

}

/* Split QUIC payload by datagram (gso) boundaries when sending over a

 * non-UDP lower filter (for example CONNECT-UDP proxy tunnel). */

static CURLcode send_packet_no_gso_cf(struct Curl_cfilter *cf,

                                      struct Curl_easy *data,

                                      const uint8_t *pkt, size_t pktlen,

                                      size_t gsolen, size_t *psent)

{

  const uint8_t *p, *end = pkt + pktlen;

  size_t sent, len;

  CURLcode result = CURLE_OK;

  VERBOSE(size_t calls = 0);

  *psent = 0;

  /* Send one datagram-sized chunk per call into the lower filter. */

  for(p = pkt; p < end; p += len) {

    len = CURLMIN(gsolen, (size_t)(end - p));

    result = Curl_conn_cf_send(cf->next, data, p, len, FALSE, &sent);

    /* Report forward progress even if we return CURLE_AGAIN later. */

    VERBOSE(++calls);

    /* Preserve lower-filter errors (including CURLE_AGAIN). */

    if(result)

      goto out;

    if(sent != len) {

      /* We can only send the complete datagram, not parts. */

      result = CURLE_SEND_ERROR;

      goto out;

    }

    *psent += sent;

  }

out:

  CURL_TRC_CF(data, cf,

              "vquic_cf_send(len=%zu, gso=%zu, calls=%zu) -> %d, sent=%zu",

              pktlen, gsolen, calls, (int)result, *psent);

  return result;

}

static CURLcode vquic_send_packets(struct Curl_cfilter *cf,

                                   struct Curl_easy *data,

                                   struct cf_quic_ctx *qctx,

                                   const uint8_t *pkt, size_t pktlen,

                                   size_t gsolen, size_t *psent)

{

  CURLcode result;

#ifdef DEBUGBUILD

  /* simulate network blocking/partial writes */

  if(qctx->wblock_percent > 0) {

    unsigned char c;

    *psent = 0;

    Curl_rand(data, &c, 1);

    if(c >= ((100 - qctx->wblock_percent) * 256 / 100)) {

      CURL_TRC_CF(data, cf, "vquic_flush() simulate EWOULDBLOCK");

      return CURLE_AGAIN;

    }

  }

#endif

  if(qctx->no_gso && pktlen > gsolen) {

    result = send_packet_no_gso(cf, data, qctx, pkt, pktlen, gsolen, psent);

  }

  else {

    result = do_sendmsg(cf, data, qctx, pkt, pktlen, gsolen, psent);

  }

  if(!result)

    qctx->last_io = qctx->last_op;

  return result;

}

CURLcode Curl_vquic_flush(struct Curl_cfilter *cf, struct Curl_easy *data,

                          struct cf_quic_ctx *qctx)

{

  const unsigned char *buf;

  size_t blen, sent;

  CURLcode result;

  size_t gsolen;

  while(Curl_bufq_peek(&qctx->sendbuf, &buf, &blen)) {

    gsolen = qctx->gsolen;

    if(qctx->split_len) {

      gsolen = qctx->split_gsolen;

      if(blen > qctx->split_len)

        blen = qctx->split_len;

    }

    if(qctx->sockfd != CURL_SOCKET_BAD) {

      /* Direct UDP socket (via happy eyeballs) */

      result = vquic_send_packets(cf, data, qctx, buf, blen, gsolen, &sent);

    }

    else {

      /* Tunneled QUIC (CONNECT-UDP through proxy) */

      if(gsolen && (blen > gsolen)) {

        /* Send one datagram at a time to preserve packet boundaries. */

        result = send_packet_no_gso_cf(cf, data, buf, blen, gsolen, &sent);

      }

      else {

        /* No GSO aggregate to split, regular lower-filter send is enough. */

        result = Curl_conn_cf_send(cf->next, data, buf, blen, FALSE, &sent);

      }

    }

    if(result) {

      if(result == CURLE_AGAIN) {

        Curl_bufq_skip(&qctx->sendbuf, sent);

        if(qctx->split_len)

          qctx->split_len -= sent;

      }

      return result;

    }

    Curl_bufq_skip(&qctx->sendbuf, sent);

    if(qctx->split_len)

      qctx->split_len -= sent;

  }

  return CURLE_OK;

}

CURLcode Curl_vquic_send(struct Curl_cfilter *cf, struct Curl_easy *data,

                         struct cf_quic_ctx *qctx, size_t gsolen)

{

  qctx->gsolen = gsolen;

  return Curl_vquic_flush(cf, data, qctx);

}

CURLcode Curl_vquic_send_tail_split(struct Curl_cfilter *cf,

                                    struct Curl_easy *data,

                                    struct cf_quic_ctx *qctx, size_t gsolen,

                                    size_t tail_len, size_t tail_gsolen)

{

  DEBUGASSERT(Curl_bufq_len(&qctx->sendbuf) > tail_len);

  qctx->split_len = Curl_bufq_len(&qctx->sendbuf) - tail_len;

  qctx->split_gsolen = gsolen;

  qctx->gsolen = tail_gsolen;

  CURL_TRC_CF(data, cf, "vquic_send_tail_split: [%zu gso=%zu][%zu gso=%zu]",

              qctx->split_len, qctx->split_gsolen, tail_len, qctx->gsolen);

  return Curl_vquic_flush(cf, data, qctx);

}

#if (defined(HAVE_SENDMMSG) || defined(HAVE_SENDMSG)) && \

  !defined(HAVE_APPLE_MSG_X)

static size_t vquic_msghdr_get_udp_gro(struct msghdr *msg)

{

  int gso_size = 0;

#if defined(__linux__) && defined(UDP_GRO)

  struct cmsghdr *cmsg;

  /* Workaround musl CMSG_NXTHDR issue */

#if defined(__clang__) && !defined(__GLIBC__)

#pragma clang diagnostic push

#pragma clang diagnostic ignored "-Wsign-compare"

#pragma clang diagnostic ignored "-Wcast-align"

#endif

  for(cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {

#if defined(__clang__) && !defined(__GLIBC__)

#pragma clang diagnostic pop

#endif

    if(cmsg->cmsg_level == SOL_UDP && cmsg->cmsg_type == UDP_GRO) {

      memcpy(&gso_size, CMSG_DATA(cmsg), sizeof(gso_size));

      break;

    }

  }

#endif /* linux && UDP_GRO */

  (void)msg;

  return (size_t)gso_size;

}

#endif /* (HAVE_SENDMMSG || HAVE_SENDMSG) && !HAVE_APPLE_MSG_X */

/* IP_RECVTOS was added in the macOS 10.13 SDK */

#if (defined(HAVE_SENDMMSG) || defined(HAVE_SENDMSG) || \

  defined(HAVE_APPLE_MSG_X)) && \

  ((defined(__APPLE__) && defined(IP_RECVTOS)) || \

   (!defined(__APPLE__) && defined(IP_TOS))) && \

  defined(IPTOS_ECN_MASK)

static uint8_t vquic_msghdr_get_ecn(struct msghdr *msg, int family)

{

  struct cmsghdr *cmsg;

  switch(family) {

  case AF_INET:

  /* Workaround musl CMSG_NXTHDR issue */

#if defined(__clang__) && !defined(__GLIBC__)

#pragma clang diagnostic push

#pragma clang diagnostic ignored "-Wsign-compare"

#pragma clang diagnostic ignored "-Wcast-align"

#endif

    for(cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {

#if defined(__clang__) && !defined(__GLIBC__)

#pragma clang diagnostic pop

#endif

      if(cmsg->cmsg_level == IPPROTO_IP &&

#ifdef __APPLE__

          cmsg->cmsg_type == IP_RECVTOS

#else

          cmsg->cmsg_type == IP_TOS

#endif

          && cmsg->cmsg_len) {

        return *(uint8_t *)(CMSG_DATA(cmsg)) & IPTOS_ECN_MASK;

      }

    }

    break;

  case AF_INET6:

  /* Workaround musl CMSG_NXTHDR issue */

#if defined(__clang__) && !defined(__GLIBC__)

#pragma clang diagnostic push

#pragma clang diagnostic ignored "-Wsign-compare"

#pragma clang diagnostic ignored "-Wcast-align"

#endif

    for(cmsg = CMSG_FIRSTHDR(msg); cmsg; cmsg = CMSG_NXTHDR(msg, cmsg)) {

#if defined(__clang__) && !defined(__GLIBC__)

#pragma clang diagnostic pop

#endif

      if(cmsg->cmsg_level == IPPROTO_IPV6 && cmsg->cmsg_type == IPV6_TCLASS &&

         cmsg->cmsg_len) {

        unsigned int tos;

        memcpy(&tos, CMSG_DATA(cmsg), sizeof(int));

        return (uint8_t)(tos & IPTOS_ECN_MASK);

      }

    }

    break;

  }

  return 0;

}

#else

#define vquic_msghdr_get_ecn(a, b)      0

#endif /* HAVE_SENDMMSG || HAVE_SENDMSG || HAVE_APPLE_MSG_X ... */

#ifdef HAVE_SENDMMSG

static CURLcode recvmmsg_packets(struct Curl_cfilter *cf,

                                 struct Curl_easy *data,

                                 struct cf_quic_ctx *qctx,

                                 size_t max_pkts,

                                 Curl_vquic_recv_pkts_cb *recv_cb, void *userp)

{

#if defined(__linux__) && defined(UDP_GRO)

#define MMSG_NUM  16

#define UDP_GRO_CNT_MAX  64

#define CMSG_PER_MSG_SIZE    (2 * CMSG_SPACE(sizeof(int)))

#else

#define MMSG_NUM  64

#define UDP_GRO_CNT_MAX  1

#define CMSG_PER_MSG_SIZE    CMSG_SPACE(sizeof(int))

#endif

#define MSG_BUF_SIZE  (UDP_GRO_CNT_MAX * 1500)

  struct iovec msg_iov[MMSG_NUM];

  struct mmsghdr mmsg[MMSG_NUM];

  uint8_t msg_ctrl[MMSG_NUM * CMSG_PER_MSG_SIZE];

  struct sockaddr_storage remote_addr[MMSG_NUM];

  size_t total_nread = 0, pkts = 0;

#ifdef CURLVERBOSE

  size_t calls = 0;

#endif

  int mcount, i, n;

  char errstr[STRERROR_LEN];

  CURLcode result = CURLE_OK;

  size_t gso_size;

  char *sockbuf = NULL;

  uint8_t (*bufs)[MSG_BUF_SIZE] = NULL;

  uint8_t ecn = 0;

  DEBUGASSERT(max_pkts > 0);

  result = Curl_multi_xfer_sockbuf_borrow(data, MMSG_NUM * MSG_BUF_SIZE,

                                          &sockbuf);

  if(result)

    goto out;

  bufs = (uint8_t (*)[MSG_BUF_SIZE])sockbuf;

  total_nread = 0;

  while(pkts < max_pkts) {

    n = (int)CURLMIN(CURLMIN(MMSG_NUM, IOV_MAX), max_pkts);

    memset(&mmsg, 0, sizeof(mmsg));

    for(i = 0; i < n; ++i) {

      msg_iov[i].iov_base = bufs[i];

      msg_iov[i].iov_len = sizeof(bufs[i]);

      mmsg[i].msg_hdr.msg_iov = &msg_iov[i];

      mmsg[i].msg_hdr.msg_iovlen = 1;

      mmsg[i].msg_hdr.msg_name = &remote_addr[i];

      mmsg[i].msg_hdr.msg_namelen = sizeof(remote_addr[i]);

      mmsg[i].msg_hdr.msg_control = &msg_ctrl[i * CMSG_PER_MSG_SIZE];

      mmsg[i].msg_hdr.msg_controllen = CMSG_PER_MSG_SIZE;

    }

    while((mcount = recvmmsg(qctx->sockfd, mmsg, n, 0, NULL)) == -1 &&

          (SOCKERRNO == SOCKEINTR || SOCKERRNO == SOCKEMSGSIZE))

      ;

    if(mcount == -1) {

      if(SOCK_EAGAIN(SOCKERRNO)) {

        CURL_TRC_CF(data, cf, "ingress, recvmmsg -> EAGAIN");

        goto out;

      }

      if(!cf->connected && SOCKERRNO == SOCKECONNREFUSED) {

        struct ip_quadruple ip;

        if(!Curl_cf_socket_peek(cf->next, data, NULL, NULL, &ip))

          failf(data, "QUIC: connection to %s port %u refused",

                ip.remote_ip, ip.remote_port);

        result = CURLE_COULDNT_CONNECT;

        goto out;

      }

      curlx_strerror(SOCKERRNO, errstr, sizeof(errstr));

      failf(data, "QUIC: recvmmsg() unexpectedly returned %d (errno=%d; %s)",

            mcount, SOCKERRNO, errstr);

      result = CURLE_RECV_ERROR;

      goto out;

    }

    VERBOSE(++calls);

    for(i = 0; i < mcount; ++i) {

      /* A zero-length UDP packet is no QUIC packet. Ignore. */

      if(!mmsg[i].msg_len) {

        ++pkts;

        continue;

      }

      total_nread += mmsg[i].msg_len;

      ecn = vquic_msghdr_get_ecn(&mmsg[i].msg_hdr, remote_addr[i].ss_family);

      gso_size = vquic_msghdr_get_udp_gro(&mmsg[i].msg_hdr);

      if(gso_size == 0)

        gso_size = mmsg[i].msg_len;

      result = recv_cb(bufs[i], mmsg[i].msg_len, gso_size,

                       mmsg[i].msg_hdr.msg_name,

                       mmsg[i].msg_hdr.msg_namelen, ecn, userp);

      if(result)

        goto out;

      pkts += (mmsg[i].msg_len + gso_size - 1) / gso_size;

    }

  }

out:

  if(total_nread || result)

    CURL_TRC_CF(data, cf,

                "vquic_recvmmsg(len=%zu, packets=%zu, calls=%zu) -> %d",

                total_nread, pkts, calls, (int)result);

  Curl_multi_xfer_sockbuf_release(data, sockbuf);

  return result;

}

#elif defined(HAVE_APPLE_MSG_X)

static CURLcode recvmsg_x_packets(struct Curl_cfilter *cf,

                                  struct Curl_easy *data,

                                  struct cf_quic_ctx *qctx,

                                  size_t max_pkts,

                                  Curl_vquic_recv_pkts_cb *recv_cb,

                                  void *userp)

{

#define MSG_X_NUM  64

#define MSG_BUF_SIZE  (2048)

#define CMSG_PER_MSG_SIZE     CMSG_SPACE(sizeof(int))

  struct iovec msg_iov[MSG_X_NUM];

  struct msghdr_x mmsg[MSG_X_NUM];

  uint8_t msg_ctrl[MSG_X_NUM * CMSG_PER_MSG_SIZE];

  struct sockaddr_storage remote_addr[MSG_X_NUM];

  size_t total_nread = 0, pkts = 0;

#ifdef CURLVERBOSE

  size_t calls = 0;

#endif

  int mcount, i;

  char errstr[STRERROR_LEN];

  CURLcode result = CURLE_OK;

  size_t gso_size;

  char *sockbuf = NULL;

  uint8_t (*bufs)[MSG_BUF_SIZE] = NULL;

  uint8_t ecn = 0;

  DEBUGASSERT(max_pkts > 0);

  result = Curl_multi_xfer_sockbuf_borrow(data, MSG_X_NUM * MSG_BUF_SIZE,

                                          &sockbuf);

  if(result)

    goto out;

  bufs = (uint8_t (*)[MSG_BUF_SIZE])sockbuf;

  total_nread = 0;

  while(pkts < max_pkts) {

    int n = (int)CURLMIN(CURLMIN(MSG_X_NUM, IOV_MAX), max_pkts);

    memset(&mmsg, 0, sizeof(mmsg));

    for(i = 0; i < n; ++i) {

      msg_iov[i].iov_base = bufs[i];

      msg_iov[i].iov_len = sizeof(bufs[i]);

      mmsg[i].msg_iov = &msg_iov[i];

      mmsg[i].msg_iovlen = 1;

      mmsg[i].msg_name = &remote_addr[i];

      mmsg[i].msg_namelen = sizeof(remote_addr[i]);

      mmsg[i].msg_control = &msg_ctrl[i * CMSG_PER_MSG_SIZE];

      mmsg[i].msg_controllen = CMSG_PER_MSG_SIZE;

    }

#if defined(CURL_HAVE_DIAG) && defined(__APPLE__)

#pragma GCC diagnostic push

#pragma GCC diagnostic ignored "-Wdeprecated-declarations"

#endif

    while((mcount = syscall(SYS_recvmsg_x, qctx->sockfd, mmsg, n, 0)) == -1 &&

          (SOCKERRNO == SOCKEINTR || SOCKERRNO == SOCKEMSGSIZE))

      ;

#if defined(CURL_HAVE_DIAG) && defined(__APPLE__)

#pragma GCC diagnostic pop

#endif

    if(mcount == -1) {

      if(SOCK_EAGAIN(SOCKERRNO)) {

        CURL_TRC_CF(data, cf, "ingress, recvmsg_x -> EAGAIN");

        goto out;

      }

      if(!cf->connected && SOCKERRNO == SOCKECONNREFUSED) {

        struct ip_quadruple ip;

        if(!Curl_cf_socket_peek(cf->next, data, NULL, NULL, &ip))

          failf(data, "QUIC: connection to %s port %u refused",

                ip.remote_ip, ip.remote_port);

        result = CURLE_COULDNT_CONNECT;

        goto out;

      }

      curlx_strerror(SOCKERRNO, errstr, sizeof(errstr));

      failf(data, "QUIC: recvmsg_x() unexpectedly returned %d (errno=%d; %s)",

            mcount, SOCKERRNO, errstr);

      result = CURLE_RECV_ERROR;

      goto out;

    }

    VERBOSE(++calls);

    for(i = 0; i < mcount; ++i) {

      /* A zero-length UDP packet is no QUIC packet. Ignore. */

      if(!mmsg[i].msg_datalen) {

        ++pkts;

        continue;

      }

      total_nread += mmsg[i].msg_datalen;

      ecn = vquic_msghdr_get_ecn((struct msghdr *)&mmsg[i],

                                 remote_addr[i].ss_family);

      gso_size = mmsg[i].msg_datalen;

      result = recv_cb(bufs[i], mmsg[i].msg_datalen, gso_size,

                       mmsg[i].msg_name,

                       mmsg[i].msg_namelen, ecn, userp);

      if(result)

        goto out;

      pkts += (mmsg[i].msg_datalen + gso_size - 1) / gso_size;

    }

  }

out:

  if(total_nread || result)

    CURL_TRC_CF(data, cf,

                "vquic_recvmsg_x(len=%zu, packets=%zu, calls=%zu) -> %d",

                total_nread, pkts, calls, (int)result);

  Curl_multi_xfer_sockbuf_release(data, sockbuf);

  return result;

}

#elif defined(HAVE_SENDMSG)

static CURLcode recvmsg_packets(struct Curl_cfilter *cf,

                                struct Curl_easy *data,

                                struct cf_quic_ctx *qctx,

                                size_t max_pkts,

                                Curl_vquic_recv_pkts_cb *recv_cb, void *userp)

{

#define CMSG_PER_MSG_SIZE    CMSG_SPACE(sizeof(int))

  struct iovec msg_iov;

  struct msghdr msg;

  uint8_t buf[64 * 1024];

  struct sockaddr_storage remote_addr;

  size_t total_nread, pkts, calls;

  ssize_t rc;

  size_t nread;

  char errstr[STRERROR_LEN];

  CURLcode result = CURLE_OK;

  uint8_t msg_ctrl[CMSG_PER_MSG_SIZE];

  size_t gso_size;

  uint8_t ecn = 0;

  DEBUGASSERT(max_pkts > 0);

  for(pkts = 0, total_nread = 0, calls = 0; pkts < max_pkts;) {

    /* fully initialize this on each call to `recvmsg()`. There seem to

     * operating systems out there that mess with `msg_iov.iov_len`. */

    memset(&msg, 0, sizeof(msg));

    msg_iov.iov_base = buf;

    msg_iov.iov_len = sizeof(buf);

    msg.msg_iov = &msg_iov;

    msg.msg_iovlen = 1;

    msg.msg_control = msg_ctrl;

    msg.msg_name = &remote_addr;

    msg.msg_namelen = sizeof(remote_addr);

    msg.msg_controllen = CMSG_PER_MSG_SIZE;

    while((rc = recvmsg(qctx->sockfd, &msg, 0)) == -1 &&

          (SOCKERRNO == SOCKEINTR || SOCKERRNO == SOCKEMSGSIZE))

      ;

    if(!curlx_sztouz(rc, &nread)) {

      if(SOCK_EAGAIN(SOCKERRNO)) {

        goto out;

      }

      if(!cf->connected && SOCKERRNO == SOCKECONNREFUSED) {

        struct ip_quadruple ip;

        if(!Curl_cf_socket_peek(cf->next, data, NULL, NULL, &ip))

          failf(data, "QUIC: connection to %s port %u refused",

                ip.remote_ip, ip.remote_port);

        result = CURLE_COULDNT_CONNECT;

        goto out;

      }

      curlx_strerror(SOCKERRNO, errstr, sizeof(errstr));

      failf(data, "QUIC: recvmsg() unexpectedly returned %zd (errno=%d; %s)",

            rc, SOCKERRNO, errstr);

      result = CURLE_RECV_ERROR;

      goto out;

    }

    total_nread += nread;

    ++calls;

    /* A 0-length UDP packet is no QUIC packet */

    if(!nread) {

      ++pkts;

      continue;

    }

    ecn = vquic_msghdr_get_ecn(&msg, remote_addr.ss_family);

    gso_size = vquic_msghdr_get_udp_gro(&msg);

    if(gso_size == 0)

      gso_size = nread;

    result = recv_cb(buf, nread, gso_size,

                     msg.msg_name, msg.msg_namelen, ecn, userp);

    if(result)

      goto out;

    pkts += (nread + gso_size - 1) / gso_size;

  }

out:

  if(total_nread || result)

    CURL_TRC_CF(data, cf,

                "vquic_recvmsg(len=%zu, packets=%zu, calls=%zu) -> %d",

                total_nread, pkts, calls, (int)result);

  return result;

}

#else /* HAVE_SENDMMSG || HAVE_SENDMSG */

static CURLcode recvfrom_packets(struct Curl_cfilter *cf,

                                 struct Curl_easy *data,

                                 struct cf_quic_ctx *qctx,

                                 size_t max_pkts,

                                 Curl_vquic_recv_pkts_cb *recv_cb, void *userp)

{

  uint8_t buf[64 * 1024];

  int bufsize = (int)sizeof(buf);

  struct sockaddr_storage remote_addr;

  socklen_t remote_addrlen = sizeof(remote_addr);

  size_t total_nread, pkts, calls = 0, nread;

  ssize_t rv;

  char errstr[STRERROR_LEN];

  CURLcode result = CURLE_OK;

  DEBUGASSERT(max_pkts > 0);

  for(pkts = 0, total_nread = 0; pkts < max_pkts;) {

    while((rv = recvfrom(qctx->sockfd, (char *)buf, bufsize, 0,

                         (struct sockaddr *)&remote_addr,

                         &remote_addrlen)) == -1 &&

          (SOCKERRNO == SOCKEINTR || SOCKERRNO == SOCKEMSGSIZE))

      ;

    if(!curlx_sztouz(rv, &nread)) {

      if(SOCK_EAGAIN(SOCKERRNO)) {

        CURL_TRC_CF(data, cf, "ingress, recvfrom -> EAGAIN");

        goto out;

      }

      if(!cf->connected && SOCKERRNO == SOCKECONNREFUSED) {

        struct ip_quadruple ip;

        if(!Curl_cf_socket_peek(cf->next, data, NULL, NULL, &ip))

          failf(data, "QUIC: connection to %s port %u refused",

                ip.remote_ip, ip.remote_port);

        result = CURLE_COULDNT_CONNECT;

        goto out;

      }

      curlx_strerror(SOCKERRNO, errstr, sizeof(errstr));

      failf(data, "QUIC: recvfrom() unexpectedly returned %zd (errno=%d; %s)",

            rv, SOCKERRNO, errstr);

      result = CURLE_RECV_ERROR;

      goto out;

    }

    ++pkts;

    ++calls;

    /* A 0-length UDP packet is no QUIC packet */

    if(!nread)

      continue;

    total_nread += nread;

    result = recv_cb(buf, nread, nread, &remote_addr, remote_addrlen,

                     0, userp);

    if(result)

      goto out;

  }

out:

  if(total_nread || result)

    CURL_TRC_CF(data, cf,

                "vquic_recvfrom(len=%zu, packets=%zu, calls=%zu) -> %d",

                total_nread, pkts, calls, (int)result);

  return result;

}

#endif /* !HAVE_SENDMMSG && !HAVE_SENDMSG */

CURLcode Curl_vquic_recv_packets(struct Curl_cfilter *cf,

                                 struct Curl_easy *data,

                                 struct cf_quic_ctx *qctx,

                                 size_t max_pkts,

                                 Curl_vquic_recv_pkts_cb *recv_cb, void *userp)

{

  CURLcode result;

#ifdef HAVE_SENDMMSG

  result = recvmmsg_packets(cf, data, qctx, max_pkts, recv_cb, userp);

#elif defined(HAVE_APPLE_MSG_X)

  result = recvmsg_x_packets(cf, data, qctx, max_pkts, recv_cb, userp);

#elif defined(HAVE_SENDMSG)

  result = recvmsg_packets(cf, data, qctx, max_pkts, recv_cb, userp);

#else

  result = recvfrom_packets(cf, data, qctx, max_pkts, recv_cb, userp);

#endif

  if(!result) {

    if(!qctx->got_first_byte) {

      qctx->got_first_byte = TRUE;

      qctx->first_byte_at = qctx->last_op;

    }

    qctx->last_io = qctx->last_op;

  }

  return result;

}

/*

 * If the QLOGDIR environment variable is set, open and return a file

 * descriptor to write the log to.

 *

 * This function returns error if something failed outside of failing to

 * create the file. Open file success is deemed by seeing if the returned fd

 * is != -1.

 */

CURLcode Curl_qlogdir(struct Curl_easy *data,

                      unsigned char *scid,

                      size_t scidlen,

                      int *qlogfdp)

{

  char *qlog_dir = curl_getenv("QLOGDIR");

  *qlogfdp = -1;

  if(qlog_dir) {

    struct dynbuf fname;

    CURLcode result;

    unsigned int i;

    curlx_dyn_init(&fname, DYN_QLOG_NAME);

    result = curlx_dyn_add(&fname, qlog_dir);

    if(!result)

      result = curlx_dyn_add(&fname, "/");

    for(i = 0; (i < scidlen) && !result; i++) {

      char hex[3];

      curl_msnprintf(hex, 3, "%02x", scid[i]);

      result = curlx_dyn_add(&fname, hex);

    }

    if(!result)

      result = curlx_dyn_add(&fname, ".sqlog");

    if(!result) {

      int qlogfd = curlx_open(curlx_dyn_ptr(&fname),

                              O_WRONLY | O_CREAT | CURL_O_BINARY,

                              data->set.new_file_perms

#ifdef _WIN32

                              & (_S_IREAD | _S_IWRITE)

#endif

                              );

      if(qlogfd != -1)

        *qlogfdp = qlogfd;

    }

    curlx_dyn_free(&fname);

    curlx_free(qlog_dir);

    if(result)

      return result;

  }

  return CURLE_OK;

}

CURLcode Curl_cf_quic_insert_after(struct Curl_cfilter *cf_at,

                                   struct Curl_easy *data,

                                   struct Curl_peer *origin,

                                   struct Curl_peer *peer)

{

  CURLcode result;

  (void)data; /* not used in all cases and compilers are stupid */

#if defined(USE_NGTCP2) && defined(USE_NGHTTP3)

  result = Curl_cf_ngtcp2_insert_after(cf_at, origin, peer);

#elif defined(USE_QUICHE)

  result = Curl_cf_quiche_insert_after(cf_at, origin, peer);

#else

  (void)cf_at;

  (void)origin;

  (void)peer;

  result = CURLE_NOT_BUILT_IN;

#endif

#if defined(USE_HTTPSRR) && defined(USE_ECH)

  /* When using ECH, kick off the HTTPS-RR resolve */

  if(!result && (origin->scheme->family == CURLPROTO_HTTP) &&

     CURLECH_ENABLED(data) &&

     Curl_ssl_supports(data, SSLSUPP_ECH) &&

     (data->set.tls_ech != CURLECH_GREASE) &&

     !CURL_EASY_STR(data, STRING_ECH_CONFIG)) {

    result = Curl_conn_dns_add_https_resolve(data, cf_at->conn,

                                             cf_at->sockindex, origin);

  }

#endif /* USE_HTTPSRR && USE_ECH */

  return result;

}

CURLcode Curl_cf_quic_create(struct Curl_cfilter **pcf,

                             struct Curl_easy *data,

                             struct Curl_peer *origin,

                             struct Curl_peer *peer,

                             uint8_t transport_peer,

                             struct connectdata *conn,

                             struct Curl_sockaddr_ex *addr,

                             struct Curl_peer *tunnel_peer,

                             uint8_t tunnel_transport)

{

  (void)transport_peer;

  (void)tunnel_transport;

  (void)tunnel_peer;

  DEBUGASSERT(transport_peer == TRNSPRT_QUIC);

#if defined(USE_NGTCP2) && defined(USE_NGHTTP3)

  return Curl_cf_ngtcp2_create(pcf, data, origin, peer, conn, addr);

#elif defined(USE_QUICHE)

  return Curl_cf_quiche_create(pcf, data, origin, peer, conn, addr);

#else

  *pcf = NULL;

  (void)data;

  (void)origin;

  (void)peer;

  (void)conn;

  (void)addr;

  (void)tunnel_peer;

  (void)tunnel_transport;

  return CURLE_NOT_BUILT_IN;

#endif

}