/***************************************************************************

 *                                  _   _ ____  _

 *  Project                     ___| | | |  _ \| |

 *                             / __| | | | |_) | |

 *                            | (__| |_| |  _ <| |___

 *                             \___|\___/|_| \_\_____|

 *

 * Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al.

 *

 * This software is licensed as described in the file COPYING, which

 * you should have received as part of this distribution. The terms

 * are also available at https://curl.se/docs/copyright.html.

 *

 * You may opt to use, copy, modify, merge, publish, distribute and/or sell

 * copies of the Software, and permit persons to whom the Software is

 * furnished to do so, under the terms of the COPYING file.

 *

 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY

 * KIND, either express or implied.

 *

 * SPDX-License-Identifier: curl

 *

 ***************************************************************************/

#include "curl_setup.h"

#include <curl/curl.h>

#include "urldata.h"

#include "transfer.h"

#include "url.h"

#include "cfilters.h"

#include "connect.h"

#include "progress.h"

#include "easyif.h"

#include "share.h"

#include "psl.h"

#include "multiif.h"

#include "multi_ev.h"

#include "sendf.h"

#include "curlx/timeval.h"

#include "http.h"

#include "select.h"

#include "curlx/warnless.h"

#include "curlx/wait.h"

#include "speedcheck.h"

#include "conncache.h"

#include "multihandle.h"

#include "sigpipe.h"

#include "vtls/vtls.h"

#include "vtls/vtls_scache.h"

#include "http_proxy.h"

#include "http2.h"

#include "socketpair.h"

#include "socks.h"

#include "urlapi-int.h"

/* The last 3 #include files should be in this order */

#include "curl_printf.h"

#include "curl_memory.h"

#include "memdebug.h"

/* initial multi->xfers table size for a full multi */

#define CURL_XFER_TABLE_SIZE    512

/*

  CURL_SOCKET_HASH_TABLE_SIZE should be a prime number. Increasing it from 97

  to 911 takes on a 32-bit machine 4 x 804 = 3211 more bytes. Still, every

  curl handle takes 6K memory, therefore this 3K are not significant.

*/

#ifndef CURL_SOCKET_HASH_TABLE_SIZE

#define CURL_SOCKET_HASH_TABLE_SIZE 911

#endif

#ifndef CURL_CONNECTION_HASH_SIZE

#define CURL_CONNECTION_HASH_SIZE 97

#endif

#ifndef CURL_DNS_HASH_SIZE

#define CURL_DNS_HASH_SIZE 71

#endif

#ifndef CURL_TLS_SESSION_SIZE

#define CURL_TLS_SESSION_SIZE 25

#endif

#define CURL_MULTI_HANDLE 0x000bab1e

#ifdef DEBUGBUILD

/* On a debug build, we want to fail hard on multi handles that

 * are not NULL, but no longer have the MAGIC touch. This gives

 * us early warning on things only discovered by valgrind otherwise. */

#define GOOD_MULTI_HANDLE(x) \

  (((x) && (x)->magic == CURL_MULTI_HANDLE)? TRUE:      \

  (DEBUGASSERT(!(x)), FALSE))

#else

#define GOOD_MULTI_HANDLE(x) \

  ((x) && (x)->magic == CURL_MULTI_HANDLE)

#endif

static void move_pending_to_connect(struct Curl_multi *multi,

                                    struct Curl_easy *data);

static CURLMcode add_next_timeout(struct curltime now,

                                  struct Curl_multi *multi,

                                  struct Curl_easy *d);

static CURLMcode multi_timeout(struct Curl_multi *multi,

                               struct curltime *expire_time,

                               long *timeout_ms);

static void process_pending_handles(struct Curl_multi *multi);

static void multi_xfer_bufs_free(struct Curl_multi *multi);

#ifdef DEBUGBUILD

static void multi_xfer_tbl_dump(struct Curl_multi *multi);

#endif

/* function pointer called once when switching TO a state */

typedef void (*init_multistate_func)(struct Curl_easy *data);

/* called in DID state, before PERFORMING state */

static void before_perform(struct Curl_easy *data)

{

  data->req.chunk = FALSE;

  Curl_pgrsTime(data, TIMER_PRETRANSFER);

}

static void init_completed(struct Curl_easy *data)

{

  /* this is a completed transfer */

  /* Important: reset the conn pointer so that we do not point to memory

     that could be freed anytime */

  Curl_detach_connection(data);

  Curl_expire_clear(data); /* stop all timers */

}

/* always use this function to change state, to make debugging easier */

static void mstate(struct Curl_easy *data, CURLMstate state

#ifdef DEBUGBUILD

                   , int lineno

#endif

)

{

  CURLMstate oldstate = data->mstate;

  static const init_multistate_func finit[MSTATE_LAST] = {

    NULL,              /* INIT */

    NULL,              /* PENDING */

    NULL,              /* SETUP */

    Curl_init_CONNECT, /* CONNECT */

    NULL,              /* RESOLVING */

    NULL,              /* CONNECTING */

    NULL,              /* TUNNELING */

    NULL,              /* PROTOCONNECT */

    NULL,              /* PROTOCONNECTING */

    NULL,              /* DO */

    NULL,              /* DOING */

    NULL,              /* DOING_MORE */

    before_perform,    /* DID */

    NULL,              /* PERFORMING */

    NULL,              /* RATELIMITING */

    NULL,              /* DONE */

    init_completed,    /* COMPLETED */

    NULL               /* MSGSENT */

  };

  if(oldstate == state)

    /* do not bother when the new state is the same as the old state */

    return;

#ifdef DEBUGBUILD

  CURL_TRC_M(data, "-> [%s] (line %d)", CURL_MSTATE_NAME(state), lineno);

#else

  CURL_TRC_M(data, "-> [%s]", CURL_MSTATE_NAME(state));

#endif

  data->mstate = state;

  if(state == MSTATE_COMPLETED) {

    /* changing to COMPLETED means it is in process and needs to go */

    DEBUGASSERT(Curl_uint_bset_contains(&data->multi->process, data->mid));

    Curl_uint_bset_remove(&data->multi->process, data->mid);

    Curl_uint_bset_remove(&data->multi->pending, data->mid); /* to be sure */

    if(Curl_uint_bset_empty(&data->multi->process)) {

      /* free the transfer buffer when we have no more active transfers */

      multi_xfer_bufs_free(data->multi);

    }

  }

  /* if this state has an init-function, run it */

  if(finit[state])

    finit[state](data);

}

#ifndef DEBUGBUILD

#define multistate(x,y) mstate(x,y)

#else

#define multistate(x,y) mstate(x,y, __LINE__)

#endif

/* multi->proto_hash destructor. Should never be called as elements

 * MUST be added with their own destructor */

static void ph_freeentry(void *p)

{

  (void)p;

  /* Will always be FALSE. Cannot use a 0 assert here since compilers

   * are not in agreement if they then want a NORETURN attribute or

   * not. *sigh* */

  DEBUGASSERT(p == NULL);

}

/*

 * multi_addmsg()

 *

 * Called when a transfer is completed. Adds the given msg pointer to

 * the list kept in the multi handle.

 */

static void multi_addmsg(struct Curl_multi *multi, struct Curl_message *msg)

{

  Curl_llist_append(&multi->msglist, msg, &msg->list);

}

struct Curl_multi *Curl_multi_handle(unsigned int xfer_table_size,

                                     size_t ev_hashsize,  /* event hash */

                                     size_t chashsize, /* connection hash */

                                     size_t dnssize,   /* dns hash */

                                     size_t sesssize)  /* TLS session cache */

{

  struct Curl_multi *multi = calloc(1, sizeof(struct Curl_multi));

  if(!multi)

    return NULL;

  multi->magic = CURL_MULTI_HANDLE;

  Curl_dnscache_init(&multi->dnscache, dnssize);

  Curl_multi_ev_init(multi, ev_hashsize);

  Curl_uint_tbl_init(&multi->xfers, NULL);

  Curl_uint_bset_init(&multi->process);

  Curl_uint_bset_init(&multi->dirty);

  Curl_uint_bset_init(&multi->pending);

  Curl_uint_bset_init(&multi->msgsent);

  Curl_hash_init(&multi->proto_hash, 23,

                 Curl_hash_str, curlx_str_key_compare, ph_freeentry);

  Curl_llist_init(&multi->msglist, NULL);

  multi->multiplexing = TRUE;

  multi->max_concurrent_streams = 100;

  multi->last_timeout_ms = -1;

  if(Curl_uint_bset_resize(&multi->process, xfer_table_size) ||

     Curl_uint_bset_resize(&multi->pending, xfer_table_size) ||

     Curl_uint_bset_resize(&multi->dirty, xfer_table_size) ||

     Curl_uint_bset_resize(&multi->msgsent, xfer_table_size) ||

     Curl_uint_tbl_resize(&multi->xfers, xfer_table_size))

    goto error;

  multi->admin = curl_easy_init();

  if(!multi->admin)

    goto error;

  /* Initialize admin handle to operate inside this multi */

  multi->admin->multi = multi;

  multi->admin->state.internal = TRUE;

  Curl_llist_init(&multi->admin->state.timeoutlist, NULL);

#ifdef DEBUGBUILD

  if(getenv("CURL_DEBUG"))

    multi->admin->set.verbose = TRUE;

#endif

  Curl_uint_tbl_add(&multi->xfers, multi->admin, &multi->admin->mid);

  if(Curl_cshutdn_init(&multi->cshutdn, multi))

    goto error;

  Curl_cpool_init(&multi->cpool, multi->admin, NULL, chashsize);

#ifdef USE_SSL

  if(Curl_ssl_scache_create(sesssize, 2, &multi->ssl_scache))

    goto error;

#else

  (void)sesssize;

#endif

#ifdef USE_WINSOCK

  multi->wsa_event = WSACreateEvent();

  if(multi->wsa_event == WSA_INVALID_EVENT)

    goto error;

#elif defined(ENABLE_WAKEUP)

  if(wakeup_create(multi->wakeup_pair, TRUE) < 0) {

    multi->wakeup_pair[0] = CURL_SOCKET_BAD;

    multi->wakeup_pair[1] = CURL_SOCKET_BAD;

  }

#endif

  return multi;

error:

  Curl_multi_ev_cleanup(multi);

  Curl_hash_destroy(&multi->proto_hash);

  Curl_dnscache_destroy(&multi->dnscache);

  Curl_cpool_destroy(&multi->cpool);

  Curl_cshutdn_destroy(&multi->cshutdn, multi->admin);

#ifdef USE_SSL

  Curl_ssl_scache_destroy(multi->ssl_scache);

#endif

  if(multi->admin) {

    multi->admin->multi = NULL;

    Curl_close(&multi->admin);

  }

  Curl_uint_bset_destroy(&multi->process);

  Curl_uint_bset_destroy(&multi->dirty);

  Curl_uint_bset_destroy(&multi->pending);

  Curl_uint_bset_destroy(&multi->msgsent);

  Curl_uint_tbl_destroy(&multi->xfers);

  free(multi);

  return NULL;

}

CURLM *curl_multi_init(void)

{

  return Curl_multi_handle(CURL_XFER_TABLE_SIZE,

                           CURL_SOCKET_HASH_TABLE_SIZE,

                           CURL_CONNECTION_HASH_SIZE,

                           CURL_DNS_HASH_SIZE,

                           CURL_TLS_SESSION_SIZE);

}

#if defined(DEBUGBUILD) && !defined(CURL_DISABLE_VERBOSE_STRINGS)

static void multi_warn_debug(struct Curl_multi *multi, struct Curl_easy *data)

{

  if(!multi->warned) {

    infof(data, "!!! WARNING !!!");

    infof(data, "This is a debug build of libcurl, "

          "do not use in production.");

    multi->warned = TRUE;

  }

}

#else

#define multi_warn_debug(x,y) Curl_nop_stmt

#endif

static CURLMcode multi_xfers_add(struct Curl_multi *multi,

                                 struct Curl_easy *data)

{

  unsigned int capacity = Curl_uint_tbl_capacity(&multi->xfers);

  unsigned int new_size = 0;

  /* Prepare to make this into a CURLMOPT_MAX_TRANSFERS, because some

   * applications may want to prevent a run-away of their memory use. */

  /* UINT_MAX is our "invalid" id, do not let the table grow up to that. */

  const unsigned int max_capacity = UINT_MAX - 1;

  if(capacity < max_capacity) {

    /* We want `multi->xfers` to have "sufficient" free rows, so that we do

     * have to reuse the `mid` from a just removed easy right away.

     * Since uint_tbl and uint_bset are quite memory efficient,

     * regard less than 25% free as insufficient.

     * (for low capacities, e.g. multi_easy, 4 or less). */

    unsigned int used = Curl_uint_tbl_count(&multi->xfers);

    unsigned int unused = capacity - used;

    unsigned int min_unused = CURLMAX(capacity >> 2, 4);

    if(unused <= min_unused) {

      /* Make sure the uint arithmetic here works on the corner

       * cases where we are close to max_capacity or UINT_MAX */

      if((min_unused >= max_capacity) ||

         ((max_capacity - min_unused) <= capacity) ||

         ((UINT_MAX - min_unused - 63) <= capacity)) {

        new_size = max_capacity; /* can not be larger than this */

      }

      else {

         /* make it a 64 multiple, since our bitsets frow by that and

          * small (easy_multi) grows to at least 64 on first resize. */

        new_size = (((used + min_unused) + 63) / 64) * 64;

      }

    }

  }

  if(new_size > capacity) {

    /* Grow the bitsets first. Should one fail, we do not need

     * to downsize the already resized ones. The sets continue

     * to work properly when larger than the table, but not

     * the other way around. */

    CURL_TRC_M(data, "increasing xfer table size to %u", new_size);

    if(Curl_uint_bset_resize(&multi->process, new_size) ||

       Curl_uint_bset_resize(&multi->dirty, new_size) ||

       Curl_uint_bset_resize(&multi->pending, new_size) ||

       Curl_uint_bset_resize(&multi->msgsent, new_size) ||

       Curl_uint_tbl_resize(&multi->xfers, new_size))

      return CURLM_OUT_OF_MEMORY;

  }

  /* Insert the easy into the table now */

  if(!Curl_uint_tbl_add(&multi->xfers, data, &data->mid)) {

    /* MUST only happen when table is full */

    DEBUGASSERT(Curl_uint_tbl_capacity(&multi->xfers) <=

                Curl_uint_tbl_count(&multi->xfers));

    return CURLM_OUT_OF_MEMORY;

  }

  return CURLM_OK;

}

CURLMcode curl_multi_add_handle(CURLM *m, CURL *d)

{

  CURLMcode rc;

  struct Curl_multi *multi = m;

  struct Curl_easy *data = d;

  /* First, make some basic checks that the CURLM handle is a good handle */

  if(!GOOD_MULTI_HANDLE(multi))

    return CURLM_BAD_HANDLE;

  /* Verify that we got a somewhat good easy handle too */

  if(!GOOD_EASY_HANDLE(data))

    return CURLM_BAD_EASY_HANDLE;

  /* Prevent users from adding same easy handle more than once and prevent

     adding to more than one multi stack */

  if(data->multi)

    return CURLM_ADDED_ALREADY;

  if(multi->in_callback)

    return CURLM_RECURSIVE_API_CALL;

  if(multi->dead) {

    /* a "dead" handle cannot get added transfers while any existing easy

       handles are still alive - but if there are none alive anymore, it is

       fine to start over and unmark the "deadness" of this handle.

       This means only the admin handle MUST be present. */

    if((Curl_uint_tbl_count(&multi->xfers) != 1) ||

       !Curl_uint_tbl_contains(&multi->xfers, 0))

      return CURLM_ABORTED_BY_CALLBACK;

    multi->dead = FALSE;

    Curl_uint_bset_clear(&multi->process);

    Curl_uint_bset_clear(&multi->dirty);

    Curl_uint_bset_clear(&multi->pending);

    Curl_uint_bset_clear(&multi->msgsent);

  }

  if(data->multi_easy) {

    /* if this easy handle was previously used for curl_easy_perform(), there

       is a private multi handle here that we can kill */

    curl_multi_cleanup(data->multi_easy);

    data->multi_easy = NULL;

  }

  /* Insert the easy into the multi->xfers table, assigning it a `mid`. */

  if(multi_xfers_add(multi, data))

    return CURLM_OUT_OF_MEMORY;

  /* Initialize timeout list for this handle */

  Curl_llist_init(&data->state.timeoutlist, NULL);

  /*

   * No failure allowed in this function beyond this point. No modification of

   * easy nor multi handle allowed before this except for potential multi's

   * connection pool growing which will not be undone in this function no

   * matter what.

   */

  if(data->set.errorbuffer)

    data->set.errorbuffer[0] = 0;

  data->state.os_errno = 0;

  /* make the Curl_easy refer back to this multi handle - before Curl_expire()

     is called. */

  data->multi = multi;

  /* set the easy handle */

  multistate(data, MSTATE_INIT);

#ifdef USE_LIBPSL

  /* Do the same for PSL. */

  if(data->share && (data->share->specifier & (1 << CURL_LOCK_DATA_PSL)))

    data->psl = &data->share->psl;

  else

    data->psl = &multi->psl;

#endif

  /* add the easy handle to the process set */

  Curl_uint_bset_add(&multi->process, data->mid);

  ++multi->xfers_alive;

  ++multi->xfers_total_ever;

  Curl_cpool_xfer_init(data);

  multi_warn_debug(multi, data);

  /* Make sure the new handle will run */

  Curl_multi_mark_dirty(data);

  /* Necessary in event based processing, where dirty handles trigger

   * a timeout callback invocation. */

  rc = Curl_update_timer(multi);

  if(rc) {

    data->multi = NULL; /* not anymore */

    Curl_uint_tbl_remove(&multi->xfers, data->mid);

    data->mid = UINT_MAX;

    return rc;

  }

  /* The admin handle only ever has default timeouts set. To improve the

     state somewhat we clone the timeouts from each added handle so that the

     admin handle always has the same timeouts as the most recently added

     easy handle. */

  multi->admin->set.timeout = data->set.timeout;

  multi->admin->set.server_response_timeout =

    data->set.server_response_timeout;

  multi->admin->set.no_signal = data->set.no_signal;

  CURL_TRC_M(data, "added to multi, mid=%u, running=%u, total=%u",

             data->mid, Curl_multi_xfers_running(multi),

             Curl_uint_tbl_count(&multi->xfers));

  return CURLM_OK;

}

#if 0

/* Debug-function, used like this:

 *

 * Curl_hash_print(&multi->sockhash, debug_print_sock_hash);

 *

 * Enable the hash print function first by editing hash.c

 */

static void debug_print_sock_hash(void *p)

{

  struct Curl_sh_entry *sh = (struct Curl_sh_entry *)p;

  fprintf(stderr, " [readers %u][writers %u]",

          sh->readers, sh->writers);

}

#endif

struct multi_done_ctx {

  BIT(premature);

};

static void multi_done_locked(struct connectdata *conn,

                              struct Curl_easy *data,

                              void *userdata)

{

  struct multi_done_ctx *mdctx = userdata;

#ifndef CURL_DISABLE_VERBOSE_STRINGS

  const char *host =

#ifndef CURL_DISABLE_PROXY

        conn->bits.socksproxy ?

        conn->socks_proxy.host.dispname :

        conn->bits.httpproxy ? conn->http_proxy.host.dispname :

#endif

        conn->bits.conn_to_host ? conn->conn_to_host.dispname :

        conn->host.dispname;

  int port =

#ifndef CURL_DISABLE_PROXY

        conn->bits.httpproxy ? conn->http_proxy.port :

#endif

        conn->bits.conn_to_port ? conn->conn_to_port :

        conn->remote_port;

#endif

  Curl_detach_connection(data);

  CURL_TRC_M(data, "multi_done_locked, in use=%u",

             Curl_uint_spbset_count(&conn->xfers_attached));

  if(CONN_INUSE(conn)) {

    /* Stop if still used. */

    CURL_TRC_M(data, "Connection still in use %u, no more multi_done now!",

               Curl_uint_spbset_count(&conn->xfers_attached));

    return;

  }

  data->state.done = TRUE; /* called just now! */

  data->state.recent_conn_id = conn->connection_id;

  Curl_resolv_unlink(data, &data->state.dns[0]); /* done with this */

  Curl_resolv_unlink(data, &data->state.dns[1]);

  Curl_dnscache_prune(data);

  /* if data->set.reuse_forbid is TRUE, it means the libcurl client has

     forced us to close this connection. This is ignored for requests taking

     place in a NTLM/NEGOTIATE authentication handshake

     if conn->bits.close is TRUE, it means that the connection should be

     closed in spite of all our efforts to be nice, due to protocol

     restrictions in our or the server's end

     if premature is TRUE, it means this connection was said to be DONE before

     the entire request operation is complete and thus we cannot know in what

     state it is for reusing, so we are forced to close it. In a perfect world

     we can add code that keep track of if we really must close it here or not,

     but currently we have no such detail knowledge.

  */

  if((data->set.reuse_forbid

#ifdef USE_NTLM

      && !(conn->http_ntlm_state == NTLMSTATE_TYPE2 ||

           conn->proxy_ntlm_state == NTLMSTATE_TYPE2)

#endif

#ifdef USE_SPNEGO

      && !(conn->http_negotiate_state == GSS_AUTHRECV ||

           conn->proxy_negotiate_state == GSS_AUTHRECV)

#endif

     ) || conn->bits.close

       || (mdctx->premature && !Curl_conn_is_multiplex(conn, FIRSTSOCKET))) {

#ifndef CURL_DISABLE_VERBOSE_STRINGS

    CURL_TRC_M(data, "multi_done, terminating conn #%" FMT_OFF_T " to %s:%d, "

               "forbid=%d, close=%d, premature=%d, conn_multiplex=%d",

               conn->connection_id, host, port, data->set.reuse_forbid,

               conn->bits.close, mdctx->premature,

               Curl_conn_is_multiplex(conn, FIRSTSOCKET));

#endif

    connclose(conn, "disconnecting");

    Curl_conn_terminate(data, conn, mdctx->premature);

  }

  else if(!Curl_conn_get_max_concurrent(data, conn, FIRSTSOCKET)) {

#ifndef CURL_DISABLE_VERBOSE_STRINGS

    CURL_TRC_M(data, "multi_done, conn #%" FMT_OFF_T " to %s:%d was shutdown"

               " by server, not reusing", conn->connection_id, host, port);

#endif

    connclose(conn, "server shutdown");

    Curl_conn_terminate(data, conn, mdctx->premature);

  }

  else {

    /* the connection is no longer in use by any transfer */

    if(Curl_cpool_conn_now_idle(data, conn)) {

      /* connection kept in the cpool */

      data->state.lastconnect_id = conn->connection_id;

#ifndef CURL_DISABLE_VERBOSE_STRINGS

      infof(data, "Connection #%" FMT_OFF_T " to host %s:%d left intact",

            conn->connection_id, host, port);

#endif

    }

    else {

      /* connection was removed from the cpool and destroyed. */

      data->state.lastconnect_id = -1;

    }

  }

}

static CURLcode multi_done(struct Curl_easy *data,

                           CURLcode status,  /* an error if this is called

                                                after an error was detected */

                           bool premature)

{

  CURLcode result;

  struct connectdata *conn = data->conn;

  struct multi_done_ctx mdctx;

  memset(&mdctx, 0, sizeof(mdctx));

  CURL_TRC_M(data, "multi_done: status: %d prem: %d done: %d",

             (int)status, (int)premature, data->state.done);

  if(data->state.done)

    /* Stop if multi_done() has already been called */

    return CURLE_OK;

  /* Shut down any ongoing async resolver operation. */

  Curl_async_shutdown(data);

  /* Cleanup possible redirect junk */

  Curl_safefree(data->req.newurl);

  Curl_safefree(data->req.location);

  switch(status) {

  case CURLE_ABORTED_BY_CALLBACK:

  case CURLE_READ_ERROR:

  case CURLE_WRITE_ERROR:

    /* When we are aborted due to a callback return code it basically have to

       be counted as premature as there is trouble ahead if we do not. We have

       many callbacks and protocols work differently, we could potentially do

       this more fine-grained in the future. */

    premature = TRUE;

    FALLTHROUGH();

  default:

    break;

  }

  /* this calls the protocol-specific function pointer previously set */

  if(conn->handler->done && (data->mstate >= MSTATE_PROTOCONNECT))

    result = conn->handler->done(data, status, premature);

  else

    result = status;

  if(CURLE_ABORTED_BY_CALLBACK != result) {

    /* avoid this if we already aborted by callback to avoid this calling

       another callback */

    int rc = Curl_pgrsDone(data);

    if(!result && rc)

      result = CURLE_ABORTED_BY_CALLBACK;

  }

  /* Make sure that transfer client writes are really done now. */

  result = Curl_1st_err(result, Curl_xfer_write_done(data, premature));

  /* Inform connection filters that this transfer is done */

  Curl_conn_ev_data_done(data, premature);

  process_pending_handles(data->multi); /* connection / multiplex */

  if(!result)

    result = Curl_req_done(&data->req, data, premature);

  /* Under the potential connection pool's share lock, decide what to

   * do with the transfer's connection. */

  mdctx.premature = premature;

  Curl_cpool_do_locked(data, data->conn, multi_done_locked, &mdctx);

  /* flush the netrc cache */

  Curl_netrc_cleanup(&data->state.netrc);

  return result;

}

static void close_connect_only(struct connectdata *conn,

                               struct Curl_easy *data,

                               void *userdata)

{

  (void)userdata;

  (void)data;

  if(conn->connect_only)

    connclose(conn, "Removing connect-only easy handle");

}

CURLMcode curl_multi_remove_handle(CURLM *m, CURL *d)

{

  struct Curl_multi *multi = m;

  struct Curl_easy *data = d;

  bool premature;

  struct Curl_llist_node *e;

  CURLMcode rc;

  bool removed_timer = FALSE;

  unsigned int mid;

  /* First, make some basic checks that the CURLM handle is a good handle */

  if(!GOOD_MULTI_HANDLE(multi))

    return CURLM_BAD_HANDLE;

  /* Verify that we got a somewhat good easy handle too */

  if(!GOOD_EASY_HANDLE(data))

    return CURLM_BAD_EASY_HANDLE;

  /* Prevent users from trying to remove same easy handle more than once */

  if(!data->multi)

    return CURLM_OK; /* it is already removed so let's say it is fine! */

  /* Prevent users from trying to remove an easy handle from the wrong multi */

  if(data->multi != multi)

    return CURLM_BAD_EASY_HANDLE;

  if(data->mid == UINT_MAX) {

    DEBUGASSERT(0);

    return CURLM_INTERNAL_ERROR;

  }

  if(Curl_uint_tbl_get(&multi->xfers, data->mid) != data) {

    DEBUGASSERT(0);

    return CURLM_INTERNAL_ERROR;

  }

  if(multi->in_callback)

    return CURLM_RECURSIVE_API_CALL;

  premature = (data->mstate < MSTATE_COMPLETED);

  /* If the 'state' is not INIT or COMPLETED, we might need to do something

     nice to put the easy_handle in a good known state when this returns. */

  if(data->conn &&

     data->mstate > MSTATE_DO &&

     data->mstate < MSTATE_COMPLETED) {

    /* Set connection owner so that the DONE function closes it. We can

       safely do this here since connection is killed. */

    streamclose(data->conn, "Removed with partial response");

  }

  if(data->conn) {

    /* multi_done() clears the association between the easy handle and the

       connection.

       Note that this ignores the return code simply because there is

       nothing really useful to do with it anyway! */

    (void)multi_done(data, data->result, premature);

  }

  /* The timer must be shut down before data->multi is set to NULL, else the

     timenode will remain in the splay tree after curl_easy_cleanup is

     called. Do it after multi_done() in case that sets another time! */

  removed_timer = Curl_expire_clear(data);

  /* If in `msgsent`, it was deducted from `multi->xfers_alive` already. */

  if(!Curl_uint_bset_contains(&multi->msgsent, data->mid))

    --multi->xfers_alive;

  Curl_wildcard_dtor(&data->wildcard);

  data->mstate = MSTATE_COMPLETED;

  /* Remove the association between the connection and the handle */

  Curl_detach_connection(data);

  /* Tell event handling that this transfer is definitely going away */

  Curl_multi_ev_xfer_done(multi, data);

  if(data->set.connect_only && !data->multi_easy) {

    /* This removes a handle that was part the multi interface that used

       CONNECT_ONLY, that connection is now left alive but since this handle

       has bits.close set nothing can use that transfer anymore and it is

       forbidden from reuse. This easy handle cannot find the connection

       anymore once removed from the multi handle

       Better close the connection here, at once.

    */

    struct connectdata *c;

    curl_socket_t s;

    s = Curl_getconnectinfo(data, &c);

    if((s != CURL_SOCKET_BAD) && c) {

      Curl_conn_terminate(data, c, TRUE);

    }

  }

  if(data->state.lastconnect_id != -1) {

    /* Mark any connect-only connection for closure */

    Curl_cpool_do_by_id(data, data->state.lastconnect_id,

                            close_connect_only, NULL);

  }

#ifdef USE_LIBPSL

  /* Remove the PSL association. */

  if(data->psl == &multi->psl)

    data->psl = NULL;

#endif

  /* make sure there is no pending message in the queue sent from this easy

     handle */

  for(e = Curl_llist_head(&multi->msglist); e; e = Curl_node_next(e)) {

    struct Curl_message *msg = Curl_node_elem(e);

    if(msg->extmsg.easy_handle == data) {

      Curl_node_remove(e);

      /* there can only be one from this specific handle */

      break;

    }

  }

  /* clear the association to this multi handle */

  mid = data->mid;

  DEBUGASSERT(Curl_uint_tbl_contains(&multi->xfers, mid));

  Curl_uint_tbl_remove(&multi->xfers, mid);

  Curl_uint_bset_remove(&multi->process, mid);

  Curl_uint_bset_remove(&multi->dirty, mid);

  Curl_uint_bset_remove(&multi->pending, mid);

  Curl_uint_bset_remove(&multi->msgsent, mid);

  data->multi = NULL;

  data->mid = UINT_MAX;

  data->master_mid = UINT_MAX;

  /* NOTE NOTE NOTE

     We do not touch the easy handle here! */

  process_pending_handles(multi);

  if(removed_timer) {

    rc = Curl_update_timer(multi);

    if(rc)

      return rc;

  }

  CURL_TRC_M(data, "removed from multi, mid=%u, running=%u, total=%u",

             mid, Curl_multi_xfers_running(multi),

             Curl_uint_tbl_count(&multi->xfers));

  return CURLM_OK;

}

/* Return TRUE if the application asked for multiplexing */

bool Curl_multiplex_wanted(const struct Curl_multi *multi)

{

  return multi && multi->multiplexing;

}

/*

 * Curl_detach_connection() removes the given transfer from the connection.

 *

 * This is the only function that should clear data->conn. This will

 * occasionally be called with the data->conn pointer already cleared.

 */

void Curl_detach_connection(struct Curl_easy *data)

{

  struct connectdata *conn = data->conn;

  if(conn) {

    Curl_uint_spbset_remove(&conn->xfers_attached, data->mid);

    if(Curl_uint_spbset_empty(&conn->xfers_attached))

      conn->attached_multi = NULL;

  }

  data->conn = NULL;

}

/*

 * Curl_attach_connection() attaches this transfer to this connection.

 *

 * This is the only function that should assign data->conn

 */

void Curl_attach_connection(struct Curl_easy *data,

                            struct connectdata *conn)

{

  DEBUGASSERT(data);

  DEBUGASSERT(!data->conn);

  DEBUGASSERT(conn);

  data->conn = conn;

  Curl_uint_spbset_add(&conn->xfers_attached, data->mid);

  /* all attached transfers must be from the same multi */

  if(!conn->attached_multi)

    conn->attached_multi = data->multi;

  DEBUGASSERT(conn->attached_multi == data->multi);

  if(conn->handler && conn->handler->attach)

    conn->handler->attach(data, conn);

}

static CURLcode mstate_connecting_pollset(struct Curl_easy *data,

                                          struct easy_pollset *ps)

{

  if(data->conn) {

    curl_socket_t sockfd = Curl_conn_get_first_socket(data);

    if(sockfd != CURL_SOCKET_BAD) {

      /* Default is to wait to something from the server */

      return Curl_pollset_change(data, ps, sockfd, CURL_POLL_IN, 0);

    }

  }

  return CURLE_OK;

}

static CURLcode mstate_protocol_pollset(struct Curl_easy *data,

                                        struct easy_pollset *ps)

{

  if(data->conn) {

    curl_socket_t sockfd;

    if(data->conn->handler->proto_pollset)

      return data->conn->handler->proto_pollset(data, ps);

    sockfd = data->conn->sock[FIRSTSOCKET];

    if(sockfd != CURL_SOCKET_BAD) {

      /* Default is to wait to something from the server */

      return Curl_pollset_change(data, ps, sockfd, CURL_POLL_IN, 0);

    }

  }

  return CURLE_OK;

}

static CURLcode mstate_do_pollset(struct Curl_easy *data,

                                  struct easy_pollset *ps)

{

  if(data->conn) {

    if(data->conn->handler->doing_pollset)

      return data->conn->handler->doing_pollset(data, ps);

    else if(CONN_SOCK_IDX_VALID(data->conn->send_idx)) {

      /* Default is that we want to send something to the server */

      return Curl_pollset_add_out(

        data, ps, data->conn->sock[data->conn->send_idx]);

    }

  }

  return CURLE_OK;

}

static CURLcode mstate_domore_pollset(struct Curl_easy *data,

                                      struct easy_pollset *ps)

{

  if(data->conn) {

    if(data->conn->handler->domore_pollset)

      return data->conn->handler->domore_pollset(data, ps);

    else if(CONN_SOCK_IDX_VALID(data->conn->send_idx)) {

      /* Default is that we want to send something to the server */

      return Curl_pollset_add_out(

        data, ps, data->conn->sock[data->conn->send_idx]);

    }

  }

  return CURLE_OK;

}

static CURLcode mstate_perform_pollset(struct Curl_easy *data,

                                       struct easy_pollset *ps)

{

  if(!data->conn)

    return CURLE_OK;

  else if(data->conn->handler->perform_pollset)

    return data->conn->handler->perform_pollset(data, ps);

  else {

    /* Default is to obey the data->req.keepon flags for send/recv */

    CURLcode result = CURLE_OK;

    if(CURL_WANT_RECV(data) && CONN_SOCK_IDX_VALID(data->conn->recv_idx)) {

      result = Curl_pollset_add_in(

        data, ps, data->conn->sock[data->conn->recv_idx]);

    }

    if(!result && Curl_req_want_send(data) &&

       CONN_SOCK_IDX_VALID(data->conn->send_idx)) {

      result = Curl_pollset_add_out(

        data, ps, data->conn->sock[data->conn->send_idx]);

    }

    return result;

  }

}

/* Initializes `poll_set` with the current socket poll actions needed

 * for transfer `data`. */

CURLMcode Curl_multi_pollset(struct Curl_easy *data,

                             struct easy_pollset *ps,

                             const char *caller)

{

  CURLMcode mresult = CURLM_OK;

  CURLcode result = CURLE_OK;

  bool expect_sockets = TRUE;

  /* If the transfer has no connection, this is fine. Happens when

     called via curl_multi_remove_handle() => Curl_multi_ev_assess() =>

     Curl_multi_pollset(). */

  Curl_pollset_reset(ps);

  if(!data->conn)

    return CURLM_OK;

  switch(data->mstate) {

  case MSTATE_INIT:

  case MSTATE_PENDING:

  case MSTATE_SETUP:

  case MSTATE_CONNECT:

    /* nothing to poll for yet */

    expect_sockets = FALSE;

    break;

  case MSTATE_RESOLVING:

    result = Curl_resolv_pollset(data, ps);

    /* connection filters are not involved in this phase. It's ok if we get no

     * sockets to wait for. Resolving can wake up from other sources. */

    expect_sockets = FALSE;

    break;

  case MSTATE_CONNECTING:

  case MSTATE_TUNNELING:

    if(!Curl_xfer_recv_is_paused(data)) {

      result = mstate_connecting_pollset(data, ps);

      if(!result)