/src/libevent/buffer.c

Source (jump to first uncovered line)
/*
 * Copyright (c) 2002-2007 Niels Provos <provos@citi.umich.edu>
 * Copyright (c) 2007-2012 Niels Provos and Nick Mathewson
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. The name of the author may not be used to endorse or promote products
 *    derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "event2/event-config.h"
#include "evconfig-private.h"

#ifdef _WIN32
#include <winsock2.h>
#include <windows.h>
#include <io.h>
#endif

#include <sys/types.h>

#ifdef EVENT__HAVE_SYS_TIME_H
#include <sys/time.h>
#endif

#ifdef EVENT__HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif

#ifdef EVENT__HAVE_SYS_UIO_H
#include <sys/uio.h>
#endif

#ifdef EVENT__HAVE_SYS_IOCTL_H
#include <sys/ioctl.h>
#endif

#ifdef EVENT__HAVE_SYS_MMAN_H
#include <sys/mman.h>
#endif

#ifdef EVENT__HAVE_SYS_SENDFILE_H
#include <sys/sendfile.h>
#endif
#ifdef EVENT__HAVE_SYS_STAT_H
#include <sys/stat.h>
#endif


#include <errno.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef EVENT__HAVE_STDARG_H
#include <stdarg.h>
#endif
#ifdef EVENT__HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <limits.h>

#include "event2/event.h"
#include "event2/buffer.h"
#include "event2/buffer_compat.h"
#include "event2/bufferevent.h"
#include "event2/bufferevent_compat.h"
#include "event2/bufferevent_struct.h"
#include "event2/thread.h"
#include "log-internal.h"
#include "mm-internal.h"
#include "util-internal.h"
#include "evthread-internal.h"
#include "evbuffer-internal.h"
#include "bufferevent-internal.h"
#include "event-internal.h"

/* some systems do not have MAP_FAILED */
#ifndef MAP_FAILED
#define MAP_FAILED  ((void *)-1)
#endif

/* send file support */
#if defined(EVENT__HAVE_SYS_SENDFILE_H) && defined(EVENT__HAVE_SENDFILE) && defined(__linux__)
#define USE_SENDFILE    1
#define SENDFILE_IS_LINUX 1
#elif defined(EVENT__HAVE_SENDFILE) && defined(__FreeBSD__)
#define USE_SENDFILE    1
#define SENDFILE_IS_FREEBSD 1
#elif defined(EVENT__HAVE_SENDFILE) && defined(__APPLE__)
#define USE_SENDFILE    1
#define SENDFILE_IS_MACOSX  1
#elif defined(EVENT__HAVE_SENDFILE) && defined(__sun__) && defined(__svr4__)
#define USE_SENDFILE    1
#define SENDFILE_IS_SOLARIS 1
#endif

/* Mask of user-selectable callback flags. */
#define EVBUFFER_CB_USER_FLAGS      0xffff
/* Mask of all internal-use-only flags. */
#define EVBUFFER_CB_INTERNAL_FLAGS  0xffff0000

/* Flag set if the callback is using the cb_obsolete function pointer  */
#define EVBUFFER_CB_OBSOLETE         0x00040000

/* evbuffer_chain support */
#define CHAIN_SPACE_PTR(ch) ((ch)->buffer + (ch)->misalign + (ch)->off)
#define CHAIN_SPACE_LEN(ch) ((ch)->flags & EVBUFFER_IMMUTABLE ? \
      0 : (ch)->buffer_len - ((ch)->misalign + (ch)->off))

#define CHAIN_PINNED(ch)  (((ch)->flags & EVBUFFER_MEM_PINNED_ANY) != 0)
#define CHAIN_PINNED_R(ch)  (((ch)->flags & EVBUFFER_MEM_PINNED_R) != 0)

/* evbuffer_ptr support */
#define PTR_NOT_FOUND(ptr) do {     \
  (ptr)->pos = -1;          \
  (ptr)->internal_.chain = NULL;    \
  (ptr)->internal_.pos_in_chain = 0;  \
} while (0)

#define EVBUFFER_MAX_READ_DEFAULT 4096

static void evbuffer_chain_align(struct evbuffer_chain *chain);
static int evbuffer_chain_should_realign(struct evbuffer_chain *chain,
    size_t datalen);
static void evbuffer_deferred_callback(struct event_callback *cb, void *arg);
static int evbuffer_ptr_memcmp(const struct evbuffer *buf,
    const struct evbuffer_ptr *pos, const char *mem, size_t len);
static struct evbuffer_chain *evbuffer_expand_singlechain(struct evbuffer *buf,
    size_t datlen);
static int evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos,
    size_t howfar);
static int evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg);
static inline void evbuffer_chain_incref(struct evbuffer_chain *chain);

static struct evbuffer_chain *
evbuffer_chain_new(size_t size)
{
  struct evbuffer_chain *chain;
  size_t to_alloc;

  if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE)
    return (NULL);

  to_alloc = size + EVBUFFER_CHAIN_SIZE;

  /* we get everything in one chunk */
  if ((chain = mm_malloc(to_alloc)) == NULL)
    return (NULL);

  memset(chain, 0, EVBUFFER_CHAIN_SIZE);

  chain->buffer_len = to_alloc - EVBUFFER_CHAIN_SIZE;

  /* this way we can manipulate the buffer to different addresses,
   * which is required for mmap for example.
   */
  chain->buffer = EVBUFFER_CHAIN_EXTRA(unsigned char, chain);

  chain->refcnt = 1;

  return (chain);
}

static struct evbuffer_chain *
evbuffer_chain_new_membuf(size_t size)
{
  size_t to_alloc;

  if (size > EVBUFFER_CHAIN_MAX - EVBUFFER_CHAIN_SIZE)
    return (NULL);

  size += EVBUFFER_CHAIN_SIZE;

  /* get the next largest memory that can hold the buffer */
  if (size < EVBUFFER_CHAIN_MAX / 2) {
    to_alloc = MIN_BUFFER_SIZE;
    while (to_alloc < size) {
      to_alloc <<= 1;
    }
  } else {
    to_alloc = size;
  }

  return evbuffer_chain_new(to_alloc - EVBUFFER_CHAIN_SIZE);
}

static inline void
evbuffer_chain_free(struct evbuffer_chain *chain)
{
  EVUTIL_ASSERT(chain->refcnt > 0);
  if (--chain->refcnt > 0) {
    /* chain is still referenced by other chains */
    return;
  }

  if (CHAIN_PINNED(chain)) {
    /* will get freed once no longer dangling */
    chain->refcnt++;
    chain->flags |= EVBUFFER_DANGLING;
    return;
  }

  /* safe to release chain, it's either a referencing
   * chain or all references to it have been freed */
  if (chain->flags & EVBUFFER_REFERENCE) {
    struct evbuffer_chain_reference *info =
        EVBUFFER_CHAIN_EXTRA(
          struct evbuffer_chain_reference,
          chain);
    if (info->cleanupfn)
      (*info->cleanupfn)(chain->buffer,
          chain->buffer_len,
          info->extra);
  }
  if (chain->flags & EVBUFFER_FILESEGMENT) {
    struct evbuffer_chain_file_segment *info =
        EVBUFFER_CHAIN_EXTRA(
          struct evbuffer_chain_file_segment,
          chain);
    if (info->segment) {
#ifdef _WIN32
      if (info->segment->is_mapping)
        UnmapViewOfFile(chain->buffer);
#endif
      evbuffer_file_segment_free(info->segment);
    }
  }
  if (chain->flags & EVBUFFER_MULTICAST) {
    struct evbuffer_multicast_parent *info =
        EVBUFFER_CHAIN_EXTRA(
          struct evbuffer_multicast_parent,
          chain);
    /* referencing chain is being freed, decrease
     * refcounts of source chain and associated
     * evbuffer (which get freed once both reach
     * zero) */
    EVUTIL_ASSERT(info->source != NULL);
    EVUTIL_ASSERT(info->parent != NULL);
    EVBUFFER_LOCK(info->source);
    evbuffer_chain_free(info->parent);
    evbuffer_decref_and_unlock_(info->source);
  }

  mm_free(chain);
}

static void
evbuffer_free_all_chains(struct evbuffer_chain *chain)
{
  struct evbuffer_chain *next;
  for (; chain; chain = next) {
    next = chain->next;
    evbuffer_chain_free(chain);
  }
}

#ifndef NDEBUG
static int
evbuffer_chains_all_empty(struct evbuffer_chain *chain)
{
  for (; chain; chain = chain->next) {
    if (chain->off)
      return 0;
  }
  return 1;
}
#else
/* The definition is needed for EVUTIL_ASSERT, which uses sizeof to avoid
"unused variable" warnings. */
static inline int evbuffer_chains_all_empty(struct evbuffer_chain *chain) {
  return 1;
}
#endif

/* Free all trailing chains in 'buf' that are neither pinned nor empty, prior
 * to replacing them all with a new chain.  Return a pointer to the place
 * where the new chain will go.
 *
 * Internal; requires lock.  The caller must fix up buf->last and buf->first
 * as needed; they might have been freed.
 */
static struct evbuffer_chain **
evbuffer_free_trailing_empty_chains(struct evbuffer *buf)
{
  struct evbuffer_chain **ch = buf->last_with_datap;
  /* Find the first victim chain.  It might be *last_with_datap */
  while ((*ch) && ((*ch)->off != 0 || CHAIN_PINNED(*ch)))
    ch = &(*ch)->next;
  if (*ch) {
    EVUTIL_ASSERT(evbuffer_chains_all_empty(*ch));
    evbuffer_free_all_chains(*ch);
    *ch = NULL;
  }
  return ch;
}

/* Add a single chain 'chain' to the end of 'buf', freeing trailing empty
 * chains as necessary.  Requires lock.  Does not schedule callbacks.
 */
static void
evbuffer_chain_insert(struct evbuffer *buf,
    struct evbuffer_chain *chain)
{
  ASSERT_EVBUFFER_LOCKED(buf);
  if (*buf->last_with_datap == NULL) {
    /* There are no chains data on the buffer at all. */
    EVUTIL_ASSERT(buf->last_with_datap == &buf->first);
    EVUTIL_ASSERT(buf->first == NULL);
    buf->first = buf->last = chain;
  } else {
    struct evbuffer_chain **chp;
    chp = evbuffer_free_trailing_empty_chains(buf);
    *chp = chain;
    if (chain->off)
      buf->last_with_datap = chp;
    buf->last = chain;
  }
  buf->total_len += chain->off;
}

static inline struct evbuffer_chain *
evbuffer_chain_insert_new(struct evbuffer *buf, size_t datlen)
{
  struct evbuffer_chain *chain;
  if ((chain = evbuffer_chain_new_membuf(datlen)) == NULL)
    return NULL;
  evbuffer_chain_insert(buf, chain);
  return chain;
}

void
evbuffer_chain_pin_(struct evbuffer_chain *chain, unsigned flag)
{
  EVUTIL_ASSERT((chain->flags & flag) == 0);
  chain->flags |= flag;
}

void
evbuffer_chain_unpin_(struct evbuffer_chain *chain, unsigned flag)
{
  EVUTIL_ASSERT((chain->flags & flag) != 0);
  chain->flags &= ~flag;
  if (chain->flags & EVBUFFER_DANGLING)
    evbuffer_chain_free(chain);
}

static inline void
evbuffer_chain_incref(struct evbuffer_chain *chain)
{
    ++chain->refcnt;
}

struct evbuffer *
evbuffer_new(void)
{
  struct evbuffer *buffer;

  buffer = mm_calloc(1, sizeof(struct evbuffer));
  if (buffer == NULL)
    return (NULL);

  LIST_INIT(&buffer->callbacks);
  buffer->refcnt = 1;
  buffer->last_with_datap = &buffer->first;
  buffer->max_read = EVBUFFER_MAX_READ_DEFAULT;

  return (buffer);
}

int
evbuffer_set_flags(struct evbuffer *buf, ev_uint64_t flags)
{
  EVBUFFER_LOCK(buf);
  buf->flags |= (ev_uint32_t)flags;
  EVBUFFER_UNLOCK(buf);
  return 0;
}

int
evbuffer_clear_flags(struct evbuffer *buf, ev_uint64_t flags)
{
  EVBUFFER_LOCK(buf);
  buf->flags &= ~(ev_uint32_t)flags;
  EVBUFFER_UNLOCK(buf);
  return 0;
}

void
evbuffer_incref_(struct evbuffer *buf)
{
  EVBUFFER_LOCK(buf);
  ++buf->refcnt;
  EVBUFFER_UNLOCK(buf);
}

void
evbuffer_incref_and_lock_(struct evbuffer *buf)
{
  EVBUFFER_LOCK(buf);
  ++buf->refcnt;
}

int
evbuffer_defer_callbacks(struct evbuffer *buffer, struct event_base *base)
{
  EVBUFFER_LOCK(buffer);
  buffer->cb_queue = base;
  buffer->deferred_cbs = 1;
  event_deferred_cb_init_(&buffer->deferred,
      event_base_get_npriorities(base) / 2,
      evbuffer_deferred_callback, buffer);
  EVBUFFER_UNLOCK(buffer);
  return 0;
}

int
evbuffer_enable_locking(struct evbuffer *buf, void *lock)
{
#ifdef EVENT__DISABLE_THREAD_SUPPORT
  return -1;
#else
  if (buf->lock)
    return -1;

  if (!lock) {
    EVTHREAD_ALLOC_LOCK(lock, EVTHREAD_LOCKTYPE_RECURSIVE);
    if (!lock)
      return -1;
    buf->lock = lock;
    buf->own_lock = 1;
  } else {
    buf->lock = lock;
    buf->own_lock = 0;
  }

  return 0;
#endif
}

void
evbuffer_set_parent_(struct evbuffer *buf, struct bufferevent *bev)
{
  EVBUFFER_LOCK(buf);
  buf->parent = bev;
  EVBUFFER_UNLOCK(buf);
}

static void
evbuffer_run_callbacks(struct evbuffer *buffer, int running_deferred)
{
  struct evbuffer_cb_entry *cbent, *next;
  struct evbuffer_cb_info info;
  size_t new_size;
  ev_uint32_t mask, masked_val;
  int clear = 1;

  if (running_deferred) {
    mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
    masked_val = EVBUFFER_CB_ENABLED;
  } else if (buffer->deferred_cbs) {
    mask = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
    masked_val = EVBUFFER_CB_NODEFER|EVBUFFER_CB_ENABLED;
    /* Don't zero-out n_add/n_del, since the deferred callbacks
       will want to see them. */
    clear = 0;
  } else {
    mask = EVBUFFER_CB_ENABLED;
    masked_val = EVBUFFER_CB_ENABLED;
  }

  ASSERT_EVBUFFER_LOCKED(buffer);

  if (LIST_EMPTY(&buffer->callbacks)) {
    buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
    return;
  }
  if (buffer->n_add_for_cb == 0 && buffer->n_del_for_cb == 0)
    return;

  new_size = buffer->total_len;
  info.orig_size = new_size + buffer->n_del_for_cb - buffer->n_add_for_cb;
  info.n_added = buffer->n_add_for_cb;
  info.n_deleted = buffer->n_del_for_cb;
  if (clear) {
    buffer->n_add_for_cb = 0;
    buffer->n_del_for_cb = 0;
  }
  for (cbent = LIST_FIRST(&buffer->callbacks);
       cbent != LIST_END(&buffer->callbacks);
       cbent = next) {
    /* Get the 'next' pointer now in case this callback decides
     * to remove itself or something. */
    next = LIST_NEXT(cbent, next);

    if ((cbent->flags & mask) != masked_val)
      continue;

    if ((cbent->flags & EVBUFFER_CB_OBSOLETE))
      cbent->cb.cb_obsolete(buffer,
          info.orig_size, new_size, cbent->cbarg);
    else
      cbent->cb.cb_func(buffer, &info, cbent->cbarg);
  }
}

void
evbuffer_invoke_callbacks_(struct evbuffer *buffer)
{
  if (LIST_EMPTY(&buffer->callbacks)) {
    buffer->n_add_for_cb = buffer->n_del_for_cb = 0;
    return;
  }

  if (buffer->deferred_cbs) {
    if (event_deferred_cb_schedule_(buffer->cb_queue, &buffer->deferred)) {
      evbuffer_incref_and_lock_(buffer);
      if (buffer->parent)
        bufferevent_incref_(buffer->parent);
      EVBUFFER_UNLOCK(buffer);
    }
  }

  evbuffer_run_callbacks(buffer, 0);
}

static void
evbuffer_deferred_callback(struct event_callback *cb, void *arg)
{
  struct bufferevent *parent = NULL;
  struct evbuffer *buffer = arg;

  /* XXXX It would be better to run these callbacks without holding the
   * lock */
  EVBUFFER_LOCK(buffer);
  parent = buffer->parent;
  evbuffer_run_callbacks(buffer, 1);
  evbuffer_decref_and_unlock_(buffer);
  if (parent)
    bufferevent_decref_(parent);
}

static void
evbuffer_remove_all_callbacks(struct evbuffer *buffer)
{
  struct evbuffer_cb_entry *cbent;

  while ((cbent = LIST_FIRST(&buffer->callbacks))) {
    LIST_REMOVE(cbent, next);
    mm_free(cbent);
  }
}

void
evbuffer_decref_and_unlock_(struct evbuffer *buffer)
{
  struct evbuffer_chain *chain, *next;
  ASSERT_EVBUFFER_LOCKED(buffer);

  EVUTIL_ASSERT(buffer->refcnt > 0);

  if (--buffer->refcnt > 0) {
    EVBUFFER_UNLOCK(buffer);
    return;
  }

  for (chain = buffer->first; chain != NULL; chain = next) {
    next = chain->next;
    evbuffer_chain_free(chain);
  }
  evbuffer_remove_all_callbacks(buffer);
  if (buffer->deferred_cbs)
    event_deferred_cb_cancel_(buffer->cb_queue, &buffer->deferred);

  EVBUFFER_UNLOCK(buffer);
  if (buffer->own_lock)
    EVTHREAD_FREE_LOCK(buffer->lock, EVTHREAD_LOCKTYPE_RECURSIVE);
  mm_free(buffer);
}

void
evbuffer_free(struct evbuffer *buffer)
{
  EVBUFFER_LOCK(buffer);
  evbuffer_decref_and_unlock_(buffer);
}

int evbuffer_set_max_read(struct evbuffer *buf, size_t max)
{
  if (max > INT_MAX) {
    return -1;
  }

  EVBUFFER_LOCK(buf);
  buf->max_read = max;
  EVBUFFER_UNLOCK(buf);
  return 0;
}
size_t evbuffer_get_max_read(struct evbuffer *buf)
{
  size_t result;
  EVBUFFER_LOCK(buf);
  result = buf->max_read;
  EVBUFFER_UNLOCK(buf);
  return result;
}

void
evbuffer_lock(struct evbuffer *buf)
{
  EVBUFFER_LOCK(buf);
}

void
evbuffer_unlock(struct evbuffer *buf)
{
  EVBUFFER_UNLOCK(buf);
}

size_t
evbuffer_get_length(const struct evbuffer *buffer)
{
  size_t result;
  EVBUFFER_LOCK(buffer);
  result = buffer->total_len;
  EVBUFFER_UNLOCK(buffer);
  return result;
}

size_t
evbuffer_get_contiguous_space(const struct evbuffer *buf)
{
  struct evbuffer_chain *chain;
  size_t result;

  EVBUFFER_LOCK(buf);
  chain = buf->first;
  result = (chain != NULL ? chain->off : 0);
  EVBUFFER_UNLOCK(buf);

  return result;
}

size_t
evbuffer_add_iovec(struct evbuffer * buf, struct evbuffer_iovec * vec, int n_vec) {
  int n;
  size_t res;
  size_t to_alloc;

  EVBUFFER_LOCK(buf);

  res = to_alloc = 0;

  for (n = 0; n < n_vec; n++) {
    to_alloc += vec[n].iov_len;
  }

  if (evbuffer_expand_fast_(buf, to_alloc, 2) < 0) {
    goto done;
  }

  for (n = 0; n < n_vec; n++) {
    /* XXX each 'add' call here does a bunch of setup that's
     * obviated by evbuffer_expand_fast_, and some cleanup that we
     * would like to do only once.  Instead we should just extract
     * the part of the code that's needed. */

    if (evbuffer_add(buf, vec[n].iov_base, vec[n].iov_len) < 0) {
      goto done;
    }

    res += vec[n].iov_len;
  }

done:
    EVBUFFER_UNLOCK(buf);
    return res;
}

int
evbuffer_reserve_space(struct evbuffer *buf, ev_ssize_t size,
    struct evbuffer_iovec *vec, int n_vecs)
{
  struct evbuffer_chain *chain, **chainp;
  int n = -1;

  EVBUFFER_LOCK(buf);
  if (buf->freeze_end)
    goto done;
  if (n_vecs < 1)
    goto done;
  if (n_vecs == 1) {
    if ((chain = evbuffer_expand_singlechain(buf, size)) == NULL)
      goto done;

    vec[0].iov_base = (void *)CHAIN_SPACE_PTR(chain);
    vec[0].iov_len = (size_t)CHAIN_SPACE_LEN(chain);
    EVUTIL_ASSERT(size<0 || (size_t)vec[0].iov_len >= (size_t)size);
    n = 1;
  } else {
    if (evbuffer_expand_fast_(buf, size, n_vecs)<0)
      goto done;
    n = evbuffer_read_setup_vecs_(buf, size, vec, n_vecs,
        &chainp, 0);
  }

done:
  EVBUFFER_UNLOCK(buf);
  return n;

}

static int
advance_last_with_data(struct evbuffer *buf)
{
  int n = 0;
  struct evbuffer_chain **chainp = buf->last_with_datap;

  ASSERT_EVBUFFER_LOCKED(buf);

  if (!*chainp)
    return 0;

  while ((*chainp)->next) {
    chainp = &(*chainp)->next;
    if ((*chainp)->off)
      buf->last_with_datap = chainp;
    ++n;
  }
  return n;
}

int
evbuffer_commit_space(struct evbuffer *buf,
    struct evbuffer_iovec *vec, int n_vecs)
{
  struct evbuffer_chain *chain, **firstchainp, **chainp;
  int result = -1;
  size_t added = 0;
  int i;

  EVBUFFER_LOCK(buf);

  if (buf->freeze_end)
    goto done;
  if (n_vecs == 0) {
    result = 0;
    goto done;
  } else if (n_vecs == 1 &&
      (buf->last && vec[0].iov_base == (void *)CHAIN_SPACE_PTR(buf->last))) {
    /* The user only got or used one chain; it might not
     * be the first one with space in it. */
    if ((size_t)vec[0].iov_len > (size_t)CHAIN_SPACE_LEN(buf->last))
      goto done;
    buf->last->off += vec[0].iov_len;
    added = vec[0].iov_len;
    if (added)
      advance_last_with_data(buf);
    goto okay;
  }

  /* Advance 'firstchain' to the first chain with space in it. */
  firstchainp = buf->last_with_datap;
  if (!*firstchainp)
    goto done;
  if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
    firstchainp = &(*firstchainp)->next;
  }

  chain = *firstchainp;
  /* pass 1: make sure that the pointers and lengths of vecs[] are in
   * bounds before we try to commit anything. */
  for (i=0; i<n_vecs; ++i) {
    if (!chain)
      goto done;
    if (vec[i].iov_base != (void *)CHAIN_SPACE_PTR(chain) ||
        (size_t)vec[i].iov_len > CHAIN_SPACE_LEN(chain))
      goto done;
    chain = chain->next;
  }
  /* pass 2: actually adjust all the chains. */
  chainp = firstchainp;
  for (i=0; i<n_vecs; ++i) {
    (*chainp)->off += vec[i].iov_len;
    added += vec[i].iov_len;
    if (vec[i].iov_len) {
      buf->last_with_datap = chainp;
    }
    chainp = &(*chainp)->next;
  }

okay:
  buf->total_len += added;
  buf->n_add_for_cb += added;
  result = 0;
  evbuffer_invoke_callbacks_(buf);

done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

static inline int
HAS_PINNED_R(struct evbuffer *buf)
{
  return (buf->last && CHAIN_PINNED_R(buf->last));
}

static inline void
ZERO_CHAIN(struct evbuffer *dst)
{
  ASSERT_EVBUFFER_LOCKED(dst);
  dst->first = NULL;
  dst->last = NULL;
  dst->last_with_datap = &(dst)->first;
  dst->total_len = 0;
}

/* Prepares the contents of src to be moved to another buffer by removing
 * read-pinned chains. The first pinned chain is saved in first, and the
 * last in last. If src has no read-pinned chains, first and last are set
 * to NULL. */
static int
PRESERVE_PINNED(struct evbuffer *src, struct evbuffer_chain **first,
    struct evbuffer_chain **last)
{
  struct evbuffer_chain *chain, **pinned;

  ASSERT_EVBUFFER_LOCKED(src);

  if (!HAS_PINNED_R(src)) {
    *first = *last = NULL;
    return 0;
  }

  pinned = src->last_with_datap;
  if (!CHAIN_PINNED_R(*pinned))
    pinned = &(*pinned)->next;
  EVUTIL_ASSERT(CHAIN_PINNED_R(*pinned));
  chain = *first = *pinned;
  *last = src->last;

  /* If there's data in the first pinned chain, we need to allocate
   * a new chain and copy the data over. */
  if (chain->off) {
    struct evbuffer_chain *tmp;

    EVUTIL_ASSERT(pinned == src->last_with_datap);
    tmp = evbuffer_chain_new_membuf(chain->off);
    if (!tmp)
      return -1;
    memcpy(tmp->buffer, chain->buffer + chain->misalign,
      chain->off);
    tmp->off = chain->off;
    *src->last_with_datap = tmp;
    src->last = tmp;
    chain->misalign += chain->off;
    chain->off = 0;
  } else {
    src->last = *src->last_with_datap;
    *pinned = NULL;
  }

  return 0;
}

static inline void
RESTORE_PINNED(struct evbuffer *src, struct evbuffer_chain *pinned,
    struct evbuffer_chain *last)
{
  ASSERT_EVBUFFER_LOCKED(src);

  if (!pinned) {
    ZERO_CHAIN(src);
    return;
  }

  src->first = pinned;
  src->last = last;
  src->last_with_datap = &src->first;
  src->total_len = 0;
}

static inline void
COPY_CHAIN(struct evbuffer *dst, struct evbuffer *src)
{
  ASSERT_EVBUFFER_LOCKED(dst);
  ASSERT_EVBUFFER_LOCKED(src);
  dst->first = src->first;
  if (src->last_with_datap == &src->first)
    dst->last_with_datap = &dst->first;
  else
    dst->last_with_datap = src->last_with_datap;
  dst->last = src->last;
  dst->total_len = src->total_len;
}

static void
APPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
{
  struct evbuffer_chain **chp;

  ASSERT_EVBUFFER_LOCKED(dst);
  ASSERT_EVBUFFER_LOCKED(src);

  chp = evbuffer_free_trailing_empty_chains(dst);
  *chp = src->first;

  if (src->last_with_datap == &src->first)
    dst->last_with_datap = chp;
  else
    dst->last_with_datap = src->last_with_datap;
  dst->last = src->last;
  dst->total_len += src->total_len;
}

static inline void
APPEND_CHAIN_MULTICAST(struct evbuffer *dst, struct evbuffer *src)
{
  struct evbuffer_chain *tmp;
  struct evbuffer_chain *chain = src->first;
  struct evbuffer_multicast_parent *extra;

  ASSERT_EVBUFFER_LOCKED(dst);
  ASSERT_EVBUFFER_LOCKED(src);

  for (; chain; chain = chain->next) {
    if (!chain->off || chain->flags & EVBUFFER_DANGLING) {
      /* skip empty chains */
      continue;
    }

    tmp = evbuffer_chain_new(sizeof(struct evbuffer_multicast_parent));
    if (!tmp) {
      event_warn("%s: out of memory", __func__);
      return;
    }
    extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_multicast_parent, tmp);
    /* reference evbuffer containing source chain so it
     * doesn't get released while the chain is still
     * being referenced to */
    evbuffer_incref_(src);
    extra->source = src;
    /* reference source chain which now becomes immutable */
    evbuffer_chain_incref(chain);
    extra->parent = chain;
    chain->flags |= EVBUFFER_IMMUTABLE;
    tmp->buffer_len = chain->buffer_len;
    tmp->misalign = chain->misalign;
    tmp->off = chain->off;
    tmp->flags |= EVBUFFER_MULTICAST|EVBUFFER_IMMUTABLE;
    tmp->buffer = chain->buffer;
    evbuffer_chain_insert(dst, tmp);
  }
}

static void
PREPEND_CHAIN(struct evbuffer *dst, struct evbuffer *src)
{
  ASSERT_EVBUFFER_LOCKED(dst);
  ASSERT_EVBUFFER_LOCKED(src);
  src->last->next = dst->first;
  dst->first = src->first;
  dst->total_len += src->total_len;
  if (*dst->last_with_datap == NULL) {
    if (src->last_with_datap == &(src)->first)
      dst->last_with_datap = &dst->first;
    else
      dst->last_with_datap = src->last_with_datap;
  } else if (dst->last_with_datap == &dst->first) {
    dst->last_with_datap = &src->last->next;
  }
}

int
evbuffer_add_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
{
  struct evbuffer_chain *pinned, *last;
  size_t in_total_len, out_total_len;
  int result = 0;

  EVBUFFER_LOCK2(inbuf, outbuf);
  in_total_len = inbuf->total_len;
  out_total_len = outbuf->total_len;

  if (in_total_len == 0 || outbuf == inbuf)
    goto done;

  if (outbuf->freeze_end || inbuf->freeze_start) {
    result = -1;
    goto done;
  }

  if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
    result = -1;
    goto done;
  }

  if (out_total_len == 0) {
    /* There might be an empty chain at the start of outbuf; free
     * it. */
    evbuffer_free_all_chains(outbuf->first);
    COPY_CHAIN(outbuf, inbuf);
  } else {
    APPEND_CHAIN(outbuf, inbuf);
  }

  RESTORE_PINNED(inbuf, pinned, last);

  inbuf->n_del_for_cb += in_total_len;
  outbuf->n_add_for_cb += in_total_len;

  evbuffer_invoke_callbacks_(inbuf);
  evbuffer_invoke_callbacks_(outbuf);

done:
  EVBUFFER_UNLOCK2(inbuf, outbuf);
  return result;
}

int
evbuffer_add_buffer_reference(struct evbuffer *outbuf, struct evbuffer *inbuf)
{
  size_t in_total_len, out_total_len;
  struct evbuffer_chain *chain;
  int result = 0;

  EVBUFFER_LOCK2(inbuf, outbuf);
  in_total_len = inbuf->total_len;
  out_total_len = outbuf->total_len;
  chain = inbuf->first;

  if (in_total_len == 0)
    goto done;

  if (outbuf->freeze_end || outbuf == inbuf) {
    result = -1;
    goto done;
  }

  for (; chain; chain = chain->next) {
    if ((chain->flags & (EVBUFFER_FILESEGMENT|EVBUFFER_SENDFILE|EVBUFFER_MULTICAST)) != 0) {
      /* chain type can not be referenced */
      result = -1;
      goto done;
    }
  }

  if (out_total_len == 0) {
    /* There might be an empty chain at the start of outbuf; free
     * it. */
    evbuffer_free_all_chains(outbuf->first);
  }
  APPEND_CHAIN_MULTICAST(outbuf, inbuf);

  outbuf->n_add_for_cb += in_total_len;
  evbuffer_invoke_callbacks_(outbuf);

done:
  EVBUFFER_UNLOCK2(inbuf, outbuf);
  return result;
}

int
evbuffer_prepend_buffer(struct evbuffer *outbuf, struct evbuffer *inbuf)
{
  struct evbuffer_chain *pinned, *last;
  size_t in_total_len, out_total_len;
  int result = 0;

  EVBUFFER_LOCK2(inbuf, outbuf);

  in_total_len = inbuf->total_len;
  out_total_len = outbuf->total_len;

  if (!in_total_len || inbuf == outbuf)
    goto done;

  if (outbuf->freeze_start || inbuf->freeze_start) {
    result = -1;
    goto done;
  }

  if (PRESERVE_PINNED(inbuf, &pinned, &last) < 0) {
    result = -1;
    goto done;
  }

  if (out_total_len == 0) {
    /* There might be an empty chain at the start of outbuf; free
     * it. */
    evbuffer_free_all_chains(outbuf->first);
    COPY_CHAIN(outbuf, inbuf);
  } else {
    PREPEND_CHAIN(outbuf, inbuf);
  }

  RESTORE_PINNED(inbuf, pinned, last);

  inbuf->n_del_for_cb += in_total_len;
  outbuf->n_add_for_cb += in_total_len;

  evbuffer_invoke_callbacks_(inbuf);
  evbuffer_invoke_callbacks_(outbuf);
done:
  EVBUFFER_UNLOCK2(inbuf, outbuf);
  return result;
}

int
evbuffer_drain(struct evbuffer *buf, size_t len)
{
  struct evbuffer_chain *chain, *next;
  size_t remaining, old_len;
  int result = 0;

  EVBUFFER_LOCK(buf);
  old_len = buf->total_len;

  if (old_len == 0)
    goto done;

  if (buf->freeze_start) {
    result = -1;
    goto done;
  }

  if (len >= old_len && !HAS_PINNED_R(buf)) {
    len = old_len;
    for (chain = buf->first; chain != NULL; chain = next) {
      next = chain->next;
      evbuffer_chain_free(chain);
    }

    ZERO_CHAIN(buf);
  } else {
    if (len >= old_len)
      len = old_len;

    buf->total_len -= len;
    remaining = len;
    for (chain = buf->first;
         remaining >= chain->off;
         chain = next) {
      next = chain->next;
      remaining -= chain->off;

      if (chain == *buf->last_with_datap) {
        buf->last_with_datap = &buf->first;
      }
      if (&chain->next == buf->last_with_datap)
        buf->last_with_datap = &buf->first;

      if (CHAIN_PINNED_R(chain)) {
        EVUTIL_ASSERT(remaining == 0);
        chain->misalign += chain->off;
        chain->off = 0;
        break;
      } else
        evbuffer_chain_free(chain);
    }

    buf->first = chain;
    EVUTIL_ASSERT(remaining <= chain->off);
    chain->misalign += remaining;
    chain->off -= remaining;
  }

  buf->n_del_for_cb += len;
  /* Tell someone about changes in this buffer */
  evbuffer_invoke_callbacks_(buf);

done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

/* Reads data from an event buffer and drains the bytes read */
int
evbuffer_remove(struct evbuffer *buf, void *data_out, size_t datlen)
{
  ev_ssize_t n;
  EVBUFFER_LOCK(buf);
  n = evbuffer_copyout_from(buf, NULL, data_out, datlen);
  if (n > 0) {
    if (evbuffer_drain(buf, n)<0)
      n = -1;
  }
  EVBUFFER_UNLOCK(buf);
  return (int)n;
}

ev_ssize_t
evbuffer_copyout(struct evbuffer *buf, void *data_out, size_t datlen)
{
  return evbuffer_copyout_from(buf, NULL, data_out, datlen);
}

ev_ssize_t
evbuffer_copyout_from(struct evbuffer *buf, const struct evbuffer_ptr *pos,
    void *data_out, size_t datlen)
{
  /*XXX fails badly on sendfile case. */
  struct evbuffer_chain *chain;
  char *data = data_out;
  size_t nread;
  ev_ssize_t result = 0;
  size_t pos_in_chain;

  EVBUFFER_LOCK(buf);

  if (pos) {
    if (datlen > (size_t)(EV_SSIZE_MAX - pos->pos)) {
      result = -1;
      goto done;
    }
    chain = pos->internal_.chain;
    pos_in_chain = pos->internal_.pos_in_chain;
    if (datlen + pos->pos > buf->total_len)
      datlen = buf->total_len - pos->pos;
  } else {
    chain = buf->first;
    pos_in_chain = 0;
    if (datlen > buf->total_len)
      datlen = buf->total_len;
  }


  if (datlen == 0)
    goto done;

  if (buf->freeze_start) {
    result = -1;
    goto done;
  }

  nread = datlen;

  while (datlen && datlen >= chain->off - pos_in_chain) {
    size_t copylen = chain->off - pos_in_chain;
    memcpy(data,
        chain->buffer + chain->misalign + pos_in_chain,
        copylen);
    data += copylen;
    datlen -= copylen;

    chain = chain->next;
    pos_in_chain = 0;
    EVUTIL_ASSERT(chain || datlen==0);
  }

  if (datlen) {
    EVUTIL_ASSERT(chain);
    EVUTIL_ASSERT(datlen+pos_in_chain <= chain->off);

    memcpy(data, chain->buffer + chain->misalign + pos_in_chain,
        datlen);
  }

  result = nread;
done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

/* reads data from the src buffer to the dst buffer, avoids memcpy as
 * possible. */
/*  XXXX should return ev_ssize_t */
int
evbuffer_remove_buffer(struct evbuffer *src, struct evbuffer *dst,
    size_t datlen)
{
  /*XXX We should have an option to force this to be zero-copy.*/

  /*XXX can fail badly on sendfile case. */
  struct evbuffer_chain *chain, *previous;
  size_t nread = 0;
  int result;

  EVBUFFER_LOCK2(src, dst);

  chain = previous = src->first;

  if (datlen == 0 || dst == src) {
    result = 0;
    goto done;
  }

  if (dst->freeze_end || src->freeze_start) {
    result = -1;
    goto done;
  }

  /* short-cut if there is no more data buffered */
  if (datlen >= src->total_len) {
    datlen = src->total_len;
    evbuffer_add_buffer(dst, src);
    result = (int)datlen; /*XXXX should return ev_ssize_t*/
    goto done;
  }

  /* removes chains if possible */
  while (chain->off <= datlen) {
    /* We can't remove the last with data from src unless we
     * remove all chains, in which case we would have done the if
     * block above */
    EVUTIL_ASSERT(chain != *src->last_with_datap);
    nread += chain->off;
    datlen -= chain->off;
    previous = chain;
    if (src->last_with_datap == &chain->next)
      src->last_with_datap = &src->first;
    chain = chain->next;
  }

  if (chain != src->first) {
    /* we can remove the chain */
    struct evbuffer_chain **chp;
    chp = evbuffer_free_trailing_empty_chains(dst);

    if (dst->first == NULL) {
      dst->first = src->first;
    } else {
      *chp = src->first;
    }
    dst->last = previous;
    previous->next = NULL;
    src->first = chain;
    advance_last_with_data(dst);

    dst->total_len += nread;
    dst->n_add_for_cb += nread;
  }

  /* we know that there is more data in the src buffer than
   * we want to read, so we manually drain the chain */
  evbuffer_add(dst, chain->buffer + chain->misalign, datlen);
  chain->misalign += datlen;
  chain->off -= datlen;
  nread += datlen;

  /* You might think we would want to increment dst->n_add_for_cb
   * here too.  But evbuffer_add above already took care of that.
   */
  src->total_len -= nread;
  src->n_del_for_cb += nread;

  if (nread) {
    evbuffer_invoke_callbacks_(dst);
    evbuffer_invoke_callbacks_(src);
  }
  result = (int)nread;/*XXXX should change return type */

done:
  EVBUFFER_UNLOCK2(src, dst);
  return result;
}

unsigned char *
evbuffer_pullup(struct evbuffer *buf, ev_ssize_t size)
{
  struct evbuffer_chain *chain, *next, *tmp, *last_with_data;
  unsigned char *buffer, *result = NULL;
  ev_ssize_t remaining;
  int removed_last_with_data = 0;
  int removed_last_with_datap = 0;

  EVBUFFER_LOCK(buf);

  chain = buf->first;

  if (size < 0)
    size = buf->total_len;
  /* if size > buf->total_len, we cannot guarantee to the user that she
   * is going to have a long enough buffer afterwards; so we return
   * NULL */
  if (size == 0 || (size_t)size > buf->total_len)
    goto done;

  /* No need to pull up anything; the first size bytes are
   * already here. */
  if (chain->off >= (size_t)size) {
    result = chain->buffer + chain->misalign;
    goto done;
  }

  /* Make sure that none of the chains we need to copy from is pinned. */
  remaining = size - chain->off;
  EVUTIL_ASSERT(remaining >= 0);
  for (tmp=chain->next; tmp; tmp=tmp->next) {
    if (CHAIN_PINNED(tmp))
      goto done;
    if (tmp->off >= (size_t)remaining)
      break;
    remaining -= tmp->off;
  }

  if (CHAIN_PINNED(chain)) {
    size_t old_off = chain->off;
    if (CHAIN_SPACE_LEN(chain) < size - chain->off) {
      /* not enough room at end of chunk. */
      goto done;
    }
    buffer = CHAIN_SPACE_PTR(chain);
    tmp = chain;
    tmp->off = size;
    size -= old_off;
    chain = chain->next;
  } else if (chain->buffer_len - chain->misalign >= (size_t)size) {
    /* already have enough space in the first chain */
    size_t old_off = chain->off;
    buffer = chain->buffer + chain->misalign + chain->off;
    tmp = chain;
    tmp->off = size;
    size -= old_off;
    chain = chain->next;
  } else {
    if ((tmp = evbuffer_chain_new_membuf(size)) == NULL) {
      event_warn("%s: out of memory", __func__);
      goto done;
    }
    buffer = tmp->buffer;
    tmp->off = size;
    buf->first = tmp;
  }

  /* TODO(niels): deal with buffers that point to NULL like sendfile */

  /* Copy and free every chunk that will be entirely pulled into tmp */
  last_with_data = *buf->last_with_datap;
  for (; chain != NULL && (size_t)size >= chain->off; chain = next) {
    next = chain->next;

    if (chain->buffer) {
      memcpy(buffer, chain->buffer + chain->misalign, chain->off);
      size -= chain->off;
      buffer += chain->off;
    }
    if (chain == last_with_data)
      removed_last_with_data = 1;
    if (&chain->next == buf->last_with_datap)
      removed_last_with_datap = 1;

    evbuffer_chain_free(chain);
  }

  if (chain != NULL) {
    memcpy(buffer, chain->buffer + chain->misalign, size);
    chain->misalign += size;
    chain->off -= size;
  } else {
    buf->last = tmp;
  }

  tmp->next = chain;

  if (removed_last_with_data) {
    buf->last_with_datap = &buf->first;
  } else if (removed_last_with_datap) {
    if (buf->first->next && buf->first->next->off)
      buf->last_with_datap = &buf->first->next;
    else
      buf->last_with_datap = &buf->first;
  }

  result = (tmp->buffer + tmp->misalign);

done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

/*
 * Reads a line terminated by either '\r\n', '\n\r' or '\r' or '\n'.
 * The returned buffer needs to be freed by the called.
 */
char *
evbuffer_readline(struct evbuffer *buffer)
{
  return evbuffer_readln(buffer, NULL, EVBUFFER_EOL_ANY);
}

static inline ev_ssize_t
evbuffer_strchr(struct evbuffer_ptr *it, const char chr)
{
  struct evbuffer_chain *chain = it->internal_.chain;
  size_t i = it->internal_.pos_in_chain;
  while (chain != NULL) {
    char *buffer = (char *)chain->buffer + chain->misalign;
    char *cp = memchr(buffer+i, chr, chain->off-i);
    if (cp) {
      it->internal_.chain = chain;
      it->internal_.pos_in_chain = cp - buffer;
      it->pos += (cp - buffer - i);
      return it->pos;
    }
    it->pos += chain->off - i;
    i = 0;
    chain = chain->next;
  }

  return (-1);
}

static inline char *
find_eol_char(char *s, size_t len)
{
#define CHUNK_SZ 128
  /* Lots of benchmarking found this approach to be faster in practice
   * than doing two memchrs over the whole buffer, doin a memchr on each
   * char of the buffer, or trying to emulate memchr by hand. */
  char *s_end, *cr, *lf;
  s_end = s+len;
  while (s < s_end) {
    size_t chunk = (s + CHUNK_SZ < s_end) ? CHUNK_SZ : (s_end - s);
    cr = memchr(s, '\r', chunk);
    lf = memchr(s, '\n', chunk);
    if (cr) {
      if (lf && lf < cr)
        return lf;
      return cr;
    } else if (lf) {
      return lf;
    }
    s += CHUNK_SZ;
  }

  return NULL;
#undef CHUNK_SZ
}

static ev_ssize_t
evbuffer_find_eol_char(struct evbuffer_ptr *it)
{
  struct evbuffer_chain *chain = it->internal_.chain;
  size_t i = it->internal_.pos_in_chain;
  while (chain != NULL) {
    char *buffer = (char *)chain->buffer + chain->misalign;
    char *cp = find_eol_char(buffer+i, chain->off-i);
    if (cp) {
      it->internal_.chain = chain;
      it->internal_.pos_in_chain = cp - buffer;
      it->pos += (cp - buffer) - i;
      return it->pos;
    }
    it->pos += chain->off - i;
    i = 0;
    chain = chain->next;
  }

  return (-1);
}

static inline size_t
evbuffer_strspn(
  struct evbuffer_ptr *ptr, const char *chrset)
{
  size_t count = 0;
  struct evbuffer_chain *chain = ptr->internal_.chain;
  size_t i = ptr->internal_.pos_in_chain;

  if (!chain)
    return 0;

  while (1) {
    char *buffer = (char *)chain->buffer + chain->misalign;
    for (; i < chain->off; ++i) {
      const char *p = chrset;
      while (*p) {
        if (buffer[i] == *p++)
          goto next;
      }
      ptr->internal_.chain = chain;
      ptr->internal_.pos_in_chain = i;
      ptr->pos += count;
      return count;
    next:
      ++count;
    }
    i = 0;

    if (! chain->next) {
      ptr->internal_.chain = chain;
      ptr->internal_.pos_in_chain = i;
      ptr->pos += count;
      return count;
    }

    chain = chain->next;
  }
}


static inline int
evbuffer_getchr(struct evbuffer_ptr *it)
{
  struct evbuffer_chain *chain = it->internal_.chain;
  size_t off = it->internal_.pos_in_chain;

  if (chain == NULL)
    return -1;

  return (unsigned char)chain->buffer[chain->misalign + off];
}

struct evbuffer_ptr
evbuffer_search_eol(struct evbuffer *buffer,
    struct evbuffer_ptr *start, size_t *eol_len_out,
    enum evbuffer_eol_style eol_style)
{
  struct evbuffer_ptr it, it2;
  size_t extra_drain = 0;
  int ok = 0;

  /* Avoid locking in trivial edge cases */
  if (start && start->internal_.chain == NULL) {
    PTR_NOT_FOUND(&it);
    if (eol_len_out)
      *eol_len_out = extra_drain;
    return it;
  }

  EVBUFFER_LOCK(buffer);

  if (start) {
    memcpy(&it, start, sizeof(it));
  } else {
    it.pos = 0;
    it.internal_.chain = buffer->first;
    it.internal_.pos_in_chain = 0;
  }

  /* the eol_style determines our first stop character and how many
   * characters we are going to drain afterwards. */
  switch (eol_style) {
  case EVBUFFER_EOL_ANY:
    if (evbuffer_find_eol_char(&it) < 0)
      goto done;
    memcpy(&it2, &it, sizeof(it));
    extra_drain = evbuffer_strspn(&it2, "\r\n");
    break;
  case EVBUFFER_EOL_CRLF_STRICT: {
    it = evbuffer_search(buffer, "\r\n", 2, &it);
    if (it.pos < 0)
      goto done;
    extra_drain = 2;
    break;
  }
  case EVBUFFER_EOL_CRLF: {
    ev_ssize_t start_pos = it.pos;
    /* Look for a LF ... */
    if (evbuffer_strchr(&it, '\n') < 0)
      goto done;
    extra_drain = 1;
    /* ... optionally preceded by a CR. */
    if (it.pos == start_pos)
      break; /* If the first character is \n, don't back up */
    /* This potentially does an extra linear walk over the first
     * few chains.  Probably, that's not too expensive unless you
     * have a really pathological setup. */
    memcpy(&it2, &it, sizeof(it));
    if (evbuffer_ptr_subtract(buffer, &it2, 1)<0)
      break;
    if (evbuffer_getchr(&it2) == '\r') {
      memcpy(&it, &it2, sizeof(it));
      extra_drain = 2;
    }
    break;
  }
  case EVBUFFER_EOL_LF:
    if (evbuffer_strchr(&it, '\n') < 0)
      goto done;
    extra_drain = 1;
    break;
  case EVBUFFER_EOL_NUL:
    if (evbuffer_strchr(&it, '\0') < 0)
      goto done;
    extra_drain = 1;
    break;
  default:
    goto done;
  }

  ok = 1;
done:
  EVBUFFER_UNLOCK(buffer);

  if (!ok)
    PTR_NOT_FOUND(&it);
  if (eol_len_out)
    *eol_len_out = extra_drain;

  return it;
}

char *
evbuffer_readln(struct evbuffer *buffer, size_t *n_read_out,
    enum evbuffer_eol_style eol_style)
{
  struct evbuffer_ptr it;
  char *line;
  size_t n_to_copy=0, extra_drain=0;
  char *result = NULL;

  EVBUFFER_LOCK(buffer);

  if (buffer->freeze_start) {
    goto done;
  }

  it = evbuffer_search_eol(buffer, NULL, &extra_drain, eol_style);
  if (it.pos < 0)
    goto done;
  n_to_copy = it.pos;

  if ((line = mm_malloc(n_to_copy+1)) == NULL) {
    event_warn("%s: out of memory", __func__);
    goto done;
  }

  evbuffer_remove(buffer, line, n_to_copy);
  line[n_to_copy] = '\0';

  evbuffer_drain(buffer, extra_drain);
  result = line;
done:
  EVBUFFER_UNLOCK(buffer);

  if (n_read_out)
    *n_read_out = result ? n_to_copy : 0;

  return result;
}

#define EVBUFFER_CHAIN_MAX_AUTO_SIZE 4096

/* Adds data to an event buffer */

int
evbuffer_add(struct evbuffer *buf, const void *data_in, size_t datlen)
{
  struct evbuffer_chain *chain, *tmp;
  const unsigned char *data = data_in;
  size_t remain, to_alloc;
  int result = -1;

  EVBUFFER_LOCK(buf);

  if (buf->freeze_end) {
    goto done;
  }
  /* Prevent buf->total_len overflow */
  if (datlen > EV_SIZE_MAX - buf->total_len) {
    goto done;
  }

  if (*buf->last_with_datap == NULL) {
    chain = buf->last;
  } else {
    chain = *buf->last_with_datap;
  }

  /* If there are no chains allocated for this buffer, allocate one
   * big enough to hold all the data. */
  if (chain == NULL) {
    chain = evbuffer_chain_insert_new(buf, datlen);
    if (!chain)
      goto done;
  }

  if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
    /* Always true for mutable buffers */
    EVUTIL_ASSERT(chain->misalign >= 0 &&
        (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX);
    remain = chain->buffer_len - (size_t)chain->misalign - chain->off;
    if (remain >= datlen) {
      /* there's enough space to hold all the data in the
       * current last chain */
      memcpy(chain->buffer + chain->misalign + chain->off,
          data, datlen);
      chain->off += datlen;
      buf->total_len += datlen;
      buf->n_add_for_cb += datlen;
      goto out;
    } else if (!CHAIN_PINNED(chain) &&
        evbuffer_chain_should_realign(chain, datlen)) {
      /* we can fit the data into the misalignment */
      evbuffer_chain_align(chain);

      memcpy(chain->buffer + chain->off, data, datlen);
      chain->off += datlen;
      buf->total_len += datlen;
      buf->n_add_for_cb += datlen;
      goto out;
    }
  } else {
    /* we cannot write any data to the last chain */
    remain = 0;
  }

  /* we need to add another chain */
  to_alloc = chain->buffer_len;
  if (to_alloc <= EVBUFFER_CHAIN_MAX_AUTO_SIZE/2)
    to_alloc <<= 1;
  if (datlen > to_alloc)
    to_alloc = datlen;
  tmp = evbuffer_chain_new_membuf(to_alloc);
  if (tmp == NULL)
    goto done;

  if (remain) {
    memcpy(chain->buffer + chain->misalign + chain->off,
        data, remain);
    chain->off += remain;
    buf->total_len += remain;
    buf->n_add_for_cb += remain;
  }

  data += remain;
  datlen -= remain;

  memcpy(tmp->buffer, data, datlen);
  tmp->off = datlen;
  evbuffer_chain_insert(buf, tmp);
  buf->n_add_for_cb += datlen;

out:
  evbuffer_invoke_callbacks_(buf);
  result = 0;
done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

int
evbuffer_prepend(struct evbuffer *buf, const void *data, size_t datlen)
{
  struct evbuffer_chain *chain, *tmp;
  int result = -1;

  EVBUFFER_LOCK(buf);

  if (datlen == 0) {
    result = 0;
    goto done;
  }
  if (buf->freeze_start) {
    goto done;
  }
  if (datlen > EV_SIZE_MAX - buf->total_len) {
    goto done;
  }

  chain = buf->first;

  if (chain == NULL) {
    chain = evbuffer_chain_insert_new(buf, datlen);
    if (!chain)
      goto done;
  }

  /* we cannot touch immutable buffers */
  if ((chain->flags & EVBUFFER_IMMUTABLE) == 0) {
    /* Always true for mutable buffers */
    EVUTIL_ASSERT(chain->misalign >= 0 &&
        (ev_uint64_t)chain->misalign <= EVBUFFER_CHAIN_MAX);

    /* If this chain is empty, we can treat it as
     * 'empty at the beginning' rather than 'empty at the end' */
    if (chain->off == 0)
      chain->misalign = chain->buffer_len;

    if ((size_t)chain->misalign >= datlen) {
      /* we have enough space to fit everything */
      memcpy(chain->buffer + chain->misalign - datlen,
          data, datlen);
      chain->off += datlen;
      chain->misalign -= datlen;
      buf->total_len += datlen;
      buf->n_add_for_cb += datlen;
      goto out;
    } else if (chain->misalign) {
      /* we can only fit some of the data. */
      memcpy(chain->buffer,
          (char*)data + datlen - chain->misalign,
          (size_t)chain->misalign);
      chain->off += (size_t)chain->misalign;
      buf->total_len += (size_t)chain->misalign;
      buf->n_add_for_cb += (size_t)chain->misalign;
      datlen -= (size_t)chain->misalign;
      chain->misalign = 0;
    }
  }

  /* we need to add another chain */
  if ((tmp = evbuffer_chain_new_membuf(datlen)) == NULL)
    goto done;
  buf->first = tmp;
  if (buf->last_with_datap == &buf->first && chain->off)
    buf->last_with_datap = &tmp->next;

  tmp->next = chain;

  tmp->off = datlen;
  EVUTIL_ASSERT(datlen <= tmp->buffer_len);
  tmp->misalign = tmp->buffer_len - datlen;

  memcpy(tmp->buffer + tmp->misalign, data, datlen);
  buf->total_len += datlen;
  buf->n_add_for_cb += datlen;

out:
  evbuffer_invoke_callbacks_(buf);
  result = 0;
done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

/** Helper: realigns the memory in chain->buffer so that misalign is 0. */
static void
evbuffer_chain_align(struct evbuffer_chain *chain)
{
  EVUTIL_ASSERT(!(chain->flags & EVBUFFER_IMMUTABLE));
  EVUTIL_ASSERT(!(chain->flags & EVBUFFER_MEM_PINNED_ANY));
  memmove(chain->buffer, chain->buffer + chain->misalign, chain->off);
  chain->misalign = 0;
}

#define MAX_TO_COPY_IN_EXPAND 4096
#define MAX_TO_REALIGN_IN_EXPAND 2048

/** Helper: return true iff we should realign chain to fit datalen bytes of
    data in it. */
static int
evbuffer_chain_should_realign(struct evbuffer_chain *chain,
    size_t datlen)
{
  return chain->buffer_len - chain->off >= datlen &&
      (chain->off < chain->buffer_len / 2) &&
      (chain->off <= MAX_TO_REALIGN_IN_EXPAND);
}

/* Expands the available space in the event buffer to at least datlen, all in
 * a single chunk.  Return that chunk. */
static struct evbuffer_chain *
evbuffer_expand_singlechain(struct evbuffer *buf, size_t datlen)
{
  struct evbuffer_chain *chain, **chainp;
  struct evbuffer_chain *result = NULL;
  ASSERT_EVBUFFER_LOCKED(buf);

  chainp = buf->last_with_datap;

  /* XXX If *chainp is no longer writeable, but has enough space in its
   * misalign, this might be a bad idea: we could still use *chainp, not
   * (*chainp)->next. */
  if (*chainp && CHAIN_SPACE_LEN(*chainp) == 0)
    chainp = &(*chainp)->next;

  /* 'chain' now points to the first chain with writable space (if any)
   * We will either use it, realign it, replace it, or resize it. */
  chain = *chainp;

  if (chain == NULL ||
      (chain->flags & (EVBUFFER_IMMUTABLE|EVBUFFER_MEM_PINNED_ANY))) {
    /* We can't use the last_with_data chain at all.  Just add a
     * new one that's big enough. */
    goto insert_new;
  }

  /* If we can fit all the data, then we don't have to do anything */
  if (CHAIN_SPACE_LEN(chain) >= datlen) {
    result = chain;
    goto ok;
  }

  /* If the chain is completely empty, just replace it by adding a new
   * empty chain. */
  if (chain->off == 0) {
    goto insert_new;
  }

  /* If the misalignment plus the remaining space fulfills our data
   * needs, we could just force an alignment to happen.  Afterwards, we
   * have enough space.  But only do this if we're saving a lot of space
   * and not moving too much data.  Otherwise the space savings are
   * probably offset by the time lost in copying.
   */
  if (evbuffer_chain_should_realign(chain, datlen)) {
    evbuffer_chain_align(chain);
    result = chain;
    goto ok;
  }

  /* At this point, we can either resize the last chunk with space in
   * it, use the next chunk after it, or   If we add a new chunk, we waste
   * CHAIN_SPACE_LEN(chain) bytes in the former last chunk.  If we
   * resize, we have to copy chain->off bytes.
   */

  /* Would expanding this chunk be affordable and worthwhile? */
  if (CHAIN_SPACE_LEN(chain) < chain->buffer_len / 8 ||
      chain->off > MAX_TO_COPY_IN_EXPAND ||
    datlen >= (EVBUFFER_CHAIN_MAX - chain->off)) {
    /* It's not worth resizing this chain. Can the next one be
     * used? */
    if (chain->next && CHAIN_SPACE_LEN(chain->next) >= datlen) {
      /* Yes, we can just use the next chain (which should
       * be empty. */
      result = chain->next;
      goto ok;
    } else {
      /* No; append a new chain (which will free all
       * terminal empty chains.) */
      goto insert_new;
    }
  } else {
    /* Okay, we're going to try to resize this chain: Not doing so
     * would waste at least 1/8 of its current allocation, and we
     * can do so without having to copy more than
     * MAX_TO_COPY_IN_EXPAND bytes. */
    /* figure out how much space we need */
    size_t length = chain->off + datlen;
    struct evbuffer_chain *tmp = evbuffer_chain_new_membuf(length);
    if (tmp == NULL)
      goto err;

    /* copy the data over that we had so far */
    tmp->off = chain->off;
    memcpy(tmp->buffer, chain->buffer + chain->misalign,
        chain->off);
    /* fix up the list */
    EVUTIL_ASSERT(*chainp == chain);
    result = *chainp = tmp;

    if (buf->last == chain)
      buf->last = tmp;

    tmp->next = chain->next;
    evbuffer_chain_free(chain);
    goto ok;
  }

insert_new:
  result = evbuffer_chain_insert_new(buf, datlen);
  if (!result)
    goto err;
ok:
  EVUTIL_ASSERT(result);
  EVUTIL_ASSERT(CHAIN_SPACE_LEN(result) >= datlen);
err:
  return result;
}

/* Make sure that datlen bytes are available for writing in the last n
 * chains.  Never copies or moves data. */
int
evbuffer_expand_fast_(struct evbuffer *buf, size_t datlen, int n)
{
  struct evbuffer_chain *chain = buf->last, *tmp, *next;
  size_t avail;
  int used;

  ASSERT_EVBUFFER_LOCKED(buf);
  EVUTIL_ASSERT(n >= 2);

  if (chain == NULL || (chain->flags & EVBUFFER_IMMUTABLE)) {
    /* There is no last chunk, or we can't touch the last chunk.
     * Just add a new chunk. */
    chain = evbuffer_chain_insert_new(buf, datlen);
    if (chain == NULL)
      return (-1);
    else
      return (0);
  }

  used = 0; /* number of chains we're using space in. */
  avail = 0; /* how much space they have. */
  /* How many bytes can we stick at the end of buffer as it is?  Iterate
   * over the chains at the end of the buffer, tring to see how much
   * space we have in the first n. */
  for (chain = *buf->last_with_datap; chain; chain = chain->next) {
    if (chain->off) {
      size_t space = (size_t) CHAIN_SPACE_LEN(chain);
      EVUTIL_ASSERT(chain == *buf->last_with_datap);
      if (space) {
        avail += space;
        ++used;
      }
    } else {
      /* No data in chain; realign it. */
      chain->misalign = 0;
      avail += chain->buffer_len;
      ++used;
    }
    if (avail >= datlen) {
      /* There is already enough space.  Just return */
      return (0);
    }
    if (used == n)
      break;
  }

  /* There wasn't enough space in the first n chains with space in
   * them. Either add a new chain with enough space, or replace all
   * empty chains with one that has enough space, depending on n. */
  if (used < n) {
    /* The loop ran off the end of the chains before it hit n
     * chains; we can add another. */
    EVUTIL_ASSERT(chain == NULL);

    tmp = evbuffer_chain_new_membuf(datlen - avail);
    if (tmp == NULL)
      return (-1);

    buf->last->next = tmp;
    buf->last = tmp;
    /* (we would only set last_with_data if we added the first
     * chain. But if the buffer had no chains, we would have
     * just allocated a new chain earlier) */
    return (0);
  } else {
    /* Nuke _all_ the empty chains. */
    int rmv_all = 0; /* True iff we removed last_with_data. */
    chain = *buf->last_with_datap;
    if (!chain->off) {
      EVUTIL_ASSERT(chain == buf->first);
      rmv_all = 1;
      avail = 0;
    } else {
      /* can't overflow, since only mutable chains have
       * huge misaligns. */
      avail = (size_t) CHAIN_SPACE_LEN(chain);
      chain = chain->next;
    }


    for (; chain; chain = next) {
      next = chain->next;
      EVUTIL_ASSERT(chain->off == 0);
      evbuffer_chain_free(chain);
    }
    EVUTIL_ASSERT(datlen >= avail);
    tmp = evbuffer_chain_new_membuf(datlen - avail);
    if (tmp == NULL) {
      if (rmv_all) {
        ZERO_CHAIN(buf);
      } else {
        buf->last = *buf->last_with_datap;
        (*buf->last_with_datap)->next = NULL;
      }
      return (-1);
    }

    if (rmv_all) {
      buf->first = buf->last = tmp;
      buf->last_with_datap = &buf->first;
    } else {
      (*buf->last_with_datap)->next = tmp;
      buf->last = tmp;
    }
    return (0);
  }
}

int
evbuffer_expand(struct evbuffer *buf, size_t datlen)
{
  struct evbuffer_chain *chain;

  EVBUFFER_LOCK(buf);
  chain = evbuffer_expand_singlechain(buf, datlen);
  EVBUFFER_UNLOCK(buf);
  return chain ? 0 : -1;
}

/*
 * Reads data from a file descriptor into a buffer.
 */

#if defined(EVENT__HAVE_SYS_UIO_H) || defined(_WIN32)
#define USE_IOVEC_IMPL
#endif

#ifdef USE_IOVEC_IMPL

#ifdef EVENT__HAVE_SYS_UIO_H
/* number of iovec we use for writev, fragmentation is going to determine
 * how much we end up writing */

#define DEFAULT_WRITE_IOVEC 128

#if defined(UIO_MAXIOV) && UIO_MAXIOV < DEFAULT_WRITE_IOVEC
#define NUM_WRITE_IOVEC UIO_MAXIOV
#elif defined(IOV_MAX) && IOV_MAX < DEFAULT_WRITE_IOVEC
#define NUM_WRITE_IOVEC IOV_MAX
#else
#define NUM_WRITE_IOVEC DEFAULT_WRITE_IOVEC
#endif

#define IOV_TYPE struct iovec
#define IOV_PTR_FIELD iov_base
#define IOV_LEN_FIELD iov_len
#define IOV_LEN_TYPE size_t
#else
#define NUM_WRITE_IOVEC 16
#define IOV_TYPE WSABUF
#define IOV_PTR_FIELD buf
#define IOV_LEN_FIELD len
#define IOV_LEN_TYPE unsigned long
#endif
#endif
#define NUM_READ_IOVEC 4

/** Helper function to figure out which space to use for reading data into
    an evbuffer.  Internal use only.

    @param buf The buffer to read into
    @param howmuch How much we want to read.
    @param vecs An array of two or more iovecs or WSABUFs.
    @param n_vecs_avail The length of vecs
    @param chainp A pointer to a variable to hold the first chain we're
      reading into.
    @param exact Boolean: if true, we do not provide more than 'howmuch'
      space in the vectors, even if more space is available.
    @return The number of buffers we're using.
 */
int
evbuffer_read_setup_vecs_(struct evbuffer *buf, ev_ssize_t howmuch,
    struct evbuffer_iovec *vecs, int n_vecs_avail,
    struct evbuffer_chain ***chainp, int exact)
{
  struct evbuffer_chain *chain;
  struct evbuffer_chain **firstchainp;
  size_t so_far;
  int i;
  ASSERT_EVBUFFER_LOCKED(buf);

  if (howmuch < 0)
    return -1;

  so_far = 0;
  /* Let firstchain be the first chain with any space on it */
  firstchainp = buf->last_with_datap;
  EVUTIL_ASSERT(*firstchainp);
  if (CHAIN_SPACE_LEN(*firstchainp) == 0) {
    firstchainp = &(*firstchainp)->next;
  }

  chain = *firstchainp;
  EVUTIL_ASSERT(chain);
  for (i = 0; i < n_vecs_avail && so_far < (size_t)howmuch; ++i) {
    size_t avail = (size_t) CHAIN_SPACE_LEN(chain);
    if (avail > (howmuch - so_far) && exact)
      avail = howmuch - so_far;
    vecs[i].iov_base = (void *)CHAIN_SPACE_PTR(chain);
    vecs[i].iov_len = avail;
    so_far += avail;
    chain = chain->next;
  }

  *chainp = firstchainp;
  return i;
}

static int
get_n_bytes_readable_on_socket(evutil_socket_t fd)
{
#if defined(FIONREAD) && defined(_WIN32)
  unsigned long lng = EVBUFFER_MAX_READ_DEFAULT;
  if (ioctlsocket(fd, FIONREAD, &lng) < 0)
    return -1;
  /* Can overflow, but mostly harmlessly. XXXX */
  return (int)lng;
#elif defined(FIONREAD)
  int n = EVBUFFER_MAX_READ_DEFAULT;
  if (ioctl(fd, FIONREAD, &n) < 0)
    return -1;
  return n;
#else
  return EVBUFFER_MAX_READ_DEFAULT;
#endif
}

/* TODO(niels): should this function return ev_ssize_t and take ev_ssize_t
 * as howmuch? */
int
evbuffer_read(struct evbuffer *buf, evutil_socket_t fd, int howmuch)
{
  int n;
  int result;

#ifdef USE_IOVEC_IMPL
  struct evbuffer_chain **chainp;
  int nvecs, i, remaining;
#else
  struct evbuffer_chain *chain;
  unsigned char *p;
#endif

  EVBUFFER_LOCK(buf);

  if (buf->freeze_end) {
    result = -1;
    goto done;
  }

  n = get_n_bytes_readable_on_socket(fd);
  if (n <= 0 || n > (int)buf->max_read)
    n = (int)buf->max_read;
  if (howmuch < 0 || howmuch > n)
    howmuch = n;

#ifdef USE_IOVEC_IMPL
  /* Since we can use iovecs, we're willing to use the last
   * NUM_READ_IOVEC chains. */
  if (evbuffer_expand_fast_(buf, howmuch, NUM_READ_IOVEC) == -1) {
    result = -1;
    goto done;
  } else {
    IOV_TYPE vecs[NUM_READ_IOVEC];
#ifdef EVBUFFER_IOVEC_IS_NATIVE_
    nvecs = evbuffer_read_setup_vecs_(buf, howmuch, vecs,
        NUM_READ_IOVEC, &chainp, 1);
#else
    /* We aren't using the native struct iovec.  Therefore,
       we are on win32. */
    struct evbuffer_iovec ev_vecs[NUM_READ_IOVEC];
    nvecs = evbuffer_read_setup_vecs_(buf, howmuch, ev_vecs, 2,
        &chainp, 1);

    for (i=0; i < nvecs; ++i)
      WSABUF_FROM_EVBUFFER_IOV(&vecs[i], &ev_vecs[i]);
#endif

#ifdef _WIN32
    {
      DWORD bytesRead;
      DWORD flags=0;
      if (WSARecv(fd, vecs, nvecs, &bytesRead, &flags, NULL, NULL)) {
        /* The read failed. It might be a close,
         * or it might be an error. */
        if (WSAGetLastError() == WSAECONNABORTED)
          n = 0;
        else
          n = -1;
      } else
        n = bytesRead;
    }
#else
    /* TODO(panjf2000): wrap it with `unlikely` as compiler hint? */
    if (nvecs == 1)
      n = read(fd, vecs[0].IOV_PTR_FIELD, vecs[0].IOV_LEN_FIELD);
    else
      n = readv(fd, vecs, nvecs);
#endif
  }

#else /* !USE_IOVEC_IMPL */
  /* If we don't have FIONREAD, we might waste some space here */
  /* XXX we _will_ waste some space here if there is any space left
   * over on buf->last. */
  if ((chain = evbuffer_expand_singlechain(buf, howmuch)) == NULL) {
    result = -1;
    goto done;
  }

  /* We can append new data at this point */
  p = chain->buffer + chain->misalign + chain->off;

#ifndef _WIN32
  n = read(fd, p, howmuch);
#else
  n = recv(fd, p, howmuch, 0);
#endif
#endif /* USE_IOVEC_IMPL */

  if (n == -1) {
    result = -1;
    goto done;
  }
  if (n == 0) {
    result = 0;
    goto done;
  }

#ifdef USE_IOVEC_IMPL
  remaining = n;
  for (i=0; i < nvecs; ++i) {
    /* can't overflow, since only mutable chains have
     * huge misaligns. */
    size_t space = (size_t) CHAIN_SPACE_LEN(*chainp);
    /* XXXX This is a kludge that can waste space in perverse
     * situations. */
    if (space > EVBUFFER_CHAIN_MAX)
      space = EVBUFFER_CHAIN_MAX;
    if ((ev_ssize_t)space < remaining) {
      (*chainp)->off += space;
      remaining -= (int)space;
    } else {
      (*chainp)->off += remaining;
      buf->last_with_datap = chainp;
      break;
    }
    chainp = &(*chainp)->next;
  }
#else
  chain->off += n;
  advance_last_with_data(buf);
#endif
  buf->total_len += n;
  buf->n_add_for_cb += n;

  /* Tell someone about changes in this buffer */
  evbuffer_invoke_callbacks_(buf);
  result = n;
done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

#ifdef USE_IOVEC_IMPL
static inline int
evbuffer_write_iovec(struct evbuffer *buffer, evutil_socket_t fd,
    ev_ssize_t howmuch)
{
  IOV_TYPE iov[NUM_WRITE_IOVEC];
  struct evbuffer_chain *chain = buffer->first;
  int n, i = 0;

  if (howmuch < 0)
    return -1;

  ASSERT_EVBUFFER_LOCKED(buffer);
  /* XXX make this top out at some maximal data length?  if the
   * buffer has (say) 1MB in it, split over 128 chains, there's
   * no way it all gets written in one go. */
  while (chain != NULL && i < NUM_WRITE_IOVEC && howmuch) {
#ifdef USE_SENDFILE
    /* we cannot write the file info via writev */
    if (chain->flags & EVBUFFER_SENDFILE)
      break;
#endif
    iov[i].IOV_PTR_FIELD = (void *) (chain->buffer + chain->misalign);
    if ((size_t)howmuch >= chain->off) {
      /* XXXcould be problematic when windows supports mmap*/
      iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)chain->off;
      howmuch -= chain->off;
    } else {
      /* XXXcould be problematic when windows supports mmap*/
      iov[i++].IOV_LEN_FIELD = (IOV_LEN_TYPE)howmuch;
      break;
    }
    chain = chain->next;
  }
  if (! i)
    return 0;

#ifdef _WIN32
  {
    DWORD bytesSent;
    if (WSASend(fd, iov, i, &bytesSent, 0, NULL, NULL))
      n = -1;
    else
      n = bytesSent;
  }
#else
  /* TODO(panjf2000): wrap it with `unlikely` as compiler hint? */
  if (i == 1)
    n = write(fd, iov[0].IOV_PTR_FIELD, iov[0].IOV_LEN_FIELD);
  else
    n = writev(fd, iov, i);
#endif
  return (n);
}
#endif

#ifdef USE_SENDFILE
static inline int
evbuffer_write_sendfile(struct evbuffer *buffer, evutil_socket_t dest_fd,
    ev_ssize_t howmuch)
{
  struct evbuffer_chain *chain = buffer->first;
  struct evbuffer_chain_file_segment *info =
      EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment,
    chain);
  const int source_fd = info->segment->fd;
#if defined(SENDFILE_IS_MACOSX) || defined(SENDFILE_IS_FREEBSD)
  int res;
  ev_off_t len = chain->off;
#elif defined(SENDFILE_IS_LINUX) || defined(SENDFILE_IS_SOLARIS)
  ev_ssize_t res;
  off_t offset = chain->misalign;
#endif

  ASSERT_EVBUFFER_LOCKED(buffer);

#if defined(SENDFILE_IS_MACOSX)
  res = sendfile(source_fd, dest_fd, chain->misalign, &len, NULL, 0);
  if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
    return (-1);

  return (len);
#elif defined(SENDFILE_IS_FREEBSD)
  res = sendfile(source_fd, dest_fd, chain->misalign, chain->off, NULL, &len, 0);
  if (res == -1 && !EVUTIL_ERR_RW_RETRIABLE(errno))
    return (-1);

  return (len);
#elif defined(SENDFILE_IS_LINUX)
  res = sendfile(dest_fd, source_fd, &offset, chain->off);
  if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
    /* if this is EAGAIN or EINTR return 0; otherwise, -1 */
    return (0);
  }
  return (res);
#elif defined(SENDFILE_IS_SOLARIS)
  {
    const off_t offset_orig = offset;
    res = sendfile(dest_fd, source_fd, &offset, chain->off);
    if (res == -1 && EVUTIL_ERR_RW_RETRIABLE(errno)) {
      if (offset - offset_orig)
        return offset - offset_orig;
      /* if this is EAGAIN or EINTR and no bytes were
       * written, return 0 */
      return (0);
    }
    return (res);
  }
#endif
}
#endif

int
evbuffer_write_atmost(struct evbuffer *buffer, evutil_socket_t fd,
    ev_ssize_t howmuch)
{
  int n = -1;

  EVBUFFER_LOCK(buffer);

  if (buffer->freeze_start) {
    goto done;
  }

  if (howmuch < 0 || (size_t)howmuch > buffer->total_len)
    howmuch = buffer->total_len;

  if (howmuch > 0) {
#ifdef USE_SENDFILE
    struct evbuffer_chain *chain = buffer->first;
    if (chain != NULL && (chain->flags & EVBUFFER_SENDFILE))
      n = evbuffer_write_sendfile(buffer, fd, howmuch);
    else {
#endif
#ifdef USE_IOVEC_IMPL
    n = evbuffer_write_iovec(buffer, fd, howmuch);
#elif defined(_WIN32)
    /* XXX(nickm) Don't disable this code until we know if
     * the WSARecv code above works. */
    void *p = evbuffer_pullup(buffer, howmuch);
    EVUTIL_ASSERT(p || !howmuch);
    n = send(fd, p, howmuch, 0);
#else
    void *p = evbuffer_pullup(buffer, howmuch);
    EVUTIL_ASSERT(p || !howmuch);
    n = write(fd, p, howmuch);
#endif
#ifdef USE_SENDFILE
    }
#endif
  }

  if (n > 0)
    evbuffer_drain(buffer, n);

done:
  EVBUFFER_UNLOCK(buffer);
  return (n);
}

int
evbuffer_write(struct evbuffer *buffer, evutil_socket_t fd)
{
  return evbuffer_write_atmost(buffer, fd, -1);
}

unsigned char *
evbuffer_find(struct evbuffer *buffer, const unsigned char *what, size_t len)
{
  unsigned char *search;
  struct evbuffer_ptr ptr;

  EVBUFFER_LOCK(buffer);

  ptr = evbuffer_search(buffer, (const char *)what, len, NULL);
  if (ptr.pos < 0) {
    search = NULL;
  } else {
    search = evbuffer_pullup(buffer, ptr.pos + len);
    if (search)
      search += ptr.pos;
  }
  EVBUFFER_UNLOCK(buffer);
  return search;
}

/* Subract <b>howfar</b> from the position of <b>pos</b> within
 * <b>buf</b>. Returns 0 on success, -1 on failure.
 *
 * This isn't exposed yet, because of potential inefficiency issues.
 * Maybe it should be. */
static int
evbuffer_ptr_subtract(struct evbuffer *buf, struct evbuffer_ptr *pos,
    size_t howfar)
{
  if (pos->pos < 0)
    return -1;
  if (howfar > (size_t)pos->pos)
    return -1;
  if (pos->internal_.chain && howfar <= pos->internal_.pos_in_chain) {
    pos->internal_.pos_in_chain -= howfar;
    pos->pos -= howfar;
    return 0;
  } else {
    const size_t newpos = pos->pos - howfar;
    /* Here's the inefficient part: it walks over the
     * chains until we hit newpos. */
    return evbuffer_ptr_set(buf, pos, newpos, EVBUFFER_PTR_SET);
  }
}

int
evbuffer_ptr_set(struct evbuffer *buf, struct evbuffer_ptr *pos,
    size_t position, enum evbuffer_ptr_how how)
{
  size_t left = position;
  struct evbuffer_chain *chain = NULL;
  int result = 0;

  EVBUFFER_LOCK(buf);

  switch (how) {
  case EVBUFFER_PTR_SET:
    chain = buf->first;
    pos->pos = position;
    position = 0;
    break;
  case EVBUFFER_PTR_ADD:
    /* this avoids iterating over all previous chains if
       we just want to advance the position */
    if (pos->pos < 0 || EV_SIZE_MAX - position < (size_t)pos->pos) {
      EVBUFFER_UNLOCK(buf);
      return -1;
    }
    chain = pos->internal_.chain;
    pos->pos += position;
    position = pos->internal_.pos_in_chain;
    break;
  }

  EVUTIL_ASSERT(EV_SIZE_MAX - left >= position);
  while (chain && position + left >= chain->off) {
    left -= chain->off - position;
    chain = chain->next;
    position = 0;
  }
  if (chain) {
    pos->internal_.chain = chain;
    pos->internal_.pos_in_chain = position + left;
  } else if (left == 0) {
    /* The first byte in the (nonexistent) chain after the last chain */
    pos->internal_.chain = NULL;
    pos->internal_.pos_in_chain = 0;
  } else {
    PTR_NOT_FOUND(pos);
    result = -1;
  }

  EVBUFFER_UNLOCK(buf);

  return result;
}

/**
   Compare the bytes in buf at position pos to the len bytes in mem.  Return
   less than 0, 0, or greater than 0 as memcmp.
 */
static int
evbuffer_ptr_memcmp(const struct evbuffer *buf, const struct evbuffer_ptr *pos,
    const char *mem, size_t len)
{
  struct evbuffer_chain *chain;
  size_t position;
  int r;

  ASSERT_EVBUFFER_LOCKED(buf);

  if (pos->pos < 0 ||
      EV_SIZE_MAX - len < (size_t)pos->pos ||
      pos->pos + len > buf->total_len)
    return -1;

  chain = pos->internal_.chain;
  position = pos->internal_.pos_in_chain;
  while (len && chain) {
    size_t n_comparable;
    if (len + position > chain->off)
      n_comparable = chain->off - position;
    else
      n_comparable = len;
    r = memcmp(chain->buffer + chain->misalign + position, mem,
        n_comparable);
    if (r)
      return r;
    mem += n_comparable;
    len -= n_comparable;
    position = 0;
    chain = chain->next;
  }

  return 0;
}

struct evbuffer_ptr
evbuffer_search(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start)
{
  return evbuffer_search_range(buffer, what, len, start, NULL);
}

struct evbuffer_ptr
evbuffer_search_range(struct evbuffer *buffer, const char *what, size_t len, const struct evbuffer_ptr *start, const struct evbuffer_ptr *end)
{
  struct evbuffer_ptr pos;
  struct evbuffer_chain *chain, *last_chain = NULL;
  const unsigned char *p;
  char first;

  EVBUFFER_LOCK(buffer);

  if (start) {
    memcpy(&pos, start, sizeof(pos));
    chain = pos.internal_.chain;
  } else {
    pos.pos = 0;
    chain = pos.internal_.chain = buffer->first;
    pos.internal_.pos_in_chain = 0;
  }

  if (end)
    last_chain = end->internal_.chain;

  if (!len || len > EV_SSIZE_MAX)
    goto done;

  first = what[0];

  while (chain) {
    const unsigned char *start_at =
        chain->buffer + chain->misalign +
        pos.internal_.pos_in_chain;
    p = memchr(start_at, first,
        chain->off - pos.internal_.pos_in_chain);
    if (p) {
      pos.pos += p - start_at;
      pos.internal_.pos_in_chain += p - start_at;
      if (!evbuffer_ptr_memcmp(buffer, &pos, what, len)) {
        if (end && pos.pos + (ev_ssize_t)len > end->pos)
          goto not_found;
        else
          goto done;
      }
      ++pos.pos;
      ++pos.internal_.pos_in_chain;
      if (pos.internal_.pos_in_chain == chain->off) {
        chain = pos.internal_.chain = chain->next;
        pos.internal_.pos_in_chain = 0;
      }
    } else {
      if (chain == last_chain)
        goto not_found;
      pos.pos += chain->off - pos.internal_.pos_in_chain;
      chain = pos.internal_.chain = chain->next;
      pos.internal_.pos_in_chain = 0;
    }
  }

not_found:
  PTR_NOT_FOUND(&pos);
done:
  EVBUFFER_UNLOCK(buffer);
  return pos;
}

int
evbuffer_peek(struct evbuffer *buffer, ev_ssize_t len,
    struct evbuffer_ptr *start_at,
    struct evbuffer_iovec *vec, int n_vec)
{
  struct evbuffer_chain *chain;
  int idx = 0;
  ev_ssize_t len_so_far = 0;

  /* Avoid locking in trivial edge cases */
  if (start_at && start_at->internal_.chain == NULL)
    return 0;

  EVBUFFER_LOCK(buffer);

  if (start_at) {
    chain = start_at->internal_.chain;
    len_so_far = chain->off
        - start_at->internal_.pos_in_chain;
    idx = 1;
    if (n_vec > 0) {
      vec[0].iov_base = (void *)(chain->buffer + chain->misalign
          + start_at->internal_.pos_in_chain);
      vec[0].iov_len = len_so_far;
    }
    chain = chain->next;
  } else {
    chain = buffer->first;
  }

  if (n_vec == 0 && len < 0) {
    /* If no vectors are provided and they asked for "everything",
     * pretend they asked for the actual available amount. */
    len = buffer->total_len;
    if (start_at) {
      len -= start_at->pos;
    }
  }

  while (chain) {
    if (len >= 0 && len_so_far >= len)
      break;
    if (idx<n_vec) {
      vec[idx].iov_base = (void *)(chain->buffer + chain->misalign);
      vec[idx].iov_len = chain->off;
    } else if (len<0) {
      break;
    }
    ++idx;
    len_so_far += chain->off;
    chain = chain->next;
  }

  EVBUFFER_UNLOCK(buffer);

  return idx;
}


int
evbuffer_add_vprintf(struct evbuffer *buf, const char *fmt, va_list ap)
{
  char *buffer;
  size_t space;
  int sz, result = -1;
  va_list aq;
  struct evbuffer_chain *chain;


  EVBUFFER_LOCK(buf);

  if (buf->freeze_end) {
    goto done;
  }

  /* make sure that at least some space is available */
  if ((chain = evbuffer_expand_singlechain(buf, 64)) == NULL)
    goto done;

  for (;;) {
#if 0
    size_t used = chain->misalign + chain->off;
    buffer = (char *)chain->buffer + chain->misalign + chain->off;
    EVUTIL_ASSERT(chain->buffer_len >= used);
    space = chain->buffer_len - used;
#endif
    buffer = (char*) CHAIN_SPACE_PTR(chain);
    space = (size_t) CHAIN_SPACE_LEN(chain);

#ifndef va_copy
#define va_copy(dst, src) memcpy(&(dst), &(src), sizeof(va_list))
#endif
    va_copy(aq, ap);

    sz = evutil_vsnprintf(buffer, space, fmt, aq);

    va_end(aq);

    if (sz < 0)
      goto done;
    if (INT_MAX >= EVBUFFER_CHAIN_MAX &&
        (size_t)sz >= EVBUFFER_CHAIN_MAX)
      goto done;
    if ((size_t)sz < space) {
      chain->off += sz;
      buf->total_len += sz;
      buf->n_add_for_cb += sz;

      advance_last_with_data(buf);
      evbuffer_invoke_callbacks_(buf);
      result = sz;
      goto done;
    }
    if ((chain = evbuffer_expand_singlechain(buf, sz + 1)) == NULL)
      goto done;
  }
  /* NOTREACHED */

done:
  EVBUFFER_UNLOCK(buf);
  return result;
}

int
evbuffer_add_printf(struct evbuffer *buf, const char *fmt, ...)
{
  int res = -1;
  va_list ap;

  va_start(ap, fmt);
  res = evbuffer_add_vprintf(buf, fmt, ap);
  va_end(ap);

  return (res);
}

int
evbuffer_add_reference(struct evbuffer *outbuf,
    const void *data, size_t datlen,
    evbuffer_ref_cleanup_cb cleanupfn, void *extra)
{
  return evbuffer_add_reference_with_offset(outbuf, data, /* offset= */ 0, datlen, cleanupfn, extra);
}

int
evbuffer_add_reference_with_offset(struct evbuffer *outbuf, const void *data,
  size_t offset, size_t datlen, evbuffer_ref_cleanup_cb cleanupfn,
  void *extra)
{
  struct evbuffer_chain *chain;
  struct evbuffer_chain_reference *info;
  int result = -1;

  chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_reference));
  if (!chain)
    return (-1);
  chain->flags |= EVBUFFER_REFERENCE | EVBUFFER_IMMUTABLE;
  chain->buffer = (unsigned char *)data;
  chain->misalign = offset;
  chain->buffer_len = offset + datlen;
  chain->off = datlen;

  info = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_reference, chain);
  info->cleanupfn = cleanupfn;
  info->extra = extra;

  EVBUFFER_LOCK(outbuf);
  if (outbuf->freeze_end) {
    /* don't call chain_free; we do not want to actually invoke
     * the cleanup function */
    mm_free(chain);
    goto done;
  }
  evbuffer_chain_insert(outbuf, chain);
  outbuf->n_add_for_cb += datlen;

  evbuffer_invoke_callbacks_(outbuf);

  result = 0;
done:
  EVBUFFER_UNLOCK(outbuf);

  return result;
}

/* TODO(niels): we may want to add to automagically convert to mmap, in
 * case evbuffer_remove() or evbuffer_pullup() are being used.
 */
struct evbuffer_file_segment *
evbuffer_file_segment_new(
  int fd, ev_off_t offset, ev_off_t length, unsigned flags)
{
  struct evbuffer_file_segment *seg =
      mm_calloc(1, sizeof(struct evbuffer_file_segment));
  if (!seg)
    return NULL;
  seg->refcnt = 1;
  seg->fd = fd;
  seg->flags = flags;
  seg->file_offset = offset;
  seg->cleanup_cb = NULL;
  seg->cleanup_cb_arg = NULL;
  if (length == -1) {
    length = evutil_fd_filesize(fd);
    if (length == -1)
      goto err;
  }
  seg->length = length;

  if (offset < 0 || length < 0 ||
      ((ev_uint64_t)length > EVBUFFER_CHAIN_MAX) ||
      (ev_uint64_t)offset > (ev_uint64_t)(EVBUFFER_CHAIN_MAX - length))
    goto err;

#if defined(USE_SENDFILE)
  if (!(flags & EVBUF_FS_DISABLE_SENDFILE)) {
    seg->can_sendfile = 1;
    goto done;
  }
#endif

  if (evbuffer_file_segment_materialize(seg)<0)
    goto err;

#if defined(USE_SENDFILE)
done:
#endif
  if (!(flags & EVBUF_FS_DISABLE_LOCKING)) {
    EVTHREAD_ALLOC_LOCK(seg->lock, 0);
  }
  return seg;
err:
  mm_free(seg);
  return NULL;
}

#ifdef EVENT__HAVE_MMAP
static long
get_page_size(void)
{
#ifdef SC_PAGE_SIZE
  return sysconf(SC_PAGE_SIZE);
#elif defined(_SC_PAGE_SIZE)
  return sysconf(_SC_PAGE_SIZE);
#else
  return 1;
#endif
}
#endif

/* DOCDOC */
/* Requires lock */
static int
evbuffer_file_segment_materialize(struct evbuffer_file_segment *seg)
{
#if defined(EVENT__HAVE_MMAP) || defined(_WIN32)
  const unsigned flags = seg->flags;
#endif
  const int fd = seg->fd;
  const ev_off_t length = seg->length;
  const ev_off_t offset = seg->file_offset;

  if (seg->contents || seg->is_mapping)
    return 0; /* already materialized */

#if defined(EVENT__HAVE_MMAP)
  if (!(flags & EVBUF_FS_DISABLE_MMAP)) {
    off_t offset_rounded = 0, offset_leftover = 0;
    int mmap_flags =
#ifdef MAP_NOCACHE
      MAP_NOCACHE | /* ??? */
#endif
#ifdef MAP_FILE
      MAP_FILE |
#endif
      MAP_PRIVATE;
    void *mapped;
    if (offset) {
      /* mmap implementations don't generally like us
       * to have an offset that isn't a round  */
      long page_size = get_page_size();
      if (page_size == -1)
        goto err;
      offset_leftover = offset % page_size;
      offset_rounded = offset - offset_leftover;
    }
#if defined(EVENT__HAVE_MMAP64)
    mapped = mmap64(NULL, length + offset_leftover,
#else
    mapped = mmap(NULL, length + offset_leftover,
#endif
      PROT_READ, mmap_flags, fd, offset_rounded);
    if (mapped == MAP_FAILED) {
      event_warn("%s: mmap(NULL, %zu, %d, %d, %d, %lld) failed", __func__,
        (size_t)(length + offset_leftover), PROT_READ, mmap_flags, fd,
        (long long)offset_rounded);
    } else {
      seg->mapping = mapped;
      seg->contents = (char*)mapped+offset_leftover;
      seg->mmap_offset = 0;
      seg->is_mapping = 1;
      goto done;
    }
  }
#endif
#ifdef _WIN32
  if (!(flags & EVBUF_FS_DISABLE_MMAP)) {
    intptr_t h = _get_osfhandle(fd);
    HANDLE m;
    ev_uint64_t total_size = length+offset;
    if ((HANDLE)h == INVALID_HANDLE_VALUE)
      goto err;
    m = CreateFileMapping((HANDLE)h, NULL, PAGE_READONLY,
        (total_size >> 32), total_size & 0xfffffffful,
        NULL);
    if (m != INVALID_HANDLE_VALUE) { /* Does h leak? */
      seg->mapping_handle = m;
      seg->mmap_offset = offset;
      seg->is_mapping = 1;
      goto done;
    }
  }
#endif
  {
    ev_off_t read_so_far = 0;
    ev_ssize_t n = 0;
    char *mem;
#ifndef EVENT__HAVE_PREAD
#ifdef _WIN32
#ifndef lseek
#define lseek _lseeki64
#endif
#endif
    ev_off_t start_pos = lseek(fd, 0, SEEK_CUR);
    ev_off_t pos;
    int e;
#endif /* no pread() */
    if (!(mem = mm_malloc(length)))
      goto err;
#ifdef EVENT__HAVE_PREAD
    while (read_so_far < length) {
      n = pread(fd, mem + read_so_far, length - read_so_far,
          offset + read_so_far);
      if (n <= 0)
        break;
      read_so_far += n;
    }
    if (n < 0 || (n == 0 && length > read_so_far)) {
      mm_free(mem);
      goto err;
    }
#else /* fallback to seek() and read() */
    if (start_pos < 0) {
      mm_free(mem);
      goto err;
    }
    if (lseek(fd, offset, SEEK_SET) < 0) {
      mm_free(mem);
      goto err;
    }
    while (read_so_far < length) {
      n = read(fd, mem+read_so_far, length-read_so_far);
      if (n <= 0)
        break;
      read_so_far += n;
    }

    e = errno;
    pos = lseek(fd, start_pos, SEEK_SET);
    if (n < 0 || (n == 0 && length > read_so_far)) {
      mm_free(mem);
      errno = e;
      goto err;
    } else if (pos < 0) {
      mm_free(mem);
      goto err;
    }
#endif /* pread */

    seg->contents = mem;
  }
#if defined(EVENT__HAVE_MMAP) || defined(_WIN32)
done:
#endif
  return 0;
err:
  return -1;
}

void evbuffer_file_segment_add_cleanup_cb(struct evbuffer_file_segment *seg,
  evbuffer_file_segment_cleanup_cb cb, void* arg)
{
  EVUTIL_ASSERT(seg->refcnt > 0);
  seg->cleanup_cb = cb;
  seg->cleanup_cb_arg = arg;
}

void
evbuffer_file_segment_free(struct evbuffer_file_segment *seg)
{
  int refcnt;
  EVLOCK_LOCK(seg->lock, 0);
  refcnt = --seg->refcnt;
  EVLOCK_UNLOCK(seg->lock, 0);
  if (refcnt > 0)
    return;
  EVUTIL_ASSERT(refcnt == 0);

  if (seg->is_mapping) {
#ifdef _WIN32
    CloseHandle(seg->mapping_handle);
#elif defined (EVENT__HAVE_MMAP)
    off_t offset_leftover;
    offset_leftover = seg->file_offset % get_page_size();
    if (munmap(seg->mapping, seg->length + offset_leftover) == -1)
      event_warn("%s: munmap failed", __func__);
#endif
  } else if (seg->contents) {
    mm_free(seg->contents);
  }

  if ((seg->flags & EVBUF_FS_CLOSE_ON_FREE) && seg->fd >= 0) {
    close(seg->fd);
  }

  if (seg->cleanup_cb) {
    (*seg->cleanup_cb)((struct evbuffer_file_segment const*)seg,
        seg->flags, seg->cleanup_cb_arg);
    seg->cleanup_cb = NULL;
    seg->cleanup_cb_arg = NULL;
  }

  EVTHREAD_FREE_LOCK(seg->lock, 0);
  mm_free(seg);
}

int
evbuffer_add_file_segment(struct evbuffer *buf,
    struct evbuffer_file_segment *seg, ev_off_t offset, ev_off_t length)
{
  struct evbuffer_chain *chain;
  struct evbuffer_chain_file_segment *extra;
  int can_use_sendfile = 0;

  EVBUFFER_LOCK(buf);
  EVLOCK_LOCK(seg->lock, 0);
  if (buf->flags & EVBUFFER_FLAG_DRAINS_TO_FD) {
    can_use_sendfile = 1;
  } else {
    if (evbuffer_file_segment_materialize(seg)<0) {
      EVLOCK_UNLOCK(seg->lock, 0);
      goto err;
    }
  }
  EVLOCK_UNLOCK(seg->lock, 0);

  if (buf->freeze_end)
    goto err;

  if (length < 0) {
    if (offset > seg->length)
      goto err;
    length = seg->length - offset;
  }

  /* Can we actually add this? */
  if (offset+length > seg->length)
    goto err;

  chain = evbuffer_chain_new(sizeof(struct evbuffer_chain_file_segment));
  if (!chain)
    goto err;
  extra = EVBUFFER_CHAIN_EXTRA(struct evbuffer_chain_file_segment, chain);

  chain->flags |= EVBUFFER_IMMUTABLE|EVBUFFER_FILESEGMENT;
  if (can_use_sendfile && seg->can_sendfile) {
    chain->flags |= EVBUFFER_SENDFILE;
    chain->misalign = seg->file_offset + offset;
    chain->off = length;
    chain->buffer_len = chain->misalign + length;
  } else if (seg->is_mapping) {
#ifdef _WIN32
    ev_uint64_t total_offset = seg->mmap_offset+offset;
    ev_uint64_t offset_rounded=0, offset_remaining=0;
    LPVOID data;
    if (total_offset) {
      SYSTEM_INFO si;
      memset(&si, 0, sizeof(si)); /* cargo cult */
      GetSystemInfo(&si);
      offset_remaining = total_offset % si.dwAllocationGranularity;
      offset_rounded = total_offset - offset_remaining;
    }
    data = MapViewOfFile(
      seg->mapping_handle,
      FILE_MAP_READ,
      offset_rounded >> 32,
      offset_rounded & 0xfffffffful,
      length + offset_remaining);
    if (data == NULL) {
      mm_free(chain);
      goto err;
    }
    chain->buffer = (unsigned char*) data;
    chain->buffer_len = length+offset_remaining;
    chain->misalign = offset_remaining;
    chain->off = length;
#else
    chain->buffer = (unsigned char*)(seg->contents + offset);
    chain->buffer_len = length;
    chain->off = length;
#endif
  } else {
    chain->buffer = (unsigned char*)(seg->contents + offset);
    chain->buffer_len = length;
    chain->off = length;
  }

  EVLOCK_LOCK(seg->lock, 0);
  ++seg->refcnt;
  EVLOCK_UNLOCK(seg->lock, 0);
  extra->segment = seg;
  buf->n_add_for_cb += length;
  evbuffer_chain_insert(buf, chain);

  evbuffer_invoke_callbacks_(buf);

  EVBUFFER_UNLOCK(buf);

  return 0;
err:
  EVBUFFER_UNLOCK(buf);
  evbuffer_file_segment_free(seg); /* Lowers the refcount */
  return -1;
}

int
evbuffer_add_file(struct evbuffer *buf, int fd, ev_off_t offset, ev_off_t length)
{
  struct evbuffer_file_segment *seg;
  unsigned flags = EVBUF_FS_CLOSE_ON_FREE;
  int r;

  seg = evbuffer_file_segment_new(fd, offset, length, flags);
  if (!seg)
    return -1;
  r = evbuffer_add_file_segment(buf, seg, 0, length);
  if (r == 0)
    evbuffer_file_segment_free(seg);
  return r;
}

int
evbuffer_setcb(struct evbuffer *buffer, evbuffer_cb cb, void *cbarg)
{
  EVBUFFER_LOCK(buffer);

  if (!LIST_EMPTY(&buffer->callbacks))
    evbuffer_remove_all_callbacks(buffer);

  if (cb) {
    struct evbuffer_cb_entry *ent =
        evbuffer_add_cb(buffer, NULL, cbarg);
    if (!ent) {
      EVBUFFER_UNLOCK(buffer);
      return -1;
    }
    ent->cb.cb_obsolete = cb;
    ent->flags |= EVBUFFER_CB_OBSOLETE;
  }
  EVBUFFER_UNLOCK(buffer);
  return 0;
}

struct evbuffer_cb_entry *
evbuffer_add_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
{
  struct evbuffer_cb_entry *e;
  if (! (e = mm_calloc(1, sizeof(struct evbuffer_cb_entry))))
    return NULL;
  EVBUFFER_LOCK(buffer);
  e->cb.cb_func = cb;
  e->cbarg = cbarg;
  e->flags = EVBUFFER_CB_ENABLED;
  LIST_INSERT_HEAD(&buffer->callbacks, e, next);
  EVBUFFER_UNLOCK(buffer);
  return e;
}

int
evbuffer_remove_cb_entry(struct evbuffer *buffer,
       struct evbuffer_cb_entry *ent)
{
  EVBUFFER_LOCK(buffer);
  LIST_REMOVE(ent, next);
  EVBUFFER_UNLOCK(buffer);
  mm_free(ent);
  return 0;
}

int
evbuffer_remove_cb(struct evbuffer *buffer, evbuffer_cb_func cb, void *cbarg)
{
  struct evbuffer_cb_entry *cbent;
  int result = -1;
  EVBUFFER_LOCK(buffer);
  LIST_FOREACH(cbent, &buffer->callbacks, next) {
    if (cb == cbent->cb.cb_func && cbarg == cbent->cbarg) {
      result = evbuffer_remove_cb_entry(buffer, cbent);
      goto done;
    }
  }
done:
  EVBUFFER_UNLOCK(buffer);
  return result;
}

int
evbuffer_cb_set_flags(struct evbuffer *buffer,
          struct evbuffer_cb_entry *cb, ev_uint32_t flags)
{
  /* the user isn't allowed to mess with these. */
  flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
  EVBUFFER_LOCK(buffer);
  cb->flags |= flags;
  EVBUFFER_UNLOCK(buffer);
  return 0;
}

int
evbuffer_cb_clear_flags(struct evbuffer *buffer,
          struct evbuffer_cb_entry *cb, ev_uint32_t flags)
{
  /* the user isn't allowed to mess with these. */
  flags &= ~EVBUFFER_CB_INTERNAL_FLAGS;
  EVBUFFER_LOCK(buffer);
  cb->flags &= ~flags;
  EVBUFFER_UNLOCK(buffer);
  return 0;
}

int
evbuffer_freeze(struct evbuffer *buffer, int start)
{
  EVBUFFER_LOCK(buffer);
  if (start)
    buffer->freeze_start = 1;
  else
    buffer->freeze_end = 1;
  EVBUFFER_UNLOCK(buffer);
  return 0;
}

int
evbuffer_unfreeze(struct evbuffer *buffer, int start)
{
  EVBUFFER_LOCK(buffer);
  if (start)
    buffer->freeze_start = 0;
  else
    buffer->freeze_end = 0;
  EVBUFFER_UNLOCK(buffer);
  return 0;
}

#if 0
void
evbuffer_cb_suspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
{
  if (!(cb->flags & EVBUFFER_CB_SUSPENDED)) {
    cb->size_before_suspend = evbuffer_get_length(buffer);
    cb->flags |= EVBUFFER_CB_SUSPENDED;
  }
}

void
evbuffer_cb_unsuspend(struct evbuffer *buffer, struct evbuffer_cb_entry *cb)
{
  if ((cb->flags & EVBUFFER_CB_SUSPENDED)) {
    unsigned call = (cb->flags & EVBUFFER_CB_CALL_ON_UNSUSPEND);
    size_t sz = cb->size_before_suspend;
    cb->flags &= ~(EVBUFFER_CB_SUSPENDED|
             EVBUFFER_CB_CALL_ON_UNSUSPEND);
    cb->size_before_suspend = 0;
    if (call && (cb->flags & EVBUFFER_CB_ENABLED)) {
      cb->cb(buffer, sz, evbuffer_get_length(buffer), cb->cbarg);
    }
  }
}
#endif

int
evbuffer_get_callbacks_(struct evbuffer *buffer, struct event_callback **cbs,
    int max_cbs)
{
  int r = 0;
  EVBUFFER_LOCK(buffer);
  if (buffer->deferred_cbs) {
    if (max_cbs < 1) {
      r = -1;
      goto done;
    }
    cbs[0] = &buffer->deferred;
    r = 1;
  }
done:
  EVBUFFER_UNLOCK(buffer);
  return r;
}

Coverage Report

Created: 2025-07-11 06:57