Coverage Report

Created: 2025-12-31 06:49

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/usrsctp/usrsctplib/user_mbuf.c
Line
Count
Source
1
/*-
2
 * Copyright (c) 1982, 1986, 1988, 1993
3
 *      The Regents of the University of California.
4
 * All rights reserved.
5
 *
6
 * Redistribution and use in source and binary forms, with or without
7
 * modification, are permitted provided that the following conditions
8
 * are met:
9
 * 1. Redistributions of source code must retain the above copyright
10
 *    notice, this list of conditions and the following disclaimer.
11
 * 2. Redistributions in binary form must reproduce the above copyright
12
 *    notice, this list of conditions and the following disclaimer in the
13
 *    documentation and/or other materials provided with the distribution.
14
 * 3. Neither the name of the University nor the names of its contributors
15
 *    may be used to endorse or promote products derived from this software
16
 *    without specific prior written permission.
17
 *
18
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
19
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
22
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28
 * SUCH DAMAGE.
29
 *
30
 */
31
32
/*
33
 *  __Userspace__ version of /usr/src/sys/kern/kern_mbuf.c
34
 *  We are initializing two zones for Mbufs and Clusters.
35
 *
36
 */
37
38
#if defined(_WIN32) && defined(__MINGW32__)
39
#include <minmax.h>
40
#endif
41
42
#include <stdio.h>
43
#include <string.h>
44
/* #include <sys/param.h> This defines MSIZE 256 */
45
#if !defined(SCTP_SIMPLE_ALLOCATOR)
46
#include "umem.h"
47
#endif
48
#include "user_mbuf.h"
49
#include "user_environment.h"
50
#include "user_atomic.h"
51
#include "netinet/sctp_pcb.h"
52
53
#define KIPC_MAX_LINKHDR        4       /* int: max length of link header (see sys/sysclt.h) */
54
#define KIPC_MAX_PROTOHDR 5 /* int: max length of network header (see sys/sysclt.h)*/
55
int max_linkhdr = KIPC_MAX_LINKHDR;
56
int max_protohdr = KIPC_MAX_PROTOHDR; /* Size of largest protocol layer header. */
57
58
/*
59
 * Zones from which we allocate.
60
 */
61
sctp_zone_t zone_mbuf;
62
sctp_zone_t zone_clust;
63
sctp_zone_t zone_ext_refcnt;
64
65
/* __Userspace__ clust_mb_args will be passed as callback data to mb_ctor_clust
66
 * and mb_dtor_clust.
67
 * Note: I had to use struct clust_args as an encapsulation for an mbuf pointer.
68
 * struct mbuf * clust_mb_args; does not work.
69
 */
70
struct clust_args clust_mb_args;
71
72
73
/* __Userspace__
74
 * Local prototypes.
75
 */
76
static int  mb_ctor_mbuf(void *, void *, int);
77
static int      mb_ctor_clust(void *, void *, int);
78
static void mb_dtor_mbuf(void *,  void *);
79
static void mb_dtor_clust(void *, void *);
80
81
82
/***************** Functions taken from user_mbuf.h *************/
83
84
static int mbuf_constructor_dup(struct mbuf *m, int pkthdr, short type)
85
13.2M
{
86
13.2M
  int flags = pkthdr;
87
88
13.2M
  m->m_next = NULL;
89
13.2M
  m->m_nextpkt = NULL;
90
13.2M
  m->m_len = 0;
91
13.2M
  m->m_flags = flags;
92
13.2M
  m->m_type = type;
93
13.2M
  if (flags & M_PKTHDR) {
94
6.79M
    m->m_data = m->m_pktdat;
95
6.79M
    m->m_pkthdr.rcvif = NULL;
96
6.79M
    m->m_pkthdr.len = 0;
97
6.79M
    m->m_pkthdr.header = NULL;
98
6.79M
    m->m_pkthdr.csum_flags = 0;
99
6.79M
    m->m_pkthdr.csum_data = 0;
100
6.79M
    m->m_pkthdr.tso_segsz = 0;
101
6.79M
    m->m_pkthdr.ether_vtag = 0;
102
6.79M
    SLIST_INIT(&m->m_pkthdr.tags);
103
6.79M
  } else
104
6.45M
    m->m_data = m->m_dat;
105
106
13.2M
  return (0);
107
13.2M
}
108
109
/* __Userspace__ */
110
struct mbuf *
111
m_get(int how, short type)
112
6.45M
{
113
6.45M
  struct mbuf *mret;
114
6.45M
#if defined(SCTP_SIMPLE_ALLOCATOR)
115
6.45M
  struct mb_args mbuf_mb_args;
116
117
  /* The following setter function is not yet being enclosed within
118
   * #if USING_MBUF_CONSTRUCTOR - #endif, until I have thoroughly tested
119
   * mb_dtor_mbuf. See comment there
120
   */
121
6.45M
  mbuf_mb_args.flags = 0;
122
6.45M
  mbuf_mb_args.type = type;
123
6.45M
#endif
124
  /* Mbuf master zone, zone_mbuf, has already been
125
   * created in mbuf_initialize() */
126
6.45M
  mret = SCTP_ZONE_GET(zone_mbuf, struct mbuf);
127
6.45M
#if defined(SCTP_SIMPLE_ALLOCATOR)
128
6.45M
  mb_ctor_mbuf(mret, &mbuf_mb_args, 0);
129
6.45M
#endif
130
  /*mret =  ((struct mbuf *)umem_cache_alloc(zone_mbuf, UMEM_DEFAULT));*/
131
132
  /* There are cases when an object available in the current CPU's
133
   * loaded magazine and in those cases the object's constructor is not applied.
134
   * If that is the case, then we are duplicating constructor initialization here,
135
   * so that the mbuf is properly constructed before returning it.
136
   */
137
6.45M
  if (mret) {
138
#if USING_MBUF_CONSTRUCTOR
139
    if (! (mret->m_type == type) ) {
140
      mbuf_constructor_dup(mret, 0, type);
141
    }
142
#else
143
6.45M
    mbuf_constructor_dup(mret, 0, type);
144
6.45M
#endif
145
146
6.45M
  }
147
6.45M
  return mret;
148
6.45M
}
149
150
151
/* __Userspace__ */
152
struct mbuf *
153
m_gethdr(int how, short type)
154
6.79M
{
155
6.79M
  struct mbuf *mret;
156
6.79M
#if defined(SCTP_SIMPLE_ALLOCATOR)
157
6.79M
  struct mb_args mbuf_mb_args;
158
159
  /* The following setter function is not yet being enclosed within
160
   * #if USING_MBUF_CONSTRUCTOR - #endif, until I have thoroughly tested
161
   * mb_dtor_mbuf. See comment there
162
   */
163
6.79M
  mbuf_mb_args.flags = M_PKTHDR;
164
6.79M
  mbuf_mb_args.type = type;
165
6.79M
#endif
166
6.79M
  mret = SCTP_ZONE_GET(zone_mbuf, struct mbuf);
167
6.79M
#if defined(SCTP_SIMPLE_ALLOCATOR)
168
6.79M
  mb_ctor_mbuf(mret, &mbuf_mb_args, 0);
169
6.79M
#endif
170
  /*mret = ((struct mbuf *)umem_cache_alloc(zone_mbuf, UMEM_DEFAULT));*/
171
  /* There are cases when an object available in the current CPU's
172
   * loaded magazine and in those cases the object's constructor is not applied.
173
   * If that is the case, then we are duplicating constructor initialization here,
174
   * so that the mbuf is properly constructed before returning it.
175
   */
176
6.79M
  if (mret) {
177
#if USING_MBUF_CONSTRUCTOR
178
    if (! ((mret->m_flags & M_PKTHDR) && (mret->m_type == type)) ) {
179
      mbuf_constructor_dup(mret, M_PKTHDR, type);
180
    }
181
#else
182
6.79M
    mbuf_constructor_dup(mret, M_PKTHDR, type);
183
6.79M
#endif
184
6.79M
  }
185
6.79M
  return mret;
186
6.79M
}
187
188
/* __Userspace__ */
189
struct mbuf *
190
m_free(struct mbuf *m)
191
13.2M
{
192
193
13.2M
  struct mbuf *n = m->m_next;
194
195
13.2M
  if (m->m_flags & M_EXT)
196
60.9k
    mb_free_ext(m);
197
13.1M
  else if ((m->m_flags & M_NOFREE) == 0) {
198
13.1M
#if defined(SCTP_SIMPLE_ALLOCATOR)
199
13.1M
    mb_dtor_mbuf(m, NULL);
200
13.1M
#endif
201
13.1M
    SCTP_ZONE_FREE(zone_mbuf, m);
202
13.1M
  }
203
    /*umem_cache_free(zone_mbuf, m);*/
204
13.2M
  return (n);
205
13.2M
}
206
207
208
static void
209
clust_constructor_dup(caddr_t m_clust, struct mbuf* m)
210
46.7k
{
211
46.7k
  u_int *refcnt;
212
46.7k
  int type, size;
213
214
46.7k
  if (m == NULL) {
215
0
    return;
216
0
  }
217
  /* Assigning cluster of MCLBYTES. TODO: Add jumbo frame functionality */
218
46.7k
  type = EXT_CLUSTER;
219
46.7k
  size = MCLBYTES;
220
221
46.7k
  refcnt = SCTP_ZONE_GET(zone_ext_refcnt, u_int);
222
  /*refcnt = (u_int *)umem_cache_alloc(zone_ext_refcnt, UMEM_DEFAULT);*/
223
#if !defined(SCTP_SIMPLE_ALLOCATOR)
224
  if (refcnt == NULL) {
225
    umem_reap();
226
    refcnt = SCTP_ZONE_GET(zone_ext_refcnt, u_int);
227
    /*refcnt = (u_int *)umem_cache_alloc(zone_ext_refcnt, UMEM_DEFAULT);*/
228
  }
229
#endif
230
46.7k
  *refcnt = 1;
231
46.7k
  m->m_ext.ext_buf = (caddr_t)m_clust;
232
46.7k
  m->m_data = m->m_ext.ext_buf;
233
46.7k
  m->m_flags |= M_EXT;
234
46.7k
  m->m_ext.ext_free = NULL;
235
46.7k
  m->m_ext.ext_args = NULL;
236
46.7k
  m->m_ext.ext_size = size;
237
46.7k
  m->m_ext.ext_type = type;
238
46.7k
  m->m_ext.ref_cnt = refcnt;
239
46.7k
  return;
240
46.7k
}
241
242
243
/* __Userspace__ */
244
void
245
m_clget(struct mbuf *m, int how)
246
46.7k
{
247
46.7k
  caddr_t mclust_ret;
248
46.7k
#if defined(SCTP_SIMPLE_ALLOCATOR)
249
46.7k
  struct clust_args clust_mb_args_l;
250
46.7k
#endif
251
46.7k
  if (m->m_flags & M_EXT) {
252
0
    SCTPDBG(SCTP_DEBUG_USR, "%s: %p mbuf already has cluster\n", __func__, (void *)m);
253
0
  }
254
46.7k
  m->m_ext.ext_buf = (char *)NULL;
255
46.7k
#if defined(SCTP_SIMPLE_ALLOCATOR)
256
46.7k
  clust_mb_args_l.parent_mbuf = m;
257
46.7k
#endif
258
46.7k
  mclust_ret = SCTP_ZONE_GET(zone_clust, char);
259
46.7k
#if defined(SCTP_SIMPLE_ALLOCATOR)
260
46.7k
  mb_ctor_clust(mclust_ret, &clust_mb_args_l, 0);
261
46.7k
#endif
262
  /*mclust_ret = umem_cache_alloc(zone_clust, UMEM_DEFAULT);*/
263
  /*
264
   On a cluster allocation failure, call umem_reap() and retry.
265
   */
266
267
46.7k
  if (mclust_ret == NULL) {
268
#if !defined(SCTP_SIMPLE_ALLOCATOR)
269
  /*  mclust_ret = SCTP_ZONE_GET(zone_clust, char);
270
    mb_ctor_clust(mclust_ret, &clust_mb_args, 0);
271
#else*/
272
    umem_reap();
273
    mclust_ret = SCTP_ZONE_GET(zone_clust, char);
274
#endif
275
    /*mclust_ret = umem_cache_alloc(zone_clust, UMEM_DEFAULT);*/
276
    /* if (NULL == mclust_ret) { */
277
0
    SCTPDBG(SCTP_DEBUG_USR, "Memory allocation failure in %s\n", __func__);
278
    /* } */
279
0
  }
280
281
#if USING_MBUF_CONSTRUCTOR
282
  if ((m->m_ext.ext_buf == NULL)) {
283
    clust_constructor_dup(mclust_ret, m);
284
  }
285
#else
286
46.7k
  clust_constructor_dup(mclust_ret, m);
287
46.7k
#endif
288
46.7k
}
289
290
struct mbuf *
291
m_getm2(struct mbuf *m, int len, int how, short type, int flags, int allonebuf)
292
6.77M
{
293
6.77M
  struct mbuf *mb, *nm = NULL, *mtail = NULL;
294
6.77M
  int size, mbuf_threshold, space_needed = len;
295
296
6.77M
  KASSERT(len >= 0, ("%s: len is < 0", __func__));
297
298
  /* Validate flags. */
299
6.77M
  flags &= (M_PKTHDR | M_EOR);
300
301
  /* Packet header mbuf must be first in chain. */
302
6.77M
  if ((flags & M_PKTHDR) && m != NULL) {
303
0
    flags &= ~M_PKTHDR;
304
0
  }
305
306
6.77M
  if (allonebuf == 0)
307
6.73M
    mbuf_threshold = SCTP_BASE_SYSCTL(sctp_mbuf_threshold_count);
308
37.4k
  else
309
37.4k
    mbuf_threshold = 1;
310
311
  /* Loop and append maximum sized mbufs to the chain tail. */
312
13.6M
  while (len > 0) {
313
6.84M
    if ((!allonebuf && len >= MCLBYTES) || (len > (int)(((mbuf_threshold - 1) * MLEN) + MHLEN))) {
314
46.7k
      mb = m_gethdr(how, type);
315
46.7k
      MCLGET(mb, how);
316
46.7k
      size = MCLBYTES;
317
      /* SCTP_BUF_LEN(mb) = MCLBYTES; */
318
6.79M
    } else if (flags & M_PKTHDR) {
319
6.73M
      mb = m_gethdr(how, type);
320
6.73M
      if (len < MHLEN) {
321
6.71M
        size = len;
322
6.71M
      } else {
323
19.0k
        size = MHLEN;
324
19.0k
      }
325
6.73M
    } else {
326
64.3k
      mb = m_get(how, type);
327
64.3k
      if (len < MLEN) {
328
54.7k
        size = len;
329
54.7k
      } else {
330
9.58k
        size = MLEN;
331
9.58k
      }
332
64.3k
    }
333
334
    /* Fail the whole operation if one mbuf can't be allocated. */
335
6.84M
    if (mb == NULL) {
336
0
      if (nm != NULL)
337
0
        m_freem(nm);
338
0
      return (NULL);
339
0
    }
340
341
6.84M
    if (allonebuf != 0 && size < space_needed) {
342
0
      m_freem(mb);
343
0
      return (NULL);
344
0
    }
345
346
    /* Book keeping. */
347
6.84M
    len -= size;
348
6.84M
    if (mtail != NULL)
349
68.5k
      mtail->m_next = mb;
350
6.77M
    else
351
6.77M
      nm = mb;
352
6.84M
    mtail = mb;
353
6.84M
    flags &= ~M_PKTHDR;     /* Only valid on the first mbuf. */
354
6.84M
  }
355
6.77M
  if (flags & M_EOR) {
356
0
    mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
357
0
  }
358
359
  /* If mbuf was supplied, append new chain to the end of it. */
360
6.77M
  if (m != NULL) {
361
0
    for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next);
362
0
    mtail->m_next = nm;
363
0
    mtail->m_flags &= ~M_EOR;
364
6.77M
  } else {
365
6.77M
    m = nm;
366
6.77M
  }
367
368
6.77M
  return (m);
369
6.77M
}
370
371
/*
372
 * Copy the contents of uio into a properly sized mbuf chain.
373
 */
374
struct mbuf *
375
m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
376
17.7k
{
377
17.7k
  struct mbuf *m, *mb;
378
17.7k
  int error, length;
379
17.7k
  ssize_t total;
380
17.7k
  int progress = 0;
381
382
  /*
383
   * len can be zero or an arbitrary large value bound by
384
   * the total data supplied by the uio.
385
   */
386
17.7k
  if (len > 0)
387
17.7k
    total = min(uio->uio_resid, len);
388
0
  else
389
0
    total = uio->uio_resid;
390
  /*
391
   * The smallest unit returned by m_getm2() is a single mbuf
392
   * with pkthdr.  We can't align past it.
393
   */
394
17.7k
  if (align >= MHLEN)
395
0
    return (NULL);
396
  /*
397
   * Give us the full allocation or nothing.
398
   * If len is zero return the smallest empty mbuf.
399
   */
400
17.7k
  m = m_getm2(NULL, (int)max(total + align, 1), how, MT_DATA, flags, 0);
401
17.7k
  if (m == NULL)
402
0
    return (NULL);
403
17.7k
  m->m_data += align;
404
405
  /* Fill all mbufs with uio data and update header information. */
406
70.9k
  for (mb = m; mb != NULL; mb = mb->m_next) {
407
53.2k
    length = (int)min(M_TRAILINGSPACE(mb), total - progress);
408
53.2k
    error = uiomove(mtod(mb, void *), length, uio);
409
53.2k
    if (error) {
410
0
      m_freem(m);
411
0
      return (NULL);
412
0
    }
413
414
53.2k
    mb->m_len = length;
415
53.2k
    progress += length;
416
53.2k
    if (flags & M_PKTHDR)
417
0
      m->m_pkthdr.len += length;
418
53.2k
  }
419
17.7k
  KASSERT(progress == total, ("%s: progress != total", __func__));
420
421
17.7k
  return (m);
422
17.7k
}
423
424
u_int
425
m_length(struct mbuf *m0, struct mbuf **last)
426
0
{
427
0
  struct mbuf *m;
428
0
  u_int len;
429
430
0
  len = 0;
431
0
  for (m = m0; m != NULL; m = m->m_next) {
432
0
    len += m->m_len;
433
0
    if (m->m_next == NULL)
434
0
      break;
435
0
  }
436
0
  if (last != NULL)
437
0
  *last = m;
438
0
  return (len);
439
0
}
440
441
struct mbuf *
442
m_last(struct mbuf *m)
443
17.7k
{
444
53.2k
  while (m->m_next) {
445
35.4k
    m = m->m_next;
446
35.4k
  }
447
17.7k
  return (m);
448
17.7k
}
449
450
/*
451
 * Unlink a tag from the list of tags associated with an mbuf.
452
 */
453
static __inline void
454
m_tag_unlink(struct mbuf *m, struct m_tag *t)
455
0
{
456
457
0
  SLIST_REMOVE(&m->m_pkthdr.tags, t, m_tag, m_tag_link);
458
0
}
459
460
/*
461
 * Reclaim resources associated with a tag.
462
 */
463
static __inline void
464
m_tag_free(struct m_tag *t)
465
0
{
466
467
0
  (*t->m_tag_free)(t);
468
0
}
469
470
/*
471
 * Set up the contents of a tag.  Note that this does not fill in the free
472
 * method; the caller is expected to do that.
473
 *
474
 * XXX probably should be called m_tag_init, but that was already taken.
475
 */
476
static __inline void
477
m_tag_setup(struct m_tag *t, uint32_t cookie, int type, int len)
478
0
{
479
480
0
  t->m_tag_id = type;
481
0
  t->m_tag_len = len;
482
0
  t->m_tag_cookie = cookie;
483
0
}
484
485
/************ End functions from user_mbuf.h  ******************/
486
487
488
489
/************ End functions to substitute umem_cache_alloc and umem_cache_free **************/
490
491
void
492
mbuf_initialize(void *dummy)
493
1
{
494
495
  /*
496
   * __Userspace__Configure UMA zones for Mbufs and Clusters.
497
   * (TODO: m_getcl() - using packet secondary zone).
498
   * There is no provision for trash_init and trash_fini in umem.
499
   *
500
   */
501
 /* zone_mbuf = umem_cache_create(MBUF_MEM_NAME, MSIZE, 0,
502
        mb_ctor_mbuf, mb_dtor_mbuf, NULL,
503
        &mbuf_mb_args,
504
        NULL, 0);
505
  zone_mbuf = umem_cache_create(MBUF_MEM_NAME, MSIZE, 0, NULL, NULL, NULL, NULL, NULL, 0);*/
506
1
#if defined(SCTP_SIMPLE_ALLOCATOR)
507
1
  SCTP_ZONE_INIT(zone_mbuf, MBUF_MEM_NAME, MSIZE, 0);
508
#else
509
  zone_mbuf = umem_cache_create(MBUF_MEM_NAME, MSIZE, 0,
510
                                mb_ctor_mbuf, mb_dtor_mbuf, NULL,
511
                                NULL,
512
                                NULL, 0);
513
#endif
514
  /*zone_ext_refcnt = umem_cache_create(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 0,
515
        NULL, NULL, NULL,
516
        NULL,
517
        NULL, 0);*/
518
1
  SCTP_ZONE_INIT(zone_ext_refcnt, MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 0);
519
520
  /*zone_clust = umem_cache_create(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 0,
521
         mb_ctor_clust, mb_dtor_clust, NULL,
522
         &clust_mb_args,
523
         NULL, 0);
524
  zone_clust = umem_cache_create(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 0, NULL, NULL, NULL, NULL, NULL,0);*/
525
1
#if defined(SCTP_SIMPLE_ALLOCATOR)
526
1
  SCTP_ZONE_INIT(zone_clust, MBUF_CLUSTER_MEM_NAME, MCLBYTES, 0);
527
#else
528
  zone_clust = umem_cache_create(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 0,
529
                   mb_ctor_clust, mb_dtor_clust, NULL,
530
                   &clust_mb_args,
531
                   NULL, 0);
532
#endif
533
534
  /* uma_prealloc() goes here... */
535
536
  /* __Userspace__ Add umem_reap here for low memory situation?
537
   *
538
   */
539
540
1
}
541
542
543
544
/*
545
 * __Userspace__
546
 *
547
 * Constructor for Mbuf master zone. We have a different constructor
548
 * for allocating the cluster.
549
 *
550
 * The 'arg' pointer points to a mb_args structure which
551
 * contains call-specific information required to support the
552
 * mbuf allocation API.  See user_mbuf.h.
553
 *
554
 * The flgs parameter below can be UMEM_DEFAULT or UMEM_NOFAIL depending on what
555
 * was passed when umem_cache_alloc was called.
556
 * TODO: Use UMEM_NOFAIL in umem_cache_alloc and also define a failure handler
557
 * and call umem_nofail_callback(my_failure_handler) in the stack initialization routines
558
 * The advantage of using UMEM_NOFAIL is that we don't have to check if umem_cache_alloc
559
 * was successful or not. The failure handler would take care of it, if we use the UMEM_NOFAIL
560
 * flag.
561
 *
562
 * NOTE Ref: http://docs.sun.com/app/docs/doc/819-2243/6n4i099p2?l=en&a=view&q=umem_zalloc)
563
 * The umem_nofail_callback() function sets the **process-wide** UMEM_NOFAIL callback.
564
 * It also mentions that umem_nofail_callback is Evolving.
565
 *
566
 */
567
static int
568
mb_ctor_mbuf(void *mem, void *arg, int flgs)
569
13.2M
{
570
#if USING_MBUF_CONSTRUCTOR
571
  struct mbuf *m;
572
  struct mb_args *args;
573
574
  int flags;
575
  short type;
576
577
  m = (struct mbuf *)mem;
578
  args = (struct mb_args *)arg;
579
  flags = args->flags;
580
  type = args->type;
581
582
  m->m_next = NULL;
583
  m->m_nextpkt = NULL;
584
  m->m_len = 0;
585
  m->m_flags = flags;
586
  m->m_type = type;
587
  if (flags & M_PKTHDR) {
588
    m->m_data = m->m_pktdat;
589
    m->m_pkthdr.rcvif = NULL;
590
    m->m_pkthdr.len = 0;
591
    m->m_pkthdr.header = NULL;
592
    m->m_pkthdr.csum_flags = 0;
593
    m->m_pkthdr.csum_data = 0;
594
    m->m_pkthdr.tso_segsz = 0;
595
    m->m_pkthdr.ether_vtag = 0;
596
    SLIST_INIT(&m->m_pkthdr.tags);
597
  } else
598
    m->m_data = m->m_dat;
599
#endif
600
13.2M
  return (0);
601
13.2M
}
602
603
604
/*
605
 * __Userspace__
606
 * The Mbuf master zone destructor.
607
 * This would be called in response to umem_cache_destroy
608
 * TODO: Recheck if this is what we want to do in this destructor.
609
 * (Note: the number of times mb_dtor_mbuf is called is equal to the
610
 * number of individual mbufs allocated from zone_mbuf.
611
 */
612
static void
613
mb_dtor_mbuf(void *mem, void *arg)
614
13.2M
{
615
13.2M
  struct mbuf *m;
616
617
13.2M
  m = (struct mbuf *)mem;
618
13.2M
  if ((m->m_flags & M_PKTHDR) != 0) {
619
6.79M
    m_tag_delete_chain(m, NULL);
620
6.79M
  }
621
13.2M
}
622
623
624
/* __Userspace__
625
 * The Cluster zone constructor.
626
 *
627
 * Here the 'arg' pointer points to the Mbuf which we
628
 * are configuring cluster storage for.  If 'arg' is
629
 * empty we allocate just the cluster without setting
630
 * the mbuf to it.  See mbuf.h.
631
 */
632
static int
633
mb_ctor_clust(void *mem, void *arg, int flgs)
634
46.7k
{
635
636
#if USING_MBUF_CONSTRUCTOR
637
  struct mbuf *m;
638
  struct clust_args * cla;
639
  u_int *refcnt;
640
  int type, size;
641
  sctp_zone_t zone;
642
643
  /* Assigning cluster of MCLBYTES. TODO: Add jumbo frame functionality */
644
  type = EXT_CLUSTER;
645
  zone = zone_clust;
646
  size = MCLBYTES;
647
648
  cla = (struct clust_args *)arg;
649
  m = cla->parent_mbuf;
650
651
  refcnt = SCTP_ZONE_GET(zone_ext_refcnt, u_int);
652
  /*refcnt = (u_int *)umem_cache_alloc(zone_ext_refcnt, UMEM_DEFAULT);*/
653
  *refcnt = 1;
654
655
  if (m != NULL) {
656
    m->m_ext.ext_buf = (caddr_t)mem;
657
    m->m_data = m->m_ext.ext_buf;
658
    m->m_flags |= M_EXT;
659
    m->m_ext.ext_free = NULL;
660
    m->m_ext.ext_args = NULL;
661
    m->m_ext.ext_size = size;
662
    m->m_ext.ext_type = type;
663
    m->m_ext.ref_cnt = refcnt;
664
  }
665
#endif
666
46.7k
  return (0);
667
46.7k
}
668
669
/* __Userspace__ */
670
static void
671
mb_dtor_clust(void *mem, void *arg)
672
46.4k
{
673
674
  /* mem is of type caddr_t.  In sys/types.h we have typedef char * caddr_t;  */
675
  /* mb_dtor_clust is called at time of umem_cache_destroy() (the number of times
676
   * mb_dtor_clust is called is equal to the number of individual mbufs allocated
677
   * from zone_clust. Similarly for mb_dtor_mbuf).
678
   * At this point the following:
679
   *  struct mbuf *m;
680
   *   m = (struct mbuf *)arg;
681
   *  assert (*(m->m_ext.ref_cnt) == 0); is not meaningful since  m->m_ext.ref_cnt = NULL;
682
   *  has been done in mb_free_ext().
683
   */
684
685
46.4k
}
686
687
688
689
690
/* Unlink and free a packet tag. */
691
void
692
m_tag_delete(struct mbuf *m, struct m_tag *t)
693
0
{
694
0
  KASSERT(m && t, ("m_tag_delete: null argument, m %p t %p", (void *)m, (void *)t));
695
0
  m_tag_unlink(m, t);
696
0
  m_tag_free(t);
697
0
}
698
699
700
/* Unlink and free a packet tag chain, starting from given tag. */
701
void
702
m_tag_delete_chain(struct mbuf *m, struct m_tag *t)
703
6.79M
{
704
705
6.79M
  struct m_tag *p, *q;
706
707
6.79M
  KASSERT(m, ("m_tag_delete_chain: null mbuf"));
708
6.79M
  if (t != NULL)
709
0
    p = t;
710
6.79M
  else
711
6.79M
    p = SLIST_FIRST(&m->m_pkthdr.tags);
712
6.79M
  if (p == NULL)
713
6.79M
    return;
714
0
  while ((q = SLIST_NEXT(p, m_tag_link)) != NULL)
715
0
    m_tag_delete(m, q);
716
0
  m_tag_delete(m, p);
717
0
}
718
719
#if 0
720
static void
721
sctp_print_mbuf_chain(struct mbuf *m)
722
{
723
  SCTP_DEBUG_USR(SCTP_DEBUG_USR, "Printing mbuf chain %p.\n", (void *)m);
724
  for(; m; m=m->m_next) {
725
    SCTP_DEBUG_USR(SCTP_DEBUG_USR, "%p: m_len = %ld, m_type = %x, m_next = %p.\n", (void *)m, m->m_len, m->m_type, (void *)m->m_next);
726
    if (m->m_flags & M_EXT)
727
      SCTP_DEBUG_USR(SCTP_DEBUG_USR, "%p: extend_size = %d, extend_buffer = %p, ref_cnt = %d.\n", (void *)m, m->m_ext.ext_size, (void *)m->m_ext.ext_buf, *(m->m_ext.ref_cnt));
728
  }
729
}
730
#endif
731
732
/*
733
 * Free an entire chain of mbufs and associated external buffers, if
734
 * applicable.
735
 */
736
void
737
m_freem(struct mbuf *mb)
738
6.75M
{
739
7.25M
  while (mb != NULL)
740
493k
    mb = m_free(mb);
741
6.75M
}
742
743
/*
744
 * __Userspace__
745
 * clean mbufs with M_EXT storage attached to them
746
 * if the reference count hits 1.
747
 */
748
void
749
mb_free_ext(struct mbuf *m)
750
60.9k
{
751
752
60.9k
  int skipmbuf;
753
754
60.9k
  KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
755
60.9k
  KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
756
757
  /*
758
   * check if the header is embedded in the cluster
759
   */
760
60.9k
  skipmbuf = (m->m_flags & M_NOFREE);
761
762
  /* Free the external attached storage if this
763
   * mbuf is the only reference to it.
764
   *__Userspace__ TODO: jumbo frames
765
   *
766
  */
767
  /* NOTE: We had the same code that SCTP_DECREMENT_AND_CHECK_REFCOUNT
768
           reduces to here before but the IPHONE malloc commit had changed
769
           this to compare to 0 instead of 1 (see next line).  Why?
770
          . .. this caused a huge memory leak in Linux.
771
  */
772
#ifdef IPHONE
773
  if (atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 0)
774
#else
775
60.9k
  if (SCTP_DECREMENT_AND_CHECK_REFCOUNT(m->m_ext.ref_cnt))
776
46.4k
#endif
777
46.4k
  {
778
46.4k
    if (m->m_ext.ext_type == EXT_CLUSTER){
779
46.4k
#if defined(SCTP_SIMPLE_ALLOCATOR)
780
46.4k
      mb_dtor_clust(m->m_ext.ext_buf, &clust_mb_args);
781
46.4k
#endif
782
46.4k
      SCTP_ZONE_FREE(zone_clust, m->m_ext.ext_buf);
783
46.4k
      SCTP_ZONE_FREE(zone_ext_refcnt, (u_int*)m->m_ext.ref_cnt);
784
46.4k
      m->m_ext.ref_cnt = NULL;
785
46.4k
    }
786
46.4k
  }
787
788
60.9k
  if (skipmbuf)
789
0
    return;
790
791
792
  /* __Userspace__ Also freeing the storage for ref_cnt
793
   * Free this mbuf back to the mbuf zone with all m_ext
794
   * information purged.
795
   */
796
60.9k
  m->m_ext.ext_buf = NULL;
797
60.9k
  m->m_ext.ext_free = NULL;
798
60.9k
  m->m_ext.ext_args = NULL;
799
60.9k
  m->m_ext.ref_cnt = NULL;
800
60.9k
  m->m_ext.ext_size = 0;
801
60.9k
  m->m_ext.ext_type = 0;
802
60.9k
  m->m_flags &= ~M_EXT;
803
60.9k
#if defined(SCTP_SIMPLE_ALLOCATOR)
804
60.9k
  mb_dtor_mbuf(m, NULL);
805
60.9k
#endif
806
60.9k
  SCTP_ZONE_FREE(zone_mbuf, m);
807
808
  /*umem_cache_free(zone_mbuf, m);*/
809
60.9k
}
810
811
/*
812
 * "Move" mbuf pkthdr from "from" to "to".
813
 * "from" must have M_PKTHDR set, and "to" must be empty.
814
 */
815
void
816
m_move_pkthdr(struct mbuf *to, struct mbuf *from)
817
6.37M
{
818
819
6.37M
  to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
820
6.37M
  if ((to->m_flags & M_EXT) == 0)
821
6.37M
    to->m_data = to->m_pktdat;
822
6.37M
  to->m_pkthdr = from->m_pkthdr;   /* especially tags */
823
6.37M
  SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */
824
6.37M
  from->m_flags &= ~M_PKTHDR;
825
6.37M
}
826
827
828
/*
829
 * Rearange an mbuf chain so that len bytes are contiguous
830
 * and in the data area of an mbuf (so that mtod and dtom
831
 * will work for a structure of size len).  Returns the resulting
832
 * mbuf chain on success, frees it and returns null on failure.
833
 * If there is room, it will add up to max_protohdr-len extra bytes to the
834
 * contiguous region in an attempt to avoid being called next time.
835
 */
836
struct mbuf *
837
m_pullup(struct mbuf *n, int len)
838
6.37M
{
839
6.37M
  struct mbuf *m;
840
6.37M
  int count;
841
6.37M
  int space;
842
843
  /*
844
   * If first mbuf has no cluster, and has room for len bytes
845
   * without shifting current data, pullup into it,
846
   * otherwise allocate a new mbuf to prepend to the chain.
847
   */
848
6.37M
  if ((n->m_flags & M_EXT) == 0 &&
849
6.37M
      n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
850
0
    if (n->m_len >= len)
851
0
      return (n);
852
0
    m = n;
853
0
    n = n->m_next;
854
0
    len -= m->m_len;
855
6.37M
  } else {
856
6.37M
    if (len > MHLEN)
857
0
      goto bad;
858
6.37M
    MGET(m, M_NOWAIT, n->m_type);
859
6.37M
    if (m == NULL)
860
0
      goto bad;
861
6.37M
    m->m_len = 0;
862
6.37M
    if (n->m_flags & M_PKTHDR)
863
6.37M
      M_MOVE_PKTHDR(m, n);
864
6.37M
  }
865
6.37M
  space = (int)(&m->m_dat[MLEN] - (m->m_data + m->m_len));
866
6.37M
  do {
867
6.37M
    count = min(min(max(len, max_protohdr), space), n->m_len);
868
6.37M
    memcpy(mtod(m, caddr_t) + m->m_len,mtod(n, caddr_t), (u_int)count);
869
6.37M
    len -= count;
870
6.37M
    m->m_len += count;
871
6.37M
    n->m_len -= count;
872
6.37M
    space -= count;
873
6.37M
    if (n->m_len)
874
0
      n->m_data += count;
875
6.37M
    else
876
6.37M
      n = m_free(n);
877
6.37M
  } while (len > 0 && n);
878
6.37M
  if (len > 0) {
879
6.37M
    (void) m_free(m);
880
6.37M
    goto bad;
881
6.37M
  }
882
0
  m->m_next = n;
883
0
  return (m);
884
6.37M
bad:
885
6.37M
  m_freem(n);
886
6.37M
  return (NULL);
887
6.37M
}
888
889
890
static struct mbuf *
891
m_dup1(struct mbuf *m, int off, int len, int wait)
892
0
{
893
0
  struct mbuf *n = NULL;
894
0
  int copyhdr;
895
896
0
  if (len > MCLBYTES)
897
0
    return NULL;
898
0
  if (off == 0 && (m->m_flags & M_PKTHDR) != 0)
899
0
    copyhdr = 1;
900
0
  else
901
0
    copyhdr = 0;
902
0
  if (len >= MINCLSIZE) {
903
0
    if (copyhdr == 1) {
904
0
      m_clget(n, wait); /* TODO: include code for copying the header */
905
0
      m_dup_pkthdr(n, m, wait);
906
0
    } else
907
0
      m_clget(n, wait);
908
0
  } else {
909
0
    if (copyhdr == 1)
910
0
      n = m_gethdr(wait, m->m_type);
911
0
    else
912
0
      n = m_get(wait, m->m_type);
913
0
  }
914
0
  if (!n)
915
0
    return NULL; /* ENOBUFS */
916
917
0
  if (copyhdr && !m_dup_pkthdr(n, m, wait)) {
918
0
    m_free(n);
919
0
    return NULL;
920
0
  }
921
0
  m_copydata(m, off, len, mtod(n, caddr_t));
922
0
  n->m_len = len;
923
0
  return n;
924
0
}
925
926
927
/* Taken from sys/kern/uipc_mbuf2.c */
928
struct mbuf *
929
m_pulldown(struct mbuf *m, int off, int len, int *offp)
930
0
{
931
0
  struct mbuf *n, *o;
932
0
  int hlen, tlen, olen;
933
0
  int writable;
934
935
  /* check invalid arguments. */
936
0
  KASSERT(m, ("m == NULL in m_pulldown()"));
937
0
  if (len > MCLBYTES) {
938
0
    m_freem(m);
939
0
    return NULL;    /* impossible */
940
0
  }
941
942
#ifdef PULLDOWN_DEBUG
943
  {
944
    struct mbuf *t;
945
    SCTP_DEBUG_USR(SCTP_DEBUG_USR, "before:");
946
    for (t = m; t; t = t->m_next)
947
      SCTP_DEBUG_USR(SCTP_DEBUG_USR, " %d", t->m_len);
948
    SCTP_DEBUG_USR(SCTP_DEBUG_USR, "\n");
949
  }
950
#endif
951
0
  n = m;
952
0
  while (n != NULL && off > 0) {
953
0
    if (n->m_len > off)
954
0
      break;
955
0
    off -= n->m_len;
956
0
    n = n->m_next;
957
0
  }
958
  /* be sure to point non-empty mbuf */
959
0
  while (n != NULL && n->m_len == 0)
960
0
    n = n->m_next;
961
0
  if (!n) {
962
0
    m_freem(m);
963
0
    return NULL;    /* mbuf chain too short */
964
0
  }
965
966
0
  writable = 0;
967
0
  if ((n->m_flags & M_EXT) == 0 ||
968
0
      (n->m_ext.ext_type == EXT_CLUSTER && M_WRITABLE(n)))
969
0
    writable = 1;
970
971
  /*
972
   * the target data is on <n, off>.
973
   * if we got enough data on the mbuf "n", we're done.
974
   */
975
0
  if ((off == 0 || offp) && len <= n->m_len - off && writable)
976
0
    goto ok;
977
978
  /*
979
   * when len <= n->m_len - off and off != 0, it is a special case.
980
   * len bytes from <n, off> sits in single mbuf, but the caller does
981
   * not like the starting position (off).
982
   * chop the current mbuf into two pieces, set off to 0.
983
   */
984
0
  if (len <= n->m_len - off) {
985
0
    o = m_dup1(n, off, n->m_len - off, M_NOWAIT);
986
0
    if (o == NULL) {
987
0
      m_freem(m);
988
0
    return NULL;    /* ENOBUFS */
989
0
    }
990
0
    n->m_len = off;
991
0
    o->m_next = n->m_next;
992
0
    n->m_next = o;
993
0
    n = n->m_next;
994
0
    off = 0;
995
0
    goto ok;
996
0
  }
997
  /*
998
   * we need to take hlen from <n, off> and tlen from <n->m_next, 0>,
999
   * and construct contiguous mbuf with m_len == len.
1000
   * note that hlen + tlen == len, and tlen > 0.
1001
   */
1002
0
  hlen = n->m_len - off;
1003
0
  tlen = len - hlen;
1004
1005
  /*
1006
   * ensure that we have enough trailing data on mbuf chain.
1007
   * if not, we can do nothing about the chain.
1008
   */
1009
0
  olen = 0;
1010
0
  for (o = n->m_next; o != NULL; o = o->m_next)
1011
0
    olen += o->m_len;
1012
0
  if (hlen + olen < len) {
1013
0
    m_freem(m);
1014
0
    return NULL;    /* mbuf chain too short */
1015
0
  }
1016
1017
  /*
1018
   * easy cases first.
1019
   * we need to use m_copydata() to get data from <n->m_next, 0>.
1020
   */
1021
0
  if ((off == 0 || offp) && (M_TRAILINGSPACE(n) >= tlen) && writable) {
1022
0
    m_copydata(n->m_next, 0, tlen, mtod(n, caddr_t) + n->m_len);
1023
0
    n->m_len += tlen;
1024
0
    m_adj(n->m_next, tlen);
1025
0
    goto ok;
1026
0
  }
1027
1028
0
  if ((off == 0 || offp) && (M_LEADINGSPACE(n->m_next) >= hlen) && writable) {
1029
0
    n->m_next->m_data -= hlen;
1030
0
    n->m_next->m_len += hlen;
1031
0
    memcpy( mtod(n->m_next, caddr_t), mtod(n, caddr_t) + off,hlen);
1032
0
    n->m_len -= hlen;
1033
0
    n = n->m_next;
1034
0
    off = 0;
1035
0
    goto ok;
1036
0
  }
1037
1038
  /*
1039
   * now, we need to do the hard way.  don't m_copy as there's no room
1040
   * on both end.
1041
   */
1042
0
  if (len > MLEN)
1043
0
    m_clget(o, M_NOWAIT);
1044
    /* o = m_getcl(M_NOWAIT, m->m_type, 0);*/
1045
0
  else
1046
0
    o = m_get(M_NOWAIT, m->m_type);
1047
0
  if (!o) {
1048
0
    m_freem(m);
1049
0
    return NULL;    /* ENOBUFS */
1050
0
  }
1051
  /* get hlen from <n, off> into <o, 0> */
1052
0
  o->m_len = hlen;
1053
0
  memcpy(mtod(o, caddr_t), mtod(n, caddr_t) + off, hlen);
1054
0
  n->m_len -= hlen;
1055
  /* get tlen from <n->m_next, 0> into <o, hlen> */
1056
0
  m_copydata(n->m_next, 0, tlen, mtod(o, caddr_t) + o->m_len);
1057
0
  o->m_len += tlen;
1058
0
  m_adj(n->m_next, tlen);
1059
0
  o->m_next = n->m_next;
1060
0
  n->m_next = o;
1061
0
  n = o;
1062
0
  off = 0;
1063
0
ok:
1064
#ifdef PULLDOWN_DEBUG
1065
  {
1066
    struct mbuf *t;
1067
    SCTP_DEBUG_USR(SCTP_DEBUG_USR, "after:");
1068
    for (t = m; t; t = t->m_next)
1069
      SCTP_DEBUG_USR(SCTP_DEBUG_USR, "%c%d", t == n ? '*' : ' ', t->m_len);
1070
    SCTP_DEBUG_USR(SCTP_DEBUG_USR, " (off=%d)\n", off);
1071
  }
1072
#endif
1073
0
  if (offp)
1074
0
    *offp = off;
1075
0
  return n;
1076
0
}
1077
1078
/*
1079
 * Attach the the cluster from *m to *n, set up m_ext in *n
1080
 * and bump the refcount of the cluster.
1081
 */
1082
static void
1083
mb_dupcl(struct mbuf *n, struct mbuf *m)
1084
14.5k
{
1085
14.5k
  KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
1086
14.5k
  KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
1087
14.5k
  KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
1088
1089
14.5k
  if (*(m->m_ext.ref_cnt) == 1)
1090
5.59k
    *(m->m_ext.ref_cnt) += 1;
1091
8.92k
  else
1092
8.92k
    atomic_add_int(m->m_ext.ref_cnt, 1);
1093
14.5k
  n->m_ext.ext_buf = m->m_ext.ext_buf;
1094
14.5k
  n->m_ext.ext_free = m->m_ext.ext_free;
1095
14.5k
  n->m_ext.ext_args = m->m_ext.ext_args;
1096
14.5k
  n->m_ext.ext_size = m->m_ext.ext_size;
1097
14.5k
  n->m_ext.ref_cnt = m->m_ext.ref_cnt;
1098
14.5k
  n->m_ext.ext_type = m->m_ext.ext_type;
1099
14.5k
  n->m_flags |= M_EXT;
1100
14.5k
}
1101
1102
1103
/*
1104
 * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
1105
 * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
1106
 * The wait parameter is a choice of M_TRYWAIT/M_NOWAIT from caller.
1107
 * Note that the copy is read-only, because clusters are not copied,
1108
 * only their reference counts are incremented.
1109
 */
1110
1111
struct mbuf *
1112
m_copym(struct mbuf *m, int off0, int len, int wait)
1113
10.6k
{
1114
10.6k
  struct mbuf *n, **np;
1115
10.6k
  int off = off0;
1116
10.6k
  struct mbuf *top;
1117
10.6k
  int copyhdr = 0;
1118
1119
10.6k
  KASSERT(off >= 0, ("m_copym, negative off %d", off));
1120
10.6k
  KASSERT(len >= 0, ("m_copym, negative len %d", len));
1121
10.6k
  KASSERT(m != NULL, ("m_copym, m is NULL"));
1122
1123
#if !defined(INVARIANTS)
1124
  if (m == NULL) {
1125
    return (NULL);
1126
  }
1127
#endif
1128
10.6k
  if (off == 0 && m->m_flags & M_PKTHDR)
1129
4.46k
    copyhdr = 1;
1130
10.6k
  while (off > 0) {
1131
1.67k
    KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
1132
1.67k
    if (off < m->m_len)
1133
1.67k
      break;
1134
0
    off -= m->m_len;
1135
0
    m = m->m_next;
1136
0
  }
1137
10.6k
  np = &top;
1138
10.6k
  top = 0;
1139
31.9k
  while (len > 0) {
1140
25.8k
    if (m == NULL) {
1141
4.46k
      KASSERT(len == M_COPYALL, ("m_copym, length > size of mbuf chain"));
1142
4.46k
      break;
1143
4.46k
    }
1144
21.3k
    if (copyhdr)
1145
4.46k
      MGETHDR(n, wait, m->m_type);
1146
16.9k
    else
1147
16.9k
      MGET(n, wait, m->m_type);
1148
21.3k
    *np = n;
1149
21.3k
    if (n == NULL)
1150
0
      goto nospace;
1151
21.3k
    if (copyhdr) {
1152
4.46k
      if (!m_dup_pkthdr(n, m, wait))
1153
0
        goto nospace;
1154
4.46k
      if (len == M_COPYALL)
1155
0
        n->m_pkthdr.len -= off0;
1156
4.46k
      else
1157
4.46k
        n->m_pkthdr.len = len;
1158
4.46k
      copyhdr = 0;
1159
4.46k
    }
1160
21.3k
    n->m_len = min(len, m->m_len - off);
1161
21.3k
    if (m->m_flags & M_EXT) {
1162
11.9k
      n->m_data = m->m_data + off;
1163
11.9k
      mb_dupcl(n, m);
1164
11.9k
    } else
1165
9.48k
      memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, (u_int)n->m_len);
1166
21.3k
    if (len != M_COPYALL)
1167
10.9k
      len -= n->m_len;
1168
21.3k
    off = 0;
1169
21.3k
    m = m->m_next;
1170
21.3k
    np = &n->m_next;
1171
21.3k
  }
1172
1173
10.6k
  return (top);
1174
0
nospace:
1175
0
  m_freem(top);
1176
0
  return (NULL);
1177
10.6k
}
1178
1179
1180
int
1181
m_tag_copy_chain(struct mbuf *to, struct mbuf *from, int how)
1182
4.46k
{
1183
4.46k
  struct m_tag *p, *t, *tprev = NULL;
1184
1185
4.46k
  KASSERT(to && from, ("m_tag_copy_chain: null argument, to %p from %p", (void *)to, (void *)from));
1186
4.46k
  m_tag_delete_chain(to, NULL);
1187
4.46k
  SLIST_FOREACH(p, &from->m_pkthdr.tags, m_tag_link) {
1188
0
    t = m_tag_copy(p, how);
1189
0
    if (t == NULL) {
1190
0
      m_tag_delete_chain(to, NULL);
1191
0
      return 0;
1192
0
    }
1193
0
    if (tprev == NULL)
1194
0
      SLIST_INSERT_HEAD(&to->m_pkthdr.tags, t, m_tag_link);
1195
0
    else
1196
0
      SLIST_INSERT_AFTER(tprev, t, m_tag_link);
1197
0
    tprev = t;
1198
0
  }
1199
4.46k
  return 1;
1200
4.46k
}
1201
1202
/*
1203
 * Duplicate "from"'s mbuf pkthdr in "to".
1204
 * "from" must have M_PKTHDR set, and "to" must be empty.
1205
 * In particular, this does a deep copy of the packet tags.
1206
 */
1207
int
1208
m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
1209
4.46k
{
1210
1211
4.46k
  KASSERT(to, ("m_dup_pkthdr: to is NULL"));
1212
4.46k
  KASSERT(from, ("m_dup_pkthdr: from is NULL"));
1213
4.46k
  to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
1214
4.46k
  if ((to->m_flags & M_EXT) == 0)
1215
4.46k
    to->m_data = to->m_pktdat;
1216
4.46k
  to->m_pkthdr = from->m_pkthdr;
1217
4.46k
  SLIST_INIT(&to->m_pkthdr.tags);
1218
4.46k
  return (m_tag_copy_chain(to, from, MBTOM(how)));
1219
4.46k
}
1220
1221
/* Copy a single tag. */
1222
struct m_tag *
1223
m_tag_copy(struct m_tag *t, int how)
1224
0
{
1225
0
  struct m_tag *p;
1226
1227
0
  KASSERT(t, ("m_tag_copy: null tag"));
1228
0
  p = m_tag_alloc(t->m_tag_cookie, t->m_tag_id, t->m_tag_len, how);
1229
0
  if (p == NULL)
1230
0
    return (NULL);
1231
0
  memcpy(p + 1, t + 1, t->m_tag_len); /* Copy the data */
1232
0
  return p;
1233
0
}
1234
1235
/* Get a packet tag structure along with specified data following. */
1236
struct m_tag *
1237
m_tag_alloc(uint32_t cookie, int type, int len, int wait)
1238
0
{
1239
0
  struct m_tag *t;
1240
1241
0
  if (len < 0)
1242
0
    return NULL;
1243
0
  t = malloc(len + sizeof(struct m_tag));
1244
0
  if (t == NULL)
1245
0
    return NULL;
1246
0
  m_tag_setup(t, cookie, type, len);
1247
0
  t->m_tag_free = m_tag_free_default;
1248
0
  return t;
1249
0
}
1250
1251
/* Free a packet tag. */
1252
void
1253
m_tag_free_default(struct m_tag *t)
1254
0
{
1255
0
  free(t);
1256
0
}
1257
1258
/*
1259
 * Copy data from a buffer back into the indicated mbuf chain,
1260
 * starting "off" bytes from the beginning, extending the mbuf
1261
 * chain if necessary.
1262
 */
1263
void
1264
m_copyback(struct mbuf *m0, int off, int len, caddr_t cp)
1265
6.72M
{
1266
6.72M
  int mlen;
1267
6.72M
  struct mbuf *m = m0, *n;
1268
6.72M
  int totlen = 0;
1269
1270
6.72M
  if (m0 == NULL)
1271
0
    return;
1272
6.72M
  while (off > (mlen = m->m_len)) {
1273
0
    off -= mlen;
1274
0
    totlen += mlen;
1275
0
    if (m->m_next == NULL) {
1276
0
      n = m_get(M_NOWAIT, m->m_type);
1277
0
      if (n == NULL)
1278
0
        goto out;
1279
0
      memset(mtod(n, caddr_t), 0, MLEN);
1280
0
      n->m_len = min(MLEN, len + off);
1281
0
      m->m_next = n;
1282
0
    }
1283
0
    m = m->m_next;
1284
0
  }
1285
6.75M
  while (len > 0) {
1286
6.75M
    mlen = min (m->m_len - off, len);
1287
6.75M
    memcpy(off + mtod(m, caddr_t), cp, (u_int)mlen);
1288
6.75M
    cp += mlen;
1289
6.75M
    len -= mlen;
1290
6.75M
    mlen += off;
1291
6.75M
    off = 0;
1292
6.75M
    totlen += mlen;
1293
6.75M
    if (len == 0)
1294
6.72M
      break;
1295
33.1k
    if (m->m_next == NULL) {
1296
0
      n = m_get(M_NOWAIT, m->m_type);
1297
0
      if (n == NULL)
1298
0
        break;
1299
0
      n->m_len = min(MLEN, len);
1300
0
      m->m_next = n;
1301
0
    }
1302
33.1k
    m = m->m_next;
1303
33.1k
  }
1304
6.72M
out:  if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1305
0
    m->m_pkthdr.len = totlen;
1306
6.72M
}
1307
1308
/*
1309
 * Apply function f to the data in an mbuf chain starting "off" bytes from
1310
 * the beginning, continuing for "len" bytes.
1311
 */
1312
int
1313
m_apply(struct mbuf *m, int off, int len,
1314
        int (*f)(void *, void *, u_int), void *arg)
1315
0
{
1316
0
  u_int count;
1317
0
  int rval;
1318
1319
0
  KASSERT(off >= 0, ("m_apply, negative off %d", off));
1320
0
  KASSERT(len >= 0, ("m_apply, negative len %d", len));
1321
0
  while (off > 0) {
1322
0
    KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1323
0
    if (off < m->m_len)
1324
0
      break;
1325
0
    off -= m->m_len;
1326
0
    m = m->m_next;
1327
0
  }
1328
0
  while (len > 0) {
1329
0
    KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1330
0
    count = min(m->m_len - off, len);
1331
0
    rval = (*f)(arg, mtod(m, caddr_t) + off, count);
1332
0
    if (rval)
1333
0
      return (rval);
1334
0
    len -= count;
1335
0
    off = 0;
1336
0
    m = m->m_next;
1337
0
  }
1338
0
  return (0);
1339
0
}
1340
1341
/*
1342
 * Lesser-used path for M_PREPEND:
1343
 * allocate new mbuf to prepend to chain,
1344
 * copy junk along.
1345
 */
1346
struct mbuf *
1347
m_prepend(struct mbuf *m, int len, int how)
1348
0
{
1349
0
  struct mbuf *mn;
1350
1351
0
  if (m->m_flags & M_PKTHDR)
1352
0
    MGETHDR(mn, how, m->m_type);
1353
0
  else
1354
0
    MGET(mn, how, m->m_type);
1355
0
  if (mn == NULL) {
1356
0
    m_freem(m);
1357
0
    return (NULL);
1358
0
  }
1359
0
  if (m->m_flags & M_PKTHDR)
1360
0
    M_MOVE_PKTHDR(mn, m);
1361
0
  mn->m_next = m;
1362
0
  m = mn;
1363
0
  if (m->m_flags & M_PKTHDR) {
1364
0
    if (len < MHLEN)
1365
0
      MH_ALIGN(m, len);
1366
0
  } else {
1367
0
    if (len < MLEN)
1368
0
      M_ALIGN(m, len);
1369
0
  }
1370
0
  m->m_len = len;
1371
0
  return (m);
1372
0
}
1373
1374
/*
1375
 * Copy data from an mbuf chain starting "off" bytes from the beginning,
1376
 * continuing for "len" bytes, into the indicated buffer.
1377
 */
1378
void
1379
m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
1380
17.7k
{
1381
17.7k
  u_int count;
1382
1383
17.7k
  KASSERT(off >= 0, ("m_copydata, negative off %d", off));
1384
17.7k
  KASSERT(len >= 0, ("m_copydata, negative len %d", len));
1385
17.7k
  while (off > 0) {
1386
0
    KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
1387
0
    if (off < m->m_len)
1388
0
      break;
1389
0
    off -= m->m_len;
1390
0
    m = m->m_next;
1391
0
  }
1392
62.5k
  while (len > 0) {
1393
44.7k
    KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
1394
44.7k
    count = min(m->m_len - off, len);
1395
44.7k
    memcpy(cp, mtod(m, caddr_t) + off, count);
1396
44.7k
    len -= count;
1397
44.7k
    cp += count;
1398
44.7k
    off = 0;
1399
44.7k
    m = m->m_next;
1400
44.7k
  }
1401
17.7k
}
1402
1403
1404
/*
1405
 * Concatenate mbuf chain n to m.
1406
 * Both chains must be of the same type (e.g. MT_DATA).
1407
 * Any m_pkthdr is not updated.
1408
 */
1409
void
1410
m_cat(struct mbuf *m, struct mbuf *n)
1411
0
{
1412
0
  while (m->m_next)
1413
0
    m = m->m_next;
1414
0
  while (n) {
1415
0
    if (m->m_flags & M_EXT ||
1416
0
        m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) {
1417
      /* just join the two chains */
1418
0
      m->m_next = n;
1419
0
      return;
1420
0
    }
1421
    /* splat the data from one into the other */
1422
0
    memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), (u_int)n->m_len);
1423
0
    m->m_len += n->m_len;
1424
0
    n = m_free(n);
1425
0
  }
1426
0
}
1427
1428
1429
void
1430
m_adj(struct mbuf *mp, int req_len)
1431
4.46k
{
1432
4.46k
  int len = req_len;
1433
4.46k
  struct mbuf *m;
1434
4.46k
  int count;
1435
1436
4.46k
  if ((m = mp) == NULL)
1437
0
    return;
1438
4.46k
  if (len >= 0) {
1439
    /*
1440
     * Trim from head.
1441
     */
1442
10.4k
    while (m != NULL && len > 0) {
1443
5.95k
      if (m->m_len <= len) {
1444
1.48k
        len -= m->m_len;
1445
1.48k
        m->m_len = 0;
1446
1.48k
        m = m->m_next;
1447
4.46k
      } else {
1448
4.46k
        m->m_len -= len;
1449
4.46k
        m->m_data += len;
1450
4.46k
        len = 0;
1451
4.46k
      }
1452
5.95k
    }
1453
4.46k
    m = mp;
1454
4.46k
    if (mp->m_flags & M_PKTHDR)
1455
4.46k
      m->m_pkthdr.len -= (req_len - len);
1456
4.46k
  } else {
1457
    /*
1458
     * Trim from tail.  Scan the mbuf chain,
1459
     * calculating its length and finding the last mbuf.
1460
     * If the adjustment only affects this mbuf, then just
1461
     * adjust and return.  Otherwise, rescan and truncate
1462
     * after the remaining size.
1463
     */
1464
0
    len = -len;
1465
0
    count = 0;
1466
0
    for (;;) {
1467
0
      count += m->m_len;
1468
0
      if (m->m_next == (struct mbuf *)0)
1469
0
        break;
1470
0
      m = m->m_next;
1471
0
    }
1472
0
    if (m->m_len >= len) {
1473
0
      m->m_len -= len;
1474
0
      if (mp->m_flags & M_PKTHDR)
1475
0
        mp->m_pkthdr.len -= len;
1476
0
      return;
1477
0
    }
1478
0
    count -= len;
1479
0
    if (count < 0)
1480
0
      count = 0;
1481
    /*
1482
     * Correct length for chain is "count".
1483
     * Find the mbuf with last data, adjust its length,
1484
     * and toss data from remaining mbufs on chain.
1485
     */
1486
0
    m = mp;
1487
0
    if (m->m_flags & M_PKTHDR)
1488
0
      m->m_pkthdr.len = count;
1489
0
    for (; m; m = m->m_next) {
1490
0
      if (m->m_len >= count) {
1491
0
        m->m_len = count;
1492
0
        if (m->m_next != NULL) {
1493
0
          m_freem(m->m_next);
1494
0
          m->m_next = NULL;
1495
0
        }
1496
0
        break;
1497
0
      }
1498
0
      count -= m->m_len;
1499
0
    }
1500
0
  }
1501
4.46k
}
1502
1503
1504
/* m_split is used within sctp_handle_cookie_echo. */
1505
1506
/*
1507
 * Partition an mbuf chain in two pieces, returning the tail --
1508
 * all but the first len0 bytes.  In case of failure, it returns NULL and
1509
 * attempts to restore the chain to its original state.
1510
 *
1511
 * Note that the resulting mbufs might be read-only, because the new
1512
 * mbuf can end up sharing an mbuf cluster with the original mbuf if
1513
 * the "breaking point" happens to lie within a cluster mbuf. Use the
1514
 * M_WRITABLE() macro to check for this case.
1515
 */
1516
struct mbuf *
1517
m_split(struct mbuf *m0, int len0, int wait)
1518
12.4k
{
1519
12.4k
  struct mbuf *m, *n;
1520
12.4k
  u_int len = len0, remain;
1521
1522
  /* MBUF_CHECKSLEEP(wait); */
1523
16.3k
  for (m = m0; m && (int)len > m->m_len; m = m->m_next)
1524
3.90k
    len -= m->m_len;
1525
12.4k
  if (m == NULL)
1526
0
    return (NULL);
1527
12.4k
  remain = m->m_len - len;
1528
12.4k
  if (m0->m_flags & M_PKTHDR) {
1529
12.4k
    MGETHDR(n, wait, m0->m_type);
1530
12.4k
    if (n == NULL)
1531
0
      return (NULL);
1532
12.4k
    n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1533
12.4k
    n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1534
12.4k
    m0->m_pkthdr.len = len0;
1535
12.4k
    if (m->m_flags & M_EXT)
1536
2.61k
      goto extpacket;
1537
9.87k
    if (remain > MHLEN) {
1538
      /* m can't be the lead packet */
1539
0
      MH_ALIGN(n, 0);
1540
0
      n->m_next = m_split(m, len, wait);
1541
0
      if (n->m_next == NULL) {
1542
0
        (void) m_free(n);
1543
0
        return (NULL);
1544
0
      } else {
1545
0
        n->m_len = 0;
1546
0
        return (n);
1547
0
      }
1548
0
    } else
1549
9.87k
      MH_ALIGN(n, remain);
1550
9.87k
  } else if (remain == 0) {
1551
0
    n = m->m_next;
1552
0
    m->m_next = NULL;
1553
0
    return (n);
1554
0
  } else {
1555
0
    MGET(n, wait, m->m_type);
1556
0
    if (n == NULL)
1557
0
      return (NULL);
1558
0
    M_ALIGN(n, remain);
1559
0
  }
1560
12.4k
extpacket:
1561
12.4k
  if (m->m_flags & M_EXT) {
1562
2.61k
    n->m_data = m->m_data + len;
1563
2.61k
    mb_dupcl(n, m);
1564
9.87k
  } else {
1565
9.87k
    memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + len, remain);
1566
9.87k
  }
1567
12.4k
  n->m_len = remain;
1568
12.4k
  m->m_len = len;
1569
12.4k
  n->m_next = m->m_next;
1570
12.4k
  m->m_next = NULL;
1571
12.4k
  return (n);
1572
12.4k
}
1573
1574
1575
1576
1577
int
1578
0
pack_send_buffer(caddr_t buffer, struct mbuf* mb){
1579
1580
0
  int count_to_copy;
1581
0
  int total_count_copied = 0;
1582
0
  int offset = 0;
1583
1584
0
  do {
1585
0
    count_to_copy = mb->m_len;
1586
0
    memcpy(buffer+offset, mtod(mb, caddr_t), count_to_copy);
1587
0
    offset += count_to_copy;
1588
0
    total_count_copied += count_to_copy;
1589
0
    mb = mb->m_next;
1590
0
  } while(mb);
1591
1592
0
  return (total_count_copied);
1593
0
}