Coverage Report

Created: 2024-06-18 06:23

/src/hpn-ssh/cipher-chachapoly-libcrypto-mt.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2023 The Board of Trustees of Carnegie Mellon University.
3
 *
4
 *  Author: Mitchell Dorrell <mwd@psc.edu>
5
 *  Author: Chris Rapier  <rapier@psc.edu>
6
 *
7
 * This library is free software; you can redistribute it and/or modify it
8
 * under the terms of the MIT License.
9
 *
10
 * This library is distributed in the hope that it will be useful, but WITHOUT
11
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12
 * FITNESS FOR A PARTICULAR PURPOSE.  See the MIT License for more details.
13
 *
14
 * You should have received a copy of the MIT License along with this library;
15
 * if not, see http://opensource.org/licenses/MIT.
16
 *
17
 */
18
19
/* TODO: audit includes */
20
21
#include "includes.h"
22
#ifdef WITH_OPENSSL
23
#include "openbsd-compat/openssl-compat.h"
24
#endif
25
26
#if defined(HAVE_EVP_CHACHA20) && !defined(HAVE_BROKEN_CHACHA20)
27
28
#include <sys/types.h>
29
#include <unistd.h> /* needed for getpid under C99 */
30
#include <stdarg.h> /* needed for log.h */
31
#include <string.h>
32
#include <stdio.h>  /* needed for misc.h */
33
#include <pthread.h>
34
35
#include <openssl/evp.h>
36
37
#include "defines.h"
38
#include "log.h"
39
#include "sshbuf.h"
40
#include "ssherr.h"
41
42
#include "xmalloc.h"
43
#include "cipher-chachapoly.h"
44
#include "cipher-chachapoly-libcrypto-mt.h"
45
46
#ifndef likely
47
# define likely(x)   __builtin_expect(!!(x), 1)
48
#endif
49
#ifndef unlikely
50
0
# define unlikely(x) __builtin_expect(!!(x), 0)
51
#endif
52
53
/* Size of keystream to pregenerate, measured in bytes
54
 * we want to round up to the nearest chacha block and have
55
 * 128 bytes for overhead */
56
0
#define ROUND_UP(x,y) (((((x)-1)/(y))+1)*(y))
57
0
#define KEYSTREAMLEN (ROUND_UP((SSH_IOBUFSZ) + 128, (CHACHA_BLOCKLEN)))
58
59
/* BEGIN TUNABLES */
60
61
/* Number of worker threads to spawn. */
62
/* the goal is to ensure that main is never
63
 * waiting on the worker threads for keystream data */
64
0
#define NUMTHREADS 1
65
66
/* 64 seems to be a pretty blance between memory and performance
67
 * 128 is another option with somewhat higher memory consumption */
68
0
#define NUMSTREAMS 64
69
70
/* END TUNABLES */
71
72
struct mt_keystream {
73
  u_char poly_key[POLY1305_KEYLEN];     /* POLY1305_KEYLEN == 32 */
74
  u_char headerStream[CHACHA_BLOCKLEN]; /* CHACHA_BLOCKLEN == 64 */
75
  u_char mainStream[KEYSTREAMLEN];      /* KEYSTREAMLEN == 32768 */
76
};
77
78
struct threadData {
79
  EVP_CIPHER_CTX * main_evp;
80
  EVP_CIPHER_CTX * header_evp;
81
  u_char seqbuf[16];
82
};
83
84
struct mt_keystream_batch {
85
  u_int batchID;
86
  struct threadData tds[NUMTHREADS];
87
  struct mt_keystream streams[NUMSTREAMS];
88
};
89
90
struct chachapoly_ctx_mt {
91
  u_int seqnr;
92
  u_int batchID;
93
94
  struct mt_keystream_batch batches[2];
95
96
  pthread_t manager_tid[2];
97
  pthread_t self_tid;
98
99
  pid_t mainpid;
100
  u_char zeros[KEYSTREAMLEN]; /* KEYSTREAMLEN == 32768 */
101
102
  /* if OpenSSL has support for Poly1305 in the MAC EVPs
103
   * use that (OSSL >= 3.0) if not then it's OSSL 1.1 so
104
   * use the Poly1305 digest methods. Failing that use the
105
   * internal poly1305 methods */
106
#ifdef OPENSSL_HAVE_POLY_EVP
107
  EVP_MAC_CTX    *poly_ctx;
108
#elif !defined(WITH_OPENSSL3) && defined(EVP_PKEY_POLY1305)
109
  EVP_PKEY_CTX   *poly_ctx;
110
  EVP_MD_CTX     *md_ctx;
111
  EVP_PKEY       *pkey;
112
  size_t         ptaglen;
113
#else
114
  char           *poly_ctx;
115
#endif
116
};
117
118
struct manager_thread_args {
119
  struct chachapoly_ctx_mt * ctx_mt;
120
  u_int oldBatchID;
121
  int retval;
122
};
123
124
struct worker_thread_args {
125
  u_int batchID;
126
  struct mt_keystream_batch * batch;
127
  int threadIndex;
128
  u_char * zeros;
129
  int retval;
130
};
131
132
/* generate the keystream and header
133
 * we use nulls for the "data" (the zeros variable) in order to
134
 * get the raw keystream
135
 * Returns 0 on success and -1 on failure */
136
int
137
generate_keystream(struct mt_keystream * ks, u_int seqnr,
138
    struct threadData * td, u_char * zeros)
139
0
{
140
  /* generate poly1305 key */
141
0
  memset(td->seqbuf, 0, sizeof(td->seqbuf));
142
0
  POKE_U64(td->seqbuf + 8, seqnr);
143
0
  memset(ks->poly_key , 0, sizeof(ks->poly_key));
144
0
  if (!EVP_CipherInit(td->main_evp, NULL, NULL, td->seqbuf, 1) ||
145
0
      EVP_Cipher(td->main_evp, ks->poly_key, ks->poly_key,
146
0
      sizeof(ks->poly_key)) < 0)
147
0
    return -1;
148
149
  /* generate header keystream for encrypting payload length */
150
0
  if (!EVP_CipherInit(td->header_evp, NULL, NULL, td->seqbuf, 1) ||
151
0
      EVP_Cipher(td->header_evp, ks->headerStream, zeros, CHACHA_BLOCKLEN)
152
0
      < 0 )
153
0
    return -1;
154
155
  /* generate main keystream for encrypting payload */
156
0
  td->seqbuf[0] = 1;
157
0
  if (!EVP_CipherInit(td->main_evp, NULL, NULL, td->seqbuf, 1) ||
158
0
      EVP_Cipher(td->main_evp, ks->mainStream, zeros, KEYSTREAMLEN) < 0)
159
0
    return -1;
160
161
0
  return 0;
162
0
}
163
164
/* free the EVP contexts associated with the give thread */
165
void
166
free_threadData(struct threadData * td)
167
0
{
168
0
  if (td == NULL)
169
0
    return;
170
0
  if (td->main_evp) /* false if initialization didn't get this far */
171
0
    EVP_CIPHER_CTX_free(td->main_evp);
172
0
  if (td->header_evp) /* false if initialization didn't get this far */
173
0
    EVP_CIPHER_CTX_free(td->header_evp);
174
0
  explicit_bzero(td, sizeof(*td));
175
0
}
176
177
/* initialize the EVPs used by the worker thread
178
   Returns 0 on success and -1 on failure */
179
int
180
initialize_threadData(struct threadData * td, const u_char *key)
181
0
{
182
0
  memset(td,0,sizeof(*td));
183
0
  if ((td->main_evp = EVP_CIPHER_CTX_new()) == NULL ||
184
0
      (td->header_evp = EVP_CIPHER_CTX_new()) == NULL)
185
0
    goto fail;
186
0
  if (!EVP_CipherInit(td->main_evp, EVP_chacha20(), key, NULL, 1))
187
0
    goto fail;
188
0
  if (!EVP_CipherInit(td->header_evp, EVP_chacha20(), key + 32, NULL, 1))
189
0
    goto fail;
190
0
  if (EVP_CIPHER_CTX_iv_length(td->header_evp) != 16)
191
0
    goto fail;
192
0
  return 0;
193
0
 fail:
194
0
  free_threadData(td);
195
0
  return -1;
196
0
}
197
198
struct worker_thread_args *
199
worker_thread(struct worker_thread_args * args)
200
0
{
201
  /* check first */
202
0
  if (args == NULL)
203
0
    return NULL;
204
0
  if (args->batch == NULL || args->zeros == NULL) {
205
0
    args->retval = 1;
206
0
    return args;
207
0
  }
208
209
0
  int threadIndex = args->threadIndex;
210
0
  struct threadData * td = &(args->batch->tds[threadIndex]);
211
0
  u_int refseqnr = args->batchID * NUMSTREAMS;
212
213
0
  for (int i = threadIndex; i < NUMSTREAMS; i += NUMTHREADS) {
214
0
    if (generate_keystream(&(args->batch->streams[i]), refseqnr + i,
215
0
        td, args->zeros) == -1) {
216
0
      args->retval = 1;
217
0
      return args;
218
0
    }
219
0
  }
220
221
0
  args->retval = 0;
222
0
  return args;
223
0
}
224
225
int
226
join_manager_thread(pthread_t manager_tid)
227
0
{
228
0
  struct manager_thread_args * args;
229
0
  if (pthread_join(manager_tid, (void **) &args) == 0) {
230
0
    if (args == NULL) {
231
0
      debug_f("Manager thread returned NULL!");
232
0
      return 1;
233
0
    } else if (args == PTHREAD_CANCELED) {
234
0
      debug_f("Manager thread canceled!");
235
0
      return 1;
236
0
    } else if (args->retval != 0) {
237
0
      debug_f("Manager thread error (%d)", args->retval);
238
0
      free(args);
239
0
      return 1;
240
0
    } else {
241
0
      free(args);
242
0
      return 0;
243
0
    }
244
0
  } else {
245
0
    debug_f("pthread_join error!");
246
0
    return 1;
247
0
  }
248
0
}
249
250
void
251
chachapoly_free_mt(struct chachapoly_ctx_mt * ctx_mt)
252
0
{
253
0
  if (ctx_mt == NULL)
254
0
    return;
255
256
#ifdef OPENSSL_HAVE_POLY_EVP
257
  if (ctx_mt->poly_ctx != NULL) {
258
    EVP_MAC_CTX_free(ctx_mt->poly_ctx);
259
    ctx_mt->poly_ctx = NULL;
260
  }
261
#elif !defined(WITH_OPENSSL3) && defined(EVP_PKEY_POLY1305)
262
0
  if (ctx_mt->md_ctx != NULL) {
263
0
    EVP_MD_CTX_free(ctx_mt->md_ctx);
264
0
    ctx_mt->md_ctx = NULL;
265
0
  }
266
0
  if (ctx_mt->pkey != NULL) {
267
0
    EVP_PKEY_free(ctx_mt->pkey);
268
0
    ctx_mt->pkey = NULL;
269
0
  }
270
0
#endif
271
272
  /*
273
   * Only cleanup the manager threads if we are the PID that initialized
274
   * them! If we're a fork, the threads don't really exist.
275
   */
276
277
0
  if (getpid() == ctx_mt->mainpid) {
278
0
    if (ctx_mt->manager_tid[0] != ctx_mt->self_tid) {
279
0
      join_manager_thread(ctx_mt->manager_tid[0]);
280
0
      ctx_mt->manager_tid[0] = ctx_mt->self_tid;
281
0
    }
282
0
    if (ctx_mt->manager_tid[1] != ctx_mt->self_tid) {
283
0
      join_manager_thread(ctx_mt->manager_tid[1]);
284
0
      ctx_mt->manager_tid[1] = ctx_mt->self_tid;
285
0
    }
286
0
  }
287
288
  /* Cleanup thread data structures. */
289
0
  for (int i=0; i<2; i++)
290
0
    for (int j=0; j<NUMTHREADS; j++)
291
0
      free_threadData(&(ctx_mt->batches[i].tds[j]));
292
293
  /* Zero and free the whole multithreaded cipher context. */
294
0
  freezero(ctx_mt, sizeof(*ctx_mt));
295
296
0
  return;
297
0
}
298
299
struct chachapoly_ctx_mt *
300
chachapoly_new_mt(u_int startseqnr, const u_char * key, u_int keylen)
301
0
{
302
0
  struct chachapoly_ctx_mt * ctx_mt = xmalloc(sizeof(*ctx_mt));
303
0
  memset(ctx_mt, 0, sizeof(*ctx_mt));
304
  /* Initialize the sequence number. When rekeying, this won't be zero. */
305
0
  ctx_mt->seqnr = startseqnr;
306
0
  ctx_mt->batchID = startseqnr / NUMSTREAMS;
307
0
  struct threadData mainData;
308
0
  int tDataI;
309
0
  int genKSfailed = 0;
310
311
#ifdef OPENSSL_HAVE_POLY_EVP
312
  EVP_MAC *mac = NULL;
313
  if ((mac = EVP_MAC_fetch(NULL, "POLY1305", NULL)) == NULL)
314
    goto fail;
315
  if ((ctx_mt->poly_ctx = EVP_MAC_CTX_new(mac)) == NULL)
316
    goto fail;
317
#elif !defined(WITH_OPENSSL3) && defined(EVP_PKEY_POLY1305)
318
0
  if ((ctx_mt->md_ctx = EVP_MD_CTX_new()) == NULL)
319
0
    goto fail;
320
0
  if ((ctx_mt->pkey = EVP_PKEY_new_mac_key(EVP_PKEY_POLY1305, NULL,
321
0
      ctx_mt->zeros, POLY1305_KEYLEN)) == NULL)
322
0
    goto fail;
323
0
  if (EVP_DigestSignInit(ctx_mt->md_ctx, &ctx_mt->poly_ctx, NULL, NULL,
324
0
      ctx_mt->pkey) == 0)
325
0
    goto fail;
326
#else
327
  ctx_mt->poly_ctx = NULL;
328
#endif
329
330
0
  ctx_mt->batches[ctx_mt->batchID % 2].batchID = ctx_mt->batchID;
331
0
  ctx_mt->batches[(ctx_mt->batchID + 1) % 2].batchID =
332
0
      ctx_mt->batchID + 1;
333
334
  /* initialize batches[0] tds */
335
0
  for (tDataI = 0; tDataI < NUMTHREADS; tDataI++) {
336
0
    if (initialize_threadData(&(ctx_mt->batches[0].tds[tDataI]),
337
0
        key) != 0)
338
0
      break;
339
0
  }
340
0
  if (tDataI < NUMTHREADS) {
341
    /* Backtrack starting with 'tDataI - 1' */
342
0
    for (tDataI--; tDataI >= 0; tDataI--)
343
0
      free_threadData(&(ctx_mt->batches[0].tds[tDataI]));
344
0
    goto fail;
345
0
  }
346
  /* initialize batches[1] tds */
347
0
  for (tDataI = 0; tDataI < NUMTHREADS; tDataI++) {
348
0
    if (initialize_threadData(&(ctx_mt->batches[1].tds[tDataI]),
349
0
        key) != 0)
350
0
      break;
351
0
  }
352
0
  if (tDataI < NUMTHREADS) {
353
    /* Backtrack starting with 'tDataI - 1' */
354
0
    for (tDataI--; tDataI >= 0; tDataI--)
355
0
      free_threadData(&(ctx_mt->batches[1].tds[tDataI]));
356
    /* Free the batches[0] tds too */
357
0
    for (tDataI = NUMTHREADS; tDataI >= 0; tDataI--)
358
0
      free_threadData(&(ctx_mt->batches[0].tds[tDataI]));
359
0
    goto fail;
360
0
  }
361
362
0
  if (initialize_threadData(&mainData, key) != 0) {
363
0
    chachapoly_free_mt(ctx_mt);
364
0
    explicit_bzero(&startseqnr, sizeof(startseqnr));
365
0
    return NULL;
366
0
  }
367
368
0
  for (int i=0; i<2; i++) {
369
0
    u_int refseqnr = ctx_mt->batches[i].batchID * NUMSTREAMS;
370
0
    for (int j = startseqnr > refseqnr ? startseqnr - refseqnr : 0;
371
0
         j<NUMSTREAMS; j++) {
372
0
      if (generate_keystream(&(ctx_mt->batches[i].streams[j]),
373
0
          refseqnr + j, &mainData, ctx_mt->zeros) == -1) {
374
0
        debug_f("generate_keystream failed in "
375
0
            "chacha20-poly1305@hpnssh.org");
376
0
        genKSfailed = 1;
377
0
        break; /* imperfect, but it helps */
378
0
      }
379
0
    }
380
0
  }
381
382
0
  free_threadData(&mainData);
383
384
0
  if (genKSfailed != 0) {
385
0
    chachapoly_free_mt(ctx_mt);
386
0
    explicit_bzero(&startseqnr, sizeof(startseqnr));
387
0
    return NULL;
388
0
  }
389
390
  /* Store the PID so that in the future, we can tell if we're a fork */
391
0
  ctx_mt->mainpid = getpid();
392
0
  ctx_mt->self_tid = pthread_self();
393
0
  ctx_mt->manager_tid[0] = ctx_mt->self_tid;
394
0
  ctx_mt->manager_tid[1] = ctx_mt->self_tid;
395
  /* was reporting the TID using gettid() but it's not portable */
396
0
  debug2_f("<main thread: pid=%u, ptid=0x%lx>", getpid(), pthread_self());
397
398
  /* Success! */
399
0
  explicit_bzero(&startseqnr, sizeof(startseqnr));
400
0
  return ctx_mt;
401
402
0
 fail:
403
#ifdef OPENSSL_HAVE_POLY_EVP
404
  if (ctx_mt->poly_ctx != NULL) {
405
    EVP_MAC_CTX_free(ctx_mt->poly_ctx);
406
    ctx_mt->poly_ctx = NULL;
407
  }
408
#elif !defined(WITH_OPENSSL3) && defined(EVP_PKEY_POLY1305)
409
0
  if (ctx_mt->md_ctx != NULL) {
410
0
    EVP_MD_CTX_free(ctx_mt->md_ctx);
411
0
    ctx_mt->md_ctx = NULL;
412
0
  }
413
0
  if (ctx_mt->pkey != NULL) {
414
0
    EVP_PKEY_free(ctx_mt->pkey);
415
0
    ctx_mt->pkey = NULL;
416
0
  }
417
0
#endif
418
0
  freezero(ctx_mt, sizeof(*ctx_mt));
419
0
  explicit_bzero(&startseqnr, sizeof(startseqnr));
420
0
  return NULL;
421
0
}
422
423
/* a fast method to XOR the keystream against the data */
424
static inline void
425
fastXOR(u_char *dest, const u_char *src, const u_char *keystream, u_int len)
426
0
{
427
428
  /* XXX: this was __uint128_t but that was causing unaligned load errors.
429
   * this works but we need to explore it more. */
430
0
  typedef uint32_t chunk;
431
0
  size_t i;
432
433
0
  for (i=0; i < (len / sizeof(chunk)); i++)
434
0
    ((chunk *)dest)[i]=((chunk *)src)[i]^((chunk *)keystream)[i];
435
0
  for (i=i*(sizeof(chunk) / sizeof(char)); i < len; i++)
436
0
    dest[i]=src[i]^keystream[i];
437
0
}
438
439
struct manager_thread_args *
440
0
manager_thread(struct manager_thread_args * margs) {
441
  /* make sure we have valid data before proceeding */
442
0
  if (margs == NULL)
443
0
    return NULL;
444
445
0
  struct chachapoly_ctx_mt * ctx_mt = margs->ctx_mt;
446
0
  if (ctx_mt == NULL) {
447
0
    margs->retval = 1;
448
0
    return margs;
449
0
  }
450
451
0
  u_int oldBatchID = margs->oldBatchID;
452
453
0
  struct mt_keystream_batch * batch = &(ctx_mt->batches[oldBatchID % 2]);
454
0
  if (batch->batchID != oldBatchID) {
455
0
    debug_f("Post-crypt batch miss! Seeking %u, found %u. Failing.",
456
0
        oldBatchID, batch->batchID);
457
0
    margs->retval = 1;
458
0
    return margs;
459
0
  }
460
461
0
  margs->retval = 0;
462
0
  u_int batchID = oldBatchID + 2;
463
464
0
  pthread_t tid[NUMTHREADS];
465
0
  struct worker_thread_args * wargs = malloc(NUMTHREADS * sizeof(*wargs));
466
0
  int ti;
467
468
0
  for (ti = 0; ti < NUMTHREADS; ti++) {
469
0
    wargs[ti].batchID = batchID;
470
0
    wargs[ti].batch = batch;
471
0
    wargs[ti].threadIndex = ti;
472
0
    wargs[ti].zeros = ctx_mt->zeros;
473
0
    if (pthread_create(&(tid[ti]), NULL, (void *) worker_thread,
474
0
        &(wargs[ti])) != 0) {
475
0
      margs->retval = 1;
476
0
      break;
477
0
    }
478
0
  }
479
0
  for (; ti < NUMTHREADS; ti++) /* for error condition */
480
0
    tid[ti] = pthread_self();
481
482
0
  struct worker_thread_args * retwargs;
483
484
0
  for (ti = 0; ti < NUMTHREADS; ti++) {
485
0
    if (tid[ti] == pthread_self()) {
486
0
      margs->retval = 1; /* redundant, but harmless */
487
0
      continue;
488
0
    }
489
0
    if (pthread_join(tid[ti], (void **) &retwargs) == 0) {
490
0
      if (retwargs == NULL) {
491
0
        debug_f("Worker thread returned NULL!");
492
0
        margs->retval = 1;
493
0
      } else if (retwargs == PTHREAD_CANCELED) {
494
0
        debug_f("Worker thread canceled!");
495
0
        margs->retval = 1;
496
0
      } else {
497
0
        if (retwargs->retval != 0) {
498
0
          debug_f("Worker thread error (%d)",
499
0
              retwargs->retval);
500
0
          margs->retval = 1;
501
0
        }
502
0
        if (retwargs != &(wargs[ti])) {
503
0
          debug_f("Worker thread didn't return "
504
0
              "expected structure!");
505
0
          margs->retval = 1;
506
0
        }
507
0
      }
508
0
    } else {
509
0
      debug_f("pthread_join error!");
510
0
      margs->retval = 1;
511
0
    }
512
0
  }
513
0
  free(wargs);
514
515
0
  if (margs->retval == 0) {
516
0
    batch->batchID = batchID;
517
0
  }
518
519
0
  return margs;
520
0
}
521
522
int
523
chachapoly_crypt_mt(struct chachapoly_ctx_mt *ctx_mt, u_int seqnr, u_char *dest,
524
    const u_char *src, u_int len, u_int aadlen, u_int authlen, int do_encrypt)
525
0
{
526
#ifdef SAFETY
527
  if (ctx_mt->mainpid != getpid()) { /* we're a fork */
528
    /*
529
     * TODO: this is EXTREMELY RARE, may never happen at all (only
530
     * if the fork calls crypt), so we should tell the compiler.
531
     */
532
    /* The worker threads don't exist, we could spawn them? */
533
    debug_f("Fork called crypt without workers!");
534
    chachapoly_free_mt(ctx_mt);
535
    return SSH_ERR_INTERNAL_ERROR;
536
  }
537
#endif
538
539
0
  pthread_t * manager_tid = &(ctx_mt->manager_tid[ctx_mt->batchID % 2]);
540
0
  if (unlikely(*manager_tid != ctx_mt->self_tid)) {
541
0
    int ret = join_manager_thread(*manager_tid);
542
0
    *manager_tid = ctx_mt->self_tid;
543
0
    if (ret != 0)
544
0
      return SSH_ERR_INTERNAL_ERROR;
545
0
  }
546
547
0
  struct mt_keystream_batch * batch =
548
0
      &(ctx_mt->batches[ctx_mt->batchID % 2]);
549
550
0
  struct mt_keystream * ks = &(batch->streams[seqnr % NUMSTREAMS]);
551
552
0
  int r = SSH_ERR_INTERNAL_ERROR;
553
554
#ifdef SAFETY
555
  if (batch->batchID == ctx_mt->batchID) { /* Safety check */
556
#endif
557
    /* check tag before anything else */
558
0
    if (!do_encrypt) {
559
0
      const u_char *tag = src + aadlen + len;
560
0
      u_char expected_tag[POLY1305_TAGLEN];
561
0
#if !defined(WITH_OPENSSL3) && defined(EVP_PKEY_POLY1305)
562
0
      if ((EVP_PKEY_CTX_ctrl(ctx_mt->poly_ctx, -1,
563
0
          EVP_PKEY_OP_SIGNCTX, EVP_PKEY_CTRL_SET_MAC_KEY,
564
0
          POLY1305_KEYLEN, ks->poly_key) <= 0) ||
565
0
          (EVP_DigestSignUpdate(ctx_mt->md_ctx, src, aadlen + len) == 0)) {
566
0
        debug_f("SSL error while decrypting poly1305 tag");
567
0
        return SSH_ERR_INTERNAL_ERROR;
568
0
      }
569
0
      ctx_mt->ptaglen = POLY1305_TAGLEN;
570
0
      if (EVP_DigestSignFinal(ctx_mt->md_ctx, expected_tag,
571
0
          &ctx_mt->ptaglen) == 0) {
572
0
        debug_f("SSL error while finalizing decyrpted poly1305");
573
0
        return SSH_ERR_INTERNAL_ERROR;
574
0
      }
575
#else
576
      poly1305_auth(ctx_mt->poly_ctx, expected_tag, src,
577
          aadlen + len, ks->poly_key);
578
#endif
579
0
      if (timingsafe_bcmp(expected_tag, tag, POLY1305_TAGLEN)
580
0
          != 0)
581
0
        r = SSH_ERR_MAC_INVALID;
582
0
      explicit_bzero(expected_tag, sizeof(expected_tag));
583
0
    }
584
0
    if (r != SSH_ERR_MAC_INVALID) {
585
      /* Crypt additional data (i.e., packet length) */
586
      /* TODO: is aadlen always four bytes? */
587
      /* TODO: do we always have an aadlen? */
588
0
      if (aadlen)
589
0
        for (u_int i=0; i<aadlen; i++)
590
0
          dest[i] = ks->headerStream[i] ^ src[i];
591
      /* Crypt payload */
592
0
      fastXOR(dest+aadlen,src+aadlen,ks->mainStream,len);
593
      /* calculate and append tag */
594
0
#if !defined(WITH_OPENSSL3) && defined(EVP_PKEY_POLY1305)
595
0
      if (do_encrypt) {
596
0
        if ((EVP_PKEY_CTX_ctrl(ctx_mt->poly_ctx, -1,
597
0
            EVP_PKEY_OP_SIGNCTX, EVP_PKEY_CTRL_SET_MAC_KEY,
598
0
            POLY1305_KEYLEN, ks->poly_key) <=0) ||
599
0
            (EVP_DigestSignUpdate(ctx_mt->md_ctx, dest, aadlen + len) == 0)) {
600
0
          debug_f ("SSL error while encrypting poly1305 tag");
601
0
          return SSH_ERR_INTERNAL_ERROR;
602
0
        }
603
0
        ctx_mt->ptaglen = POLY1305_TAGLEN;
604
0
        if (EVP_DigestSignFinal(ctx_mt->md_ctx, dest+aadlen+len,
605
0
            &ctx_mt->ptaglen) == 0) {
606
0
          debug_f("SSL error while finalizing decyrpted poly1305");
607
0
          return SSH_ERR_INTERNAL_ERROR;
608
0
        }
609
0
      }
610
#else
611
      if (do_encrypt)
612
        poly1305_auth(ctx_mt->poly_ctx, dest+aadlen+len,
613
            dest, aadlen+len, ks->poly_key);
614
#endif
615
0
      r=0; /* Success! */
616
0
    }
617
0
    if (r) /* Anything nonzero is an error. */
618
0
      return r;
619
620
0
    ctx_mt->seqnr = seqnr + 1;
621
622
0
    if (unlikely(ctx_mt->seqnr / NUMSTREAMS > ctx_mt->batchID)) {
623
0
      struct manager_thread_args * args =
624
0
          malloc(sizeof(*args));
625
0
      if (args == NULL) {
626
0
        return SSH_ERR_INTERNAL_ERROR;
627
0
      }
628
0
      args->ctx_mt = ctx_mt;
629
0
      args->oldBatchID = ctx_mt->batchID;
630
0
      if (pthread_create(&(ctx_mt->manager_tid[ctx_mt->batchID
631
0
          % 2]), NULL, (void *) manager_thread, args) != 0) {
632
0
        free(args);
633
0
        return SSH_ERR_INTERNAL_ERROR;
634
0
      }
635
0
      ctx_mt->batchID = ctx_mt->seqnr / NUMSTREAMS;
636
0
    }
637
638
    /* TODO: Nothing we need to sanitize here? */
639
640
0
    return 0;
641
#ifdef SAFETY
642
  } else { /* Bad, it's the wrong batch. */
643
    debug_f( "Pre-crypt batch miss! Seeking %u, found %u. Failing.",
644
        ctx_mt->batchID, batch->batchID);
645
    return SSH_ERR_INTERNAL_ERROR;
646
  }
647
#endif
648
0
}
649
650
int
651
chachapoly_get_length_mt(struct chachapoly_ctx_mt *ctx_mt, u_int *plenp,
652
    u_int seqnr, const u_char *cp, u_int len)
653
0
{
654
  /* TODO: add compiler hints */
655
#ifdef SAFETY
656
  if (ctx_mt->mainpid != getpid()) { /* Use serial mode if we're a fork */
657
    debug_f("We're a fork. Failing.");
658
    return SSH_ERR_INTERNAL_ERROR;
659
  }
660
#endif
661
662
0
  if (len < 4)
663
0
    return SSH_ERR_MESSAGE_INCOMPLETE;
664
665
0
  pthread_t * manager_tid = &(ctx_mt->manager_tid[ctx_mt->batchID % 2]);
666
0
  if (unlikely(*manager_tid != ctx_mt->self_tid)) {
667
0
    int ret = join_manager_thread(*manager_tid);
668
0
    *manager_tid = ctx_mt->self_tid;
669
0
    if (ret != 0)
670
0
      return SSH_ERR_INTERNAL_ERROR;
671
0
  }
672
673
0
  u_char buf[4];
674
#ifdef SAFETY
675
  u_int sought_batchID = seqnr / NUMSTREAMS;
676
#endif
677
0
  struct mt_keystream_batch * batch =
678
0
      &(ctx_mt->batches[ctx_mt->batchID % 2]);
679
0
  struct mt_keystream * ks = &(batch->streams[seqnr % NUMSTREAMS]);
680
#ifdef SAFETY
681
  if (batch->batchID == sought_batchID) {
682
#endif
683
0
    for (u_int i=0; i < sizeof(buf); i++)
684
0
      buf[i]=ks->headerStream[i] ^ cp[i];
685
0
    *plenp = PEEK_U32(buf);
686
0
    return 0;
687
#ifdef SAFETY
688
  } else {
689
    debug_f("Batch miss! Seeking %u, found %u. Failing.",
690
        sought_batchID, batch->batchID);
691
    return SSH_ERR_INTERNAL_ERROR;
692
  }
693
#endif
694
0
}
695
#endif /* defined(HAVE_EVP_CHACHA20) && !defined(HAVE_BROKEN_CHACHA20) */