Coverage Report

Created: 2026-04-01 06:40

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libvncserver/src/common/crypto_openssl.c
Line
Count
Source
1
/*
2
 * crypto_openssl.c - Crypto wrapper (openssl version)
3
 */
4
5
/*
6
 *  Copyright (C) 2011 Gernot Tenchio
7
 *  Copyright (C) 2019 Christian Beier
8
 *
9
 *  This is free software; you can redistribute it and/or modify
10
 *  it under the terms of the GNU General Public License as published by
11
 *  the Free Software Foundation; either version 2 of the License, or
12
 *  (at your option) any later version.
13
 *
14
 *  This software is distributed in the hope that it will be useful,
15
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 *  GNU General Public License for more details.
18
 *
19
 *  You should have received a copy of the GNU General Public License
20
 *  along with this software; if not, write to the Free Software
21
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
22
 *  USA.
23
 */
24
25
#include <string.h>
26
#include <openssl/sha.h>
27
#include <openssl/md5.h>
28
#include <openssl/dh.h>
29
#include <openssl/evp.h>
30
#include <openssl/rand.h>
31
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
32
#include <openssl/provider.h>
33
#endif
34
#include "crypto.h"
35
36
0
static unsigned char reverseByte(unsigned char b) {
37
0
   b = (b & 0xF0) >> 4 | (b & 0x0F) << 4;
38
0
   b = (b & 0xCC) >> 2 | (b & 0x33) << 2;
39
0
   b = (b & 0xAA) >> 1 | (b & 0x55) << 1;
40
0
   return b;
41
0
}
42
43
int hash_md5(void *out, const void *in, const size_t in_len)
44
0
{
45
0
    MD5_CTX md5;
46
0
    if(!MD5_Init(&md5))
47
0
  return 0;
48
0
    if(!MD5_Update(&md5, in, in_len))
49
0
  return 0;
50
0
    if(!MD5_Final(out, &md5))
51
0
  return 0;
52
0
    return 1;
53
0
}
54
55
#ifdef LIBVNCSERVER_WITH_WEBSOCKETS
56
int hash_sha1(void *out, const void *in, const size_t in_len)
57
0
{
58
0
    SHA_CTX sha1;
59
0
    if(!SHA1_Init(&sha1))
60
0
  return 0;
61
0
    if(!SHA1_Update(&sha1, in, in_len))
62
0
  return 0;
63
0
    if(!SHA1_Final(out, &sha1))
64
0
  return 0;
65
0
    return 1;
66
0
}
67
#endif /* LIBVNCSERVER_WITH_WEBSOCKETS */
68
69
void random_bytes(void *out, size_t len)
70
0
{
71
0
    RAND_bytes(out, len);
72
0
}
73
74
int encrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
75
0
{
76
0
    int result = 0;
77
0
    EVP_CIPHER_CTX *des = NULL;
78
0
    unsigned char mungedkey[8];
79
0
    int i;
80
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
81
    OSSL_PROVIDER *providerLegacy = NULL;
82
    OSSL_PROVIDER *providerDefault = NULL;
83
#endif
84
85
0
    for (i = 0; i < 8; i++)
86
0
      mungedkey[i] = reverseByte(key[i]);
87
88
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
89
    /* Load Multiple providers into the default (NULL) library context */
90
    if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy")))
91
      goto out;
92
    if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default")))
93
      goto out;
94
#endif
95
96
0
    if(!(des = EVP_CIPHER_CTX_new()))
97
0
  goto out;
98
0
    if(!EVP_EncryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL))
99
0
  goto out;
100
0
    if(!EVP_EncryptUpdate(des, out, out_len, in, in_len))
101
0
  goto out;
102
103
0
    result = 1;
104
105
0
 out:
106
0
    if (des)
107
0
      EVP_CIPHER_CTX_free(des);
108
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
109
    if (providerLegacy)
110
      OSSL_PROVIDER_unload(providerLegacy);
111
    if (providerDefault)
112
      OSSL_PROVIDER_unload(providerDefault);
113
#endif
114
0
    return result;
115
0
}
116
117
int decrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
118
0
{
119
0
    int result = 0;
120
0
    EVP_CIPHER_CTX *des = NULL;
121
0
    unsigned char mungedkey[8];
122
0
    int i;
123
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
124
    OSSL_PROVIDER *providerLegacy = NULL;
125
    OSSL_PROVIDER *providerDefault = NULL;
126
#endif
127
128
0
    for (i = 0; i < 8; i++)
129
0
      mungedkey[i] = reverseByte(key[i]);
130
131
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
132
    /* Load Multiple providers into the default (NULL) library context */
133
    if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy")))
134
  goto out;
135
    if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default")))
136
  goto out;
137
#endif
138
139
0
    if(!(des = EVP_CIPHER_CTX_new()))
140
0
  goto out;
141
0
    if(!EVP_DecryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL))
142
0
  goto out;
143
0
    if(!EVP_CIPHER_CTX_set_padding(des, 0))
144
0
  goto out;
145
0
    if(!EVP_DecryptUpdate(des, out, out_len, in, in_len))
146
0
  goto out;
147
148
0
    result = 1;
149
150
0
 out:
151
0
    if (des)
152
0
  EVP_CIPHER_CTX_free(des);
153
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
154
    if (providerLegacy)
155
  OSSL_PROVIDER_unload(providerLegacy);
156
    if (providerDefault)
157
  OSSL_PROVIDER_unload(providerDefault);
158
#endif
159
0
    return result;
160
0
}
161
162
int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], const void *in, const size_t in_len)
163
0
{
164
0
    int result = 0;
165
0
    EVP_CIPHER_CTX *aes;
166
167
0
    if(!(aes = EVP_CIPHER_CTX_new()))
168
0
  goto out;
169
0
    EVP_CIPHER_CTX_set_padding(aes, 0);
170
0
    if(!EVP_EncryptInit_ex(aes, EVP_aes_128_ecb(), NULL, key, NULL))
171
0
  goto out;
172
0
    if(!EVP_EncryptUpdate(aes, out, out_len, in, in_len))
173
0
  goto out;
174
175
0
    result = 1;
176
177
0
 out:
178
0
    EVP_CIPHER_CTX_free(aes);
179
0
    return result;
180
0
}
181
182
0
static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) {
183
0
    if (current_len >= expected_len || expected_len < 1)
184
0
        return;
185
186
0
    size_t diff = expected_len - current_len;
187
0
    memmove(out + diff, out, current_len);
188
0
    memset(out, 0, diff);
189
0
}
190
191
int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen)
192
0
{
193
0
    int result = 0;
194
0
    DH *dh;
195
0
#if OPENSSL_VERSION_NUMBER >= 0x10100000L || \
196
0
  (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x30500000)
197
0
    const BIGNUM *pub_key = NULL;
198
0
    const BIGNUM *priv_key = NULL;
199
0
#endif
200
201
0
    if(!(dh = DH_new()))
202
0
  goto out;
203
#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
204
  (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
205
    dh->p = BN_bin2bn(prime, keylen, NULL);
206
    dh->g = BN_bin2bn(gen, gen_len, NULL);
207
#else
208
0
    if(!DH_set0_pqg(dh, BN_bin2bn(prime, keylen, NULL), NULL, BN_bin2bn(gen, gen_len, NULL)))
209
0
  goto out;
210
0
#endif
211
0
    if(!DH_generate_key(dh))
212
0
  goto out;
213
#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
214
  (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
215
    if(BN_bn2bin(dh->priv_key, priv_out) == 0)
216
  goto out;
217
    if(BN_bn2bin(dh->pub_key, pub_out) == 0)
218
  goto out;
219
220
    pad_leading_zeros(priv_out, BN_num_bytes(dh->priv_key), keylen);
221
    pad_leading_zeros(pub_out, BN_num_bytes(dh->pub_key), keylen);
222
#else
223
0
    DH_get0_key(dh, &pub_key, &priv_key);
224
0
    if(BN_bn2binpad(priv_key, priv_out, keylen) == -1)
225
0
  goto out;
226
0
    if(BN_bn2binpad(pub_key, pub_out, keylen) == -1)
227
0
  goto out;
228
0
#endif
229
230
0
    result = 1;
231
232
0
 out:
233
0
    DH_free(dh);
234
0
    return result;
235
0
}
236
237
int dh_compute_shared_key(uint8_t *shared_out, const uint8_t *priv, const uint8_t *pub, const uint8_t *prime, const size_t keylen)
238
0
{
239
0
    int result = 0;
240
0
    DH *dh;
241
242
0
    if(!(dh = DH_new()))
243
0
  goto out;
244
#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
245
  (defined LIBRESSL_VERSION_NUMBER && LIBRESSL_VERSION_NUMBER < 0x30500000)
246
    dh->p = BN_bin2bn(prime, keylen, NULL);
247
    dh->priv_key = BN_bin2bn(priv, keylen, NULL);
248
#else
249
0
    if(!DH_set0_pqg(dh, BN_bin2bn(prime, keylen, NULL), NULL, BN_new()))
250
0
  goto out;
251
0
    if(!DH_set0_key(dh, NULL, BN_bin2bn(priv, keylen, NULL)))
252
0
  goto out;
253
0
#endif
254
0
    int shared_len = DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh);
255
0
    if(shared_len == -1)
256
0
        goto out;
257
258
0
    pad_leading_zeros(shared_out, shared_len, keylen);
259
0
    result = 1;
260
261
0
 out:
262
0
    DH_free(dh);
263
0
    return result;
264
0
}