Coverage Report

Created: 2025-07-11 06:24

/src/libvncserver/src/common/crypto_openssl.c
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * crypto_openssl.c - Crypto wrapper (openssl version)
3
 */
4
5
/*
6
 *  Copyright (C) 2011 Gernot Tenchio
7
 *  Copyright (C) 2019 Christian Beier
8
 *
9
 *  This is free software; you can redistribute it and/or modify
10
 *  it under the terms of the GNU General Public License as published by
11
 *  the Free Software Foundation; either version 2 of the License, or
12
 *  (at your option) any later version.
13
 *
14
 *  This software is distributed in the hope that it will be useful,
15
 *  but WITHOUT ANY WARRANTY; without even the implied warranty of
16
 *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
17
 *  GNU General Public License for more details.
18
 *
19
 *  You should have received a copy of the GNU General Public License
20
 *  along with this software; if not, write to the Free Software
21
 *  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
22
 *  USA.
23
 */
24
25
#include <string.h>
26
#include <openssl/sha.h>
27
#include <openssl/md5.h>
28
#include <openssl/dh.h>
29
#include <openssl/evp.h>
30
#include <openssl/rand.h>
31
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
32
#include <openssl/provider.h>
33
#endif
34
#include "crypto.h"
35
36
0
static unsigned char reverseByte(unsigned char b) {
37
0
   b = (b & 0xF0) >> 4 | (b & 0x0F) << 4;
38
0
   b = (b & 0xCC) >> 2 | (b & 0x33) << 2;
39
0
   b = (b & 0xAA) >> 1 | (b & 0x55) << 1;
40
0
   return b;
41
0
}
42
43
int hash_md5(void *out, const void *in, const size_t in_len)
44
0
{
45
0
    MD5_CTX md5;
46
0
    if(!MD5_Init(&md5))
47
0
  return 0;
48
0
    if(!MD5_Update(&md5, in, in_len))
49
0
  return 0;
50
0
    if(!MD5_Final(out, &md5))
51
0
  return 0;
52
0
    return 1;
53
0
}
54
55
int hash_sha1(void *out, const void *in, const size_t in_len)
56
0
{
57
0
    SHA_CTX sha1;
58
0
    if(!SHA1_Init(&sha1))
59
0
  return 0;
60
0
    if(!SHA1_Update(&sha1, in, in_len))
61
0
  return 0;
62
0
    if(!SHA1_Final(out, &sha1))
63
0
  return 0;
64
0
    return 1;
65
0
}
66
67
void random_bytes(void *out, size_t len)
68
0
{
69
0
    RAND_bytes(out, len);
70
0
}
71
72
int encrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
73
0
{
74
0
    int result = 0;
75
0
    EVP_CIPHER_CTX *des = NULL;
76
0
    unsigned char mungedkey[8];
77
0
    int i;
78
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
79
    OSSL_PROVIDER *providerLegacy = NULL;
80
    OSSL_PROVIDER *providerDefault = NULL;
81
#endif
82
83
0
    for (i = 0; i < 8; i++)
84
0
      mungedkey[i] = reverseByte(key[i]);
85
86
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
87
    /* Load Multiple providers into the default (NULL) library context */
88
    if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy")))
89
      goto out;
90
    if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default")))
91
      goto out;
92
#endif
93
94
0
    if(!(des = EVP_CIPHER_CTX_new()))
95
0
  goto out;
96
0
    if(!EVP_EncryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL))
97
0
  goto out;
98
0
    if(!EVP_EncryptUpdate(des, out, out_len, in, in_len))
99
0
  goto out;
100
101
0
    result = 1;
102
103
0
 out:
104
0
    if (des)
105
0
      EVP_CIPHER_CTX_free(des);
106
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
107
    if (providerLegacy)
108
      OSSL_PROVIDER_unload(providerLegacy);
109
    if (providerDefault)
110
      OSSL_PROVIDER_unload(providerDefault);
111
#endif
112
0
    return result;
113
0
}
114
115
int decrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len)
116
0
{
117
0
    int result = 0;
118
0
    EVP_CIPHER_CTX *des = NULL;
119
0
    unsigned char mungedkey[8];
120
0
    int i;
121
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
122
    OSSL_PROVIDER *providerLegacy = NULL;
123
    OSSL_PROVIDER *providerDefault = NULL;
124
#endif
125
126
0
    for (i = 0; i < 8; i++)
127
0
      mungedkey[i] = reverseByte(key[i]);
128
129
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
130
    /* Load Multiple providers into the default (NULL) library context */
131
    if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy")))
132
  goto out;
133
    if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default")))
134
  goto out;
135
#endif
136
137
0
    if(!(des = EVP_CIPHER_CTX_new()))
138
0
  goto out;
139
0
    if(!EVP_DecryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL))
140
0
  goto out;
141
0
    if(!EVP_CIPHER_CTX_set_padding(des, 0))
142
0
  goto out;
143
0
    if(!EVP_DecryptUpdate(des, out, out_len, in, in_len))
144
0
  goto out;
145
146
0
    result = 1;
147
148
0
 out:
149
0
    if (des)
150
0
  EVP_CIPHER_CTX_free(des);
151
#if (OPENSSL_VERSION_NUMBER >= 0x30000000L)
152
    if (providerLegacy)
153
  OSSL_PROVIDER_unload(providerLegacy);
154
    if (providerDefault)
155
  OSSL_PROVIDER_unload(providerDefault);
156
#endif
157
0
    return result;
158
0
}
159
160
int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], const void *in, const size_t in_len)
161
0
{
162
0
    int result = 0;
163
0
    EVP_CIPHER_CTX *aes;
164
165
0
    if(!(aes = EVP_CIPHER_CTX_new()))
166
0
  goto out;
167
0
    EVP_CIPHER_CTX_set_padding(aes, 0);
168
0
    if(!EVP_EncryptInit_ex(aes, EVP_aes_128_ecb(), NULL, key, NULL))
169
0
  goto out;
170
0
    if(!EVP_EncryptUpdate(aes, out, out_len, in, in_len))
171
0
  goto out;
172
173
0
    result = 1;
174
175
0
 out:
176
0
    EVP_CIPHER_CTX_free(aes);
177
0
    return result;
178
0
}
179
180
0
static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) {
181
0
    if (current_len >= expected_len || expected_len < 1)
182
0
        return;
183
184
0
    size_t diff = expected_len - current_len;
185
0
    memmove(out + diff, out, current_len);
186
0
    memset(out, 0, diff);
187
0
}
188
189
int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen)
190
0
{
191
0
    int result = 0;
192
0
    DH *dh;
193
0
#if OPENSSL_VERSION_NUMBER >= 0x10100000L || \
194
0
  (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x30500000)
195
0
    const BIGNUM *pub_key = NULL;
196
0
    const BIGNUM *priv_key = NULL;
197
0
#endif
198
199
0
    if(!(dh = DH_new()))
200
0
  goto out;
201
#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
202
  (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
203
    dh->p = BN_bin2bn(prime, keylen, NULL);
204
    dh->g = BN_bin2bn(gen, gen_len, NULL);
205
#else
206
0
    if(!DH_set0_pqg(dh, BN_bin2bn(prime, keylen, NULL), NULL, BN_bin2bn(gen, gen_len, NULL)))
207
0
  goto out;
208
0
#endif
209
0
    if(!DH_generate_key(dh))
210
0
  goto out;
211
#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
212
  (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000)
213
    if(BN_bn2bin(dh->priv_key, priv_out) == 0)
214
  goto out;
215
    if(BN_bn2bin(dh->pub_key, pub_out) == 0)
216
  goto out;
217
218
    pad_leading_zeros(priv_out, BN_num_bytes(dh->priv_key), keylen);
219
    pad_leading_zeros(pub_out, BN_num_bytes(dh->pub_key), keylen);
220
#else
221
0
    DH_get0_key(dh, &pub_key, &priv_key);
222
0
    if(BN_bn2binpad(priv_key, priv_out, keylen) == -1)
223
0
  goto out;
224
0
    if(BN_bn2binpad(pub_key, pub_out, keylen) == -1)
225
0
  goto out;
226
0
#endif
227
228
0
    result = 1;
229
230
0
 out:
231
0
    DH_free(dh);
232
0
    return result;
233
0
}
234
235
int dh_compute_shared_key(uint8_t *shared_out, const uint8_t *priv, const uint8_t *pub, const uint8_t *prime, const size_t keylen)
236
0
{
237
0
    int result = 0;
238
0
    DH *dh;
239
240
0
    if(!(dh = DH_new()))
241
0
  goto out;
242
#if OPENSSL_VERSION_NUMBER < 0x10100000L || \
243
  (defined LIBRESSL_VERSION_NUMBER && LIBRESSL_VERSION_NUMBER < 0x30500000)
244
    dh->p = BN_bin2bn(prime, keylen, NULL);
245
    dh->priv_key = BN_bin2bn(priv, keylen, NULL);
246
#else
247
0
    if(!DH_set0_pqg(dh, BN_bin2bn(prime, keylen, NULL), NULL, BN_new()))
248
0
  goto out;
249
0
    if(!DH_set0_key(dh, NULL, BN_bin2bn(priv, keylen, NULL)))
250
0
  goto out;
251
0
#endif
252
0
    int shared_len = DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh);
253
0
    if(shared_len == -1)
254
0
        goto out;
255
256
0
    pad_leading_zeros(shared_out, shared_len, keylen);
257
0
    result = 1;
258
259
0
 out:
260
0
    DH_free(dh);
261
0
    return result;
262
0
}