/src/libvncserver/src/common/crypto_openssl.c
Line | Count | Source (jump to first uncovered line) |
1 | | /* |
2 | | * crypto_openssl.c - Crypto wrapper (openssl version) |
3 | | */ |
4 | | |
5 | | /* |
6 | | * Copyright (C) 2011 Gernot Tenchio |
7 | | * Copyright (C) 2019 Christian Beier |
8 | | * |
9 | | * This is free software; you can redistribute it and/or modify |
10 | | * it under the terms of the GNU General Public License as published by |
11 | | * the Free Software Foundation; either version 2 of the License, or |
12 | | * (at your option) any later version. |
13 | | * |
14 | | * This software is distributed in the hope that it will be useful, |
15 | | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
16 | | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
17 | | * GNU General Public License for more details. |
18 | | * |
19 | | * You should have received a copy of the GNU General Public License |
20 | | * along with this software; if not, write to the Free Software |
21 | | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, |
22 | | * USA. |
23 | | */ |
24 | | |
25 | | #include <string.h> |
26 | | #include <openssl/sha.h> |
27 | | #include <openssl/md5.h> |
28 | | #include <openssl/dh.h> |
29 | | #include <openssl/evp.h> |
30 | | #include <openssl/rand.h> |
31 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
32 | | #include <openssl/provider.h> |
33 | | #endif |
34 | | #include "crypto.h" |
35 | | |
36 | 0 | static unsigned char reverseByte(unsigned char b) { |
37 | 0 | b = (b & 0xF0) >> 4 | (b & 0x0F) << 4; |
38 | 0 | b = (b & 0xCC) >> 2 | (b & 0x33) << 2; |
39 | 0 | b = (b & 0xAA) >> 1 | (b & 0x55) << 1; |
40 | 0 | return b; |
41 | 0 | } |
42 | | |
43 | | int hash_md5(void *out, const void *in, const size_t in_len) |
44 | 0 | { |
45 | 0 | MD5_CTX md5; |
46 | 0 | if(!MD5_Init(&md5)) |
47 | 0 | return 0; |
48 | 0 | if(!MD5_Update(&md5, in, in_len)) |
49 | 0 | return 0; |
50 | 0 | if(!MD5_Final(out, &md5)) |
51 | 0 | return 0; |
52 | 0 | return 1; |
53 | 0 | } |
54 | | |
55 | | int hash_sha1(void *out, const void *in, const size_t in_len) |
56 | 0 | { |
57 | 0 | SHA_CTX sha1; |
58 | 0 | if(!SHA1_Init(&sha1)) |
59 | 0 | return 0; |
60 | 0 | if(!SHA1_Update(&sha1, in, in_len)) |
61 | 0 | return 0; |
62 | 0 | if(!SHA1_Final(out, &sha1)) |
63 | 0 | return 0; |
64 | 0 | return 1; |
65 | 0 | } |
66 | | |
67 | | void random_bytes(void *out, size_t len) |
68 | 0 | { |
69 | 0 | RAND_bytes(out, len); |
70 | 0 | } |
71 | | |
72 | | int encrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len) |
73 | 0 | { |
74 | 0 | int result = 0; |
75 | 0 | EVP_CIPHER_CTX *des = NULL; |
76 | 0 | unsigned char mungedkey[8]; |
77 | 0 | int i; |
78 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
79 | | OSSL_PROVIDER *providerLegacy = NULL; |
80 | | OSSL_PROVIDER *providerDefault = NULL; |
81 | | #endif |
82 | |
|
83 | 0 | for (i = 0; i < 8; i++) |
84 | 0 | mungedkey[i] = reverseByte(key[i]); |
85 | |
|
86 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
87 | | /* Load Multiple providers into the default (NULL) library context */ |
88 | | if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy"))) |
89 | | goto out; |
90 | | if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default"))) |
91 | | goto out; |
92 | | #endif |
93 | |
|
94 | 0 | if(!(des = EVP_CIPHER_CTX_new())) |
95 | 0 | goto out; |
96 | 0 | if(!EVP_EncryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL)) |
97 | 0 | goto out; |
98 | 0 | if(!EVP_EncryptUpdate(des, out, out_len, in, in_len)) |
99 | 0 | goto out; |
100 | | |
101 | 0 | result = 1; |
102 | |
|
103 | 0 | out: |
104 | 0 | if (des) |
105 | 0 | EVP_CIPHER_CTX_free(des); |
106 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
107 | | if (providerLegacy) |
108 | | OSSL_PROVIDER_unload(providerLegacy); |
109 | | if (providerDefault) |
110 | | OSSL_PROVIDER_unload(providerDefault); |
111 | | #endif |
112 | 0 | return result; |
113 | 0 | } |
114 | | |
115 | | int decrypt_rfbdes(void *out, int *out_len, const unsigned char key[8], const void *in, const size_t in_len) |
116 | 0 | { |
117 | 0 | int result = 0; |
118 | 0 | EVP_CIPHER_CTX *des = NULL; |
119 | 0 | unsigned char mungedkey[8]; |
120 | 0 | int i; |
121 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
122 | | OSSL_PROVIDER *providerLegacy = NULL; |
123 | | OSSL_PROVIDER *providerDefault = NULL; |
124 | | #endif |
125 | |
|
126 | 0 | for (i = 0; i < 8; i++) |
127 | 0 | mungedkey[i] = reverseByte(key[i]); |
128 | |
|
129 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
130 | | /* Load Multiple providers into the default (NULL) library context */ |
131 | | if (!(providerLegacy = OSSL_PROVIDER_load(NULL, "legacy"))) |
132 | | goto out; |
133 | | if (!(providerDefault = OSSL_PROVIDER_load(NULL, "default"))) |
134 | | goto out; |
135 | | #endif |
136 | |
|
137 | 0 | if(!(des = EVP_CIPHER_CTX_new())) |
138 | 0 | goto out; |
139 | 0 | if(!EVP_DecryptInit_ex(des, EVP_des_ecb(), NULL, mungedkey, NULL)) |
140 | 0 | goto out; |
141 | 0 | if(!EVP_CIPHER_CTX_set_padding(des, 0)) |
142 | 0 | goto out; |
143 | 0 | if(!EVP_DecryptUpdate(des, out, out_len, in, in_len)) |
144 | 0 | goto out; |
145 | | |
146 | 0 | result = 1; |
147 | |
|
148 | 0 | out: |
149 | 0 | if (des) |
150 | 0 | EVP_CIPHER_CTX_free(des); |
151 | | #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) |
152 | | if (providerLegacy) |
153 | | OSSL_PROVIDER_unload(providerLegacy); |
154 | | if (providerDefault) |
155 | | OSSL_PROVIDER_unload(providerDefault); |
156 | | #endif |
157 | 0 | return result; |
158 | 0 | } |
159 | | |
160 | | int encrypt_aes128ecb(void *out, int *out_len, const unsigned char key[16], const void *in, const size_t in_len) |
161 | 0 | { |
162 | 0 | int result = 0; |
163 | 0 | EVP_CIPHER_CTX *aes; |
164 | |
|
165 | 0 | if(!(aes = EVP_CIPHER_CTX_new())) |
166 | 0 | goto out; |
167 | 0 | EVP_CIPHER_CTX_set_padding(aes, 0); |
168 | 0 | if(!EVP_EncryptInit_ex(aes, EVP_aes_128_ecb(), NULL, key, NULL)) |
169 | 0 | goto out; |
170 | 0 | if(!EVP_EncryptUpdate(aes, out, out_len, in, in_len)) |
171 | 0 | goto out; |
172 | | |
173 | 0 | result = 1; |
174 | |
|
175 | 0 | out: |
176 | 0 | EVP_CIPHER_CTX_free(aes); |
177 | 0 | return result; |
178 | 0 | } |
179 | | |
180 | 0 | static void pad_leading_zeros(uint8_t *out, const size_t current_len, const size_t expected_len) { |
181 | 0 | if (current_len >= expected_len || expected_len < 1) |
182 | 0 | return; |
183 | | |
184 | 0 | size_t diff = expected_len - current_len; |
185 | 0 | memmove(out + diff, out, current_len); |
186 | 0 | memset(out, 0, diff); |
187 | 0 | } |
188 | | |
189 | | int dh_generate_keypair(uint8_t *priv_out, uint8_t *pub_out, const uint8_t *gen, const size_t gen_len, const uint8_t *prime, const size_t keylen) |
190 | 0 | { |
191 | 0 | int result = 0; |
192 | 0 | DH *dh; |
193 | 0 | #if OPENSSL_VERSION_NUMBER >= 0x10100000L || \ |
194 | 0 | (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER >= 0x30500000) |
195 | 0 | const BIGNUM *pub_key = NULL; |
196 | 0 | const BIGNUM *priv_key = NULL; |
197 | 0 | #endif |
198 | |
|
199 | 0 | if(!(dh = DH_new())) |
200 | 0 | goto out; |
201 | | #if OPENSSL_VERSION_NUMBER < 0x10100000L || \ |
202 | | (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000) |
203 | | dh->p = BN_bin2bn(prime, keylen, NULL); |
204 | | dh->g = BN_bin2bn(gen, gen_len, NULL); |
205 | | #else |
206 | 0 | if(!DH_set0_pqg(dh, BN_bin2bn(prime, keylen, NULL), NULL, BN_bin2bn(gen, gen_len, NULL))) |
207 | 0 | goto out; |
208 | 0 | #endif |
209 | 0 | if(!DH_generate_key(dh)) |
210 | 0 | goto out; |
211 | | #if OPENSSL_VERSION_NUMBER < 0x10100000L || \ |
212 | | (defined (LIBRESSL_VERSION_NUMBER) && LIBRESSL_VERSION_NUMBER < 0x30500000) |
213 | | if(BN_bn2bin(dh->priv_key, priv_out) == 0) |
214 | | goto out; |
215 | | if(BN_bn2bin(dh->pub_key, pub_out) == 0) |
216 | | goto out; |
217 | | |
218 | | pad_leading_zeros(priv_out, BN_num_bytes(dh->priv_key), keylen); |
219 | | pad_leading_zeros(pub_out, BN_num_bytes(dh->pub_key), keylen); |
220 | | #else |
221 | 0 | DH_get0_key(dh, &pub_key, &priv_key); |
222 | 0 | if(BN_bn2binpad(priv_key, priv_out, keylen) == -1) |
223 | 0 | goto out; |
224 | 0 | if(BN_bn2binpad(pub_key, pub_out, keylen) == -1) |
225 | 0 | goto out; |
226 | 0 | #endif |
227 | | |
228 | 0 | result = 1; |
229 | |
|
230 | 0 | out: |
231 | 0 | DH_free(dh); |
232 | 0 | return result; |
233 | 0 | } |
234 | | |
235 | | int dh_compute_shared_key(uint8_t *shared_out, const uint8_t *priv, const uint8_t *pub, const uint8_t *prime, const size_t keylen) |
236 | 0 | { |
237 | 0 | int result = 0; |
238 | 0 | DH *dh; |
239 | |
|
240 | 0 | if(!(dh = DH_new())) |
241 | 0 | goto out; |
242 | | #if OPENSSL_VERSION_NUMBER < 0x10100000L || \ |
243 | | (defined LIBRESSL_VERSION_NUMBER && LIBRESSL_VERSION_NUMBER < 0x30500000) |
244 | | dh->p = BN_bin2bn(prime, keylen, NULL); |
245 | | dh->priv_key = BN_bin2bn(priv, keylen, NULL); |
246 | | #else |
247 | 0 | if(!DH_set0_pqg(dh, BN_bin2bn(prime, keylen, NULL), NULL, BN_new())) |
248 | 0 | goto out; |
249 | 0 | if(!DH_set0_key(dh, NULL, BN_bin2bn(priv, keylen, NULL))) |
250 | 0 | goto out; |
251 | 0 | #endif |
252 | 0 | int shared_len = DH_compute_key(shared_out, BN_bin2bn(pub, keylen, NULL), dh); |
253 | 0 | if(shared_len == -1) |
254 | 0 | goto out; |
255 | | |
256 | 0 | pad_leading_zeros(shared_out, shared_len, keylen); |
257 | 0 | result = 1; |
258 | |
|
259 | 0 | out: |
260 | 0 | DH_free(dh); |
261 | 0 | return result; |
262 | 0 | } |