Coverage Report

Created: 2025-08-28 06:26

/src/serenity/Userland/Libraries/LibCrypto/Padding/OAEP.h
Line
Count
Source (jump to first uncovered line)
1
/*
2
 * Copyright (c) 2024, stelar7 <dudedbz@gmail.com>
3
 *
4
 * SPDX-License-Identifier: BSD-2-Clause
5
 */
6
7
#pragma once
8
9
#include <AK/ByteBuffer.h>
10
#include <AK/ByteReader.h>
11
#include <AK/Endian.h>
12
#include <AK/Function.h>
13
#include <AK/Random.h>
14
#include <LibCrypto/BigInt/UnsignedBigInteger.h>
15
16
namespace Crypto::Padding {
17
18
// https://datatracker.ietf.org/doc/html/rfc2437#section-9.1.1
19
class OAEP {
20
public:
21
    // https://datatracker.ietf.org/doc/html/rfc2437#section-9.1.1.1
22
    template<typename HashFunction, typename MaskGenerationFunction>
23
    static ErrorOr<ByteBuffer> encode(ReadonlyBytes message, ReadonlyBytes parameters, size_t length, Function<void(Bytes)> seed_function = fill_with_random)
24
    {
25
        // FIXME: 1. If the length of P is greater than the input limitation for the
26
        // hash function (2^61-1 octets for SHA-1) then output "parameter string
27
        // too long" and stop.
28
29
        // 2. If ||M|| > emLen - 2hLen - 1 then output "message too long" and stop.
30
        auto h_len = HashFunction::digest_size();
31
        auto max_message_size = length - (2 * h_len) - 1;
32
        if (message.size() > max_message_size)
33
            return Error::from_string_literal("message too long");
34
35
        // 3. Generate an octet string PS consisting of emLen-||M||-2hLen-1 zero octets. The length of PS may be 0.
36
        auto padding_size = length - message.size() - (2 * h_len) - 1;
37
        auto ps = TRY(ByteBuffer::create_zeroed(padding_size));
38
39
        // 4. Let pHash = Hash(P), an octet string of length hLen.
40
        HashFunction hash;
41
        hash.update(parameters);
42
        auto digest = hash.digest();
43
        auto p_hash = digest.bytes();
44
45
        // 5. Concatenate pHash, PS, the message M, and other padding to form a data block DB as: DB = pHash || PS || 01 || M
46
        auto db = TRY(ByteBuffer::create_uninitialized(0));
47
        TRY(db.try_append(p_hash));
48
        TRY(db.try_append(ps.bytes()));
49
        TRY(db.try_append(0x01));
50
        TRY(db.try_append(message));
51
52
        // 6. Generate a random octet string seed of length hLen.
53
        auto seed = TRY(ByteBuffer::create_uninitialized(h_len));
54
        seed_function(seed);
55
56
        // 7. Let dbMask = MGF(seed, emLen-hLen).
57
        auto db_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(seed, length - h_len));
58
59
        // 8. Let maskedDB = DB \xor dbMask.
60
        auto masked_db = TRY(ByteBuffer::xor_buffers(db, db_mask));
61
62
        // 9. Let seedMask = MGF(maskedDB, hLen).
63
        auto seed_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(masked_db, h_len));
64
65
        // 10. Let maskedSeed = seed \xor seedMask.
66
        auto masked_seed = TRY(ByteBuffer::xor_buffers(seed, seed_mask));
67
68
        // 11. Let EM = maskedSeed || maskedDB.
69
        auto em = TRY(ByteBuffer::create_uninitialized(0));
70
        TRY(em.try_append(masked_seed));
71
        TRY(em.try_append(masked_db));
72
73
        // 12. Output EM.
74
        return em;
75
    }
76
77
    // https://www.rfc-editor.org/rfc/rfc3447#section-7.1.1
78
    template<typename HashFunction, typename MaskGenerationFunction>
79
    static ErrorOr<ByteBuffer> eme_encode(ReadonlyBytes message, ReadonlyBytes label, u32 rsa_modulus_n, Function<void(Bytes)> seed_function = fill_with_random)
80
0
    {
81
        // FIXME: 1. If the length of L is greater than the input limitation for the
82
        //           hash function (2^61 - 1 octets for SHA-1), output "label too
83
        //           long" and stop.
84
85
        // 2. If mLen > k - 2hLen - 2, output "message too long" and stop.
86
0
        auto m_len = message.size();
87
0
        auto k = rsa_modulus_n;
88
0
        auto h_len = HashFunction::digest_size();
89
0
        auto max_message_size = k - (2 * h_len) - 2;
90
91
0
        if (m_len > max_message_size)
92
0
            return Error::from_string_view("message too long"sv);
93
94
        // 3. If the label L is not provided, let L be the empty string. Let lHash = Hash(L), an octet string of length hLen.
95
0
        HashFunction hash;
96
0
        hash.update(label);
97
0
        auto digest = hash.digest();
98
0
        auto l_hash = digest.bytes();
99
100
        // 4. Generate an octet string PS consisting of k - mLen - 2hLen - 2 zero octets.  The length of PS may be zero.
101
0
        auto ps_size = k - m_len - (2 * h_len) - 2;
102
0
        auto ps = TRY(ByteBuffer::create_zeroed(ps_size));
103
104
        // 5. Concatenate lHash, PS, a single octet with hexadecimal value 0x01, and the message M
105
        //    to form a data block DB of length k - hLen - 1 octets as DB = lHash || PS || 0x01 || M.
106
0
        auto db = TRY(ByteBuffer::create_uninitialized(0));
107
0
        TRY(db.try_append(l_hash));
108
0
        TRY(db.try_append(ps.bytes()));
109
0
        TRY(db.try_append(0x01));
110
0
        TRY(db.try_append(message));
111
112
        // 6. Generate a random octet string seed of length hLen.
113
0
        auto seed = TRY(ByteBuffer::create_uninitialized(h_len));
114
0
        seed_function(seed);
115
116
        // 7. Let dbMask = MGF(seed, k - hLen - 1).
117
0
        auto db_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(seed, k - h_len - 1));
118
119
        // 8. Let maskedDB = DB \xor dbMask.
120
0
        auto masked_db = TRY(ByteBuffer::xor_buffers(db, db_mask));
121
122
        // 9. Let seedMask = MGF(maskedDB, hLen).
123
0
        auto seed_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(masked_db, h_len));
124
125
        // 10. Let maskedSeed = seed \xor seedMask.
126
0
        auto masked_seed = TRY(ByteBuffer::xor_buffers(seed, seed_mask));
127
128
        // 11. Concatenate a single octet with hexadecimal value 0x00, maskedSeed, and maskedDB
129
        //     to form an encoded message EM of length k octets as EM = 0x00 || maskedSeed || maskedDB.
130
0
        auto em = TRY(ByteBuffer::create_uninitialized(0));
131
0
        TRY(em.try_append(0x00));
132
0
        TRY(em.try_append(masked_seed));
133
0
        TRY(em.try_append(masked_db));
134
135
        // 12. Output EM.
136
0
        return em;
137
0
    }
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_encode<Crypto::Hash::SHA1, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int, AK::Function<void (AK::Span<unsigned char>)>)
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_encode<Crypto::Hash::SHA256, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int, AK::Function<void (AK::Span<unsigned char>)>)
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_encode<Crypto::Hash::SHA384, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int, AK::Function<void (AK::Span<unsigned char>)>)
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_encode<Crypto::Hash::SHA512, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int, AK::Function<void (AK::Span<unsigned char>)>)
138
139
    // https://datatracker.ietf.org/doc/html/rfc2437#section-9.1.1.2
140
    template<typename HashFunction, typename MaskGenerationFunction>
141
    static ErrorOr<ByteBuffer> decode(ReadonlyBytes encoded_message, ReadonlyBytes parameters)
142
    {
143
        // FIXME: 1. If the length of P is greater than the input limitation for the
144
        //           hash function (2^61-1 octets for SHA-1) then output "parameter string
145
        //           too long" and stop.
146
147
        // 2. If ||EM|| < 2hLen+1, then output "decoding error" and stop.
148
        auto h_len = HashFunction::digest_size();
149
        auto max_message_size = (2 * h_len) + 1;
150
        if (encoded_message.size() < max_message_size)
151
            return Error::from_string_view("decoding error"sv);
152
153
        // 3. Let maskedSeed be the first hLen octets of EM and let maskedDB be the remaining ||EM|| - hLen octets.
154
        auto masked_seed = encoded_message.slice(0, h_len);
155
        auto masked_db = encoded_message.slice(h_len, encoded_message.size() - h_len);
156
157
        // 4. Let seedMask = MGF(maskedDB, hLen).
158
        auto seed_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(masked_db, h_len));
159
160
        // 5. Let seed = maskedSeed \xor seedMask.
161
        auto seed = TRY(ByteBuffer::xor_buffers(masked_seed, seed_mask));
162
163
        // 6. Let dbMask = MGF(seed, ||EM|| - hLen).
164
        auto db_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(seed, encoded_message.size() - h_len));
165
166
        // 7. Let DB = maskedDB \xor dbMask.
167
        auto db = TRY(ByteBuffer::xor_buffers(masked_db, db_mask));
168
169
        // 8. Let pHash = Hash(P), an octet string of length hLen.
170
        HashFunction hash;
171
        hash.update(parameters);
172
        auto digest = hash.digest();
173
        auto p_hash = digest.bytes();
174
175
        // 9. Separate DB into an octet string pHash' consisting of the first hLen octets of DB,
176
        //    a (possibly empty) octet string PS consisting of consecutive zero octets following pHash',
177
        //    and a message M as: DB = pHash' || PS || 01 || M
178
        auto p_hash_prime = TRY(db.slice(0, h_len));
179
180
        size_t i = h_len;
181
        for (; i < db.size(); ++i) {
182
            if (db[i] == 0x01)
183
                break;
184
        }
185
186
        //    If there is no 01 octet to separate PS from M, output "decoding error" and stop.
187
        if (i == db.size())
188
            return Error::from_string_view("decoding error"sv);
189
190
        auto ps = TRY(db.slice(h_len, i - h_len));
191
        auto message = TRY(db.slice(i + 1, db.size() - i - 1));
192
193
        // 10. If pHash' does not equal pHash, output "decoding error" and stop.
194
        if (p_hash_prime.span() != p_hash)
195
            return Error::from_string_view("decoding error"sv);
196
197
        // 11. Output M.
198
        return message;
199
    }
200
201
    // https://www.rfc-editor.org/rfc/rfc3447#section-7.1.2
202
    template<typename HashFunction, typename MaskGenerationFunction>
203
    static ErrorOr<ByteBuffer> eme_decode(ReadonlyBytes encoded_message, ReadonlyBytes label, u32 rsa_modulus_n)
204
0
    {
205
0
        auto h_len = HashFunction::digest_size();
206
0
        auto k = rsa_modulus_n;
207
208
        // 1. If the label L is not provided, let L be the empty string.
209
        // Let lHash = Hash(L), an octet string of length hLen (see the note in Section 7.1.1).
210
0
        HashFunction hash;
211
0
        hash.update(label);
212
0
        auto digest = hash.digest();
213
0
        auto l_hash = digest.bytes();
214
215
        // 2. Separate the encoded message EM into
216
        //    a single octet Y,
217
        //    an octet string maskedSeed of length hLen,
218
        //    and an octet string maskedDB of length k - hLen - 1
219
        //    as EM = Y || maskedSeed || maskedDB.
220
0
        auto y = encoded_message[0];
221
0
        auto masked_seed = encoded_message.slice(1, h_len);
222
0
        auto masked_db = encoded_message.slice(h_len + 1, k - h_len - 1);
223
224
        // 3. Let seedMask = MGF(maskedDB, hLen).
225
0
        auto seed_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(masked_db, h_len));
226
227
        // 4. Let seed = maskedSeed \xor seedMask.
228
0
        auto seed = TRY(ByteBuffer::xor_buffers(masked_seed, seed_mask));
229
230
        // 5. Let dbMask = MGF(seed, k - hLen - 1).
231
0
        auto db_mask = TRY(MaskGenerationFunction::template mgf1<HashFunction>(seed, k - h_len - 1));
232
233
        // 6. Let DB = maskedDB \xor dbMask.
234
0
        auto db = TRY(ByteBuffer::xor_buffers(masked_db, db_mask));
235
236
        // 7. Separate DB into
237
        // an octet string lHash' of length hLen,
238
        // a (possibly empty) padding string PS consisting of octets withhexadecimal value 0x00,
239
        // and a message M
240
        // as DB = lHash' || PS || 0x01 || M.
241
0
        auto l_hash_prime = TRY(db.slice(0, h_len));
242
243
0
        size_t i = h_len;
244
0
        for (; i < db.size(); ++i) {
245
0
            if (db[i] == 0x01)
246
0
                break;
247
0
        }
248
249
0
        auto message = TRY(db.slice(i + 1, db.size() - i - 1));
250
251
        // NOTE: We have to make sure to do all these steps before returning an error due to timing attacks
252
0
        bool is_valid = true;
253
254
        // If there is no octet with hexadecimal value 0x01 to separate PS from M,
255
0
        if (i == db.size())
256
0
            is_valid = false;
257
258
        // if lHash does not equal lHash',
259
0
        if (l_hash_prime.span() != l_hash)
260
0
            is_valid = false;
261
262
        // if Y is nonzero, output "decryption error" and stop.
263
0
        if (y != 0x00)
264
0
            is_valid = false;
265
266
0
        if (!is_valid)
267
0
            return Error::from_string_view("decryption error"sv);
268
269
        // 8. Output the message M.
270
0
        return message;
271
0
    }
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_decode<Crypto::Hash::SHA1, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int)
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_decode<Crypto::Hash::SHA256, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int)
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_decode<Crypto::Hash::SHA384, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int)
Unexecuted instantiation: AK::ErrorOr<AK::Detail::ByteBuffer<32ul>, AK::Error> Crypto::Padding::OAEP::eme_decode<Crypto::Hash::SHA512, Crypto::Hash::MGF>(AK::Span<unsigned char const>, AK::Span<unsigned char const>, unsigned int)
272
};
273
274
}