Coverage Report

Created: 2026-11-06 11:29

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/libreoffice/vcl/source/pdf/PDFEncryptorR6.cxx
Line
Count
Source
1
/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */
2
/*
3
 * This file is part of the LibreOffice project.
4
 *
5
 * This Source Code Form is subject to the terms of the Mozilla Public
6
 * License, v. 2.0. If a copy of the MPL was not distributed with this
7
 * file, You can obtain one at http://mozilla.org/MPL/2.0/.
8
 */
9
10
#include <pdf/PDFEncryptorR6.hxx>
11
#include <pdf/EncryptionHashTransporter.hxx>
12
#include <pdf/pdfwriter_impl.hxx>
13
#include <comphelper/crypto/Crypto.hxx>
14
#include <comphelper/hash.hxx>
15
#include <rtl/random.h>
16
17
using namespace css;
18
19
namespace vcl::pdf
20
{
21
namespace
22
{
23
constexpr size_t IV_SIZE = 16;
24
constexpr size_t BLOCK_SIZE = 16;
25
constexpr size_t KEY_SIZE = 32;
26
constexpr size_t SALT_SIZE = 8;
27
28
/** Calculates modulo 3 of the 128-bit integer, using the first 16 bytes of the vector */
29
sal_Int32 calculateModulo3(std::vector<sal_uInt8> const& rInput)
30
0
{
31
0
    sal_Int32 nSum = 0;
32
0
    for (size_t i = 0; i < 16; ++i)
33
0
        nSum += rInput[i];
34
0
    return nSum % 3;
35
0
}
36
37
void generateBytes(std::vector<sal_uInt8>& rBytes, size_t nSize)
38
0
{
39
0
    rBytes.resize(nSize);
40
41
0
    if (rBytes.empty())
42
0
        return;
43
44
0
    (void)rtl_random_getBytes(nullptr, rBytes.data(), rBytes.size());
45
0
}
46
47
} // end anonymous
48
49
std::vector<sal_uInt8> generateKey()
50
0
{
51
0
    std::vector<sal_uInt8> aKey;
52
0
    generateBytes(aKey, KEY_SIZE);
53
0
    return aKey;
54
0
}
55
56
bool validateUserPassword(const sal_uInt8* pPass, size_t nLength, std::vector<sal_uInt8>& U)
57
0
{
58
0
    std::vector<sal_uInt8> aHash(U.begin(), U.begin() + KEY_SIZE);
59
0
    std::vector<sal_uInt8> aValidationSalt(U.begin() + KEY_SIZE, U.begin() + KEY_SIZE + SALT_SIZE);
60
0
    std::vector<sal_uInt8> aCalculatedHash
61
0
        = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt);
62
0
    return aHash == aCalculatedHash;
63
0
}
64
65
bool validateOwnerPassword(const sal_uInt8* pPass, size_t nLength, const std::vector<sal_uInt8>& U,
66
                           std::vector<sal_uInt8>& O)
67
0
{
68
0
    std::vector<sal_uInt8> aHash(O.begin(), O.begin() + KEY_SIZE);
69
0
    std::vector<sal_uInt8> aValidationSalt(O.begin() + KEY_SIZE, O.begin() + KEY_SIZE + SALT_SIZE);
70
0
    std::vector<sal_uInt8> aCalculatedHash
71
0
        = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt, U);
72
0
    return aHash == aCalculatedHash;
73
0
}
74
75
/** Algorithm 8 */
76
void generateUandUE(const sal_uInt8* pPass, size_t nLength,
77
                    std::vector<sal_uInt8>& rFileEncryptionKey, std::vector<sal_uInt8>& U,
78
                    std::vector<sal_uInt8>& UE)
79
0
{
80
0
    std::vector<sal_uInt8> aValidationSalt;
81
0
    generateBytes(aValidationSalt, SALT_SIZE);
82
0
    std::vector<sal_uInt8> aKeySalt;
83
0
    generateBytes(aKeySalt, SALT_SIZE);
84
85
0
    U = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt);
86
0
    U.insert(U.end(), aValidationSalt.begin(), aValidationSalt.end());
87
0
    U.insert(U.end(), aKeySalt.begin(), aKeySalt.end());
88
89
0
    std::vector<sal_uInt8> aKeyHash = vcl::pdf::computeHashR6(pPass, nLength, aKeySalt);
90
0
    std::vector<sal_uInt8> iv(IV_SIZE, 0); // zero IV
91
0
    UE = std::vector<sal_uInt8>(rFileEncryptionKey.size(), 0);
92
0
    comphelper::Encrypt aEncrypt(aKeyHash, iv, comphelper::CryptoType::AES_256_CBC);
93
0
    aEncrypt.update(UE, rFileEncryptionKey);
94
0
}
95
96
/** Algorithm 9 */
97
void generateOandOE(const sal_uInt8* pPass, size_t nLength,
98
                    std::vector<sal_uInt8>& rFileEncryptionKey, const std::vector<sal_uInt8>& U,
99
                    std::vector<sal_uInt8>& O, std::vector<sal_uInt8>& OE)
100
0
{
101
0
    std::vector<sal_uInt8> aValidationSalt;
102
0
    generateBytes(aValidationSalt, SALT_SIZE);
103
0
    std::vector<sal_uInt8> aKeySalt;
104
0
    generateBytes(aKeySalt, SALT_SIZE);
105
106
0
    O = vcl::pdf::computeHashR6(pPass, nLength, aValidationSalt, U);
107
0
    O.insert(O.end(), aValidationSalt.begin(), aValidationSalt.end());
108
0
    O.insert(O.end(), aKeySalt.begin(), aKeySalt.end());
109
110
0
    std::vector<sal_uInt8> aKeyHash = vcl::pdf::computeHashR6(pPass, nLength, aKeySalt, U);
111
0
    std::vector<sal_uInt8> iv(IV_SIZE, 0); // zero IV
112
0
    OE = std::vector<sal_uInt8>(rFileEncryptionKey.size(), 0);
113
0
    comphelper::Encrypt aEncrypt(aKeyHash, iv, comphelper::CryptoType::AES_256_CBC);
114
0
    aEncrypt.update(OE, rFileEncryptionKey);
115
0
}
116
117
/** Algorithm 8 step b) */
118
std::vector<sal_uInt8> decryptKey(const sal_uInt8* pPass, size_t nLength, std::vector<sal_uInt8>& U,
119
                                  std::vector<sal_uInt8>& UE)
120
0
{
121
0
    std::vector<sal_uInt8> aKeySalt(U.begin() + KEY_SIZE + SALT_SIZE,
122
0
                                    U.begin() + KEY_SIZE + SALT_SIZE + SALT_SIZE);
123
124
0
    auto aKeyHash = vcl::pdf::computeHashR6(pPass, nLength, aKeySalt);
125
126
0
    std::vector<sal_uInt8> aEncryptedKey(UE.begin(), UE.begin() + KEY_SIZE);
127
0
    std::vector<sal_uInt8> iv(IV_SIZE, 0);
128
129
0
    comphelper::Decrypt aDecryptCBC(aKeyHash, iv, comphelper::CryptoType::AES_256_CBC);
130
0
    std::vector<sal_uInt8> aFileEncryptionKey(aEncryptedKey.size());
131
0
    sal_uInt32 nDecrypted = aDecryptCBC.update(aFileEncryptionKey, aEncryptedKey);
132
0
    if (nDecrypted == 0)
133
0
        return std::vector<sal_uInt8>();
134
0
    return aFileEncryptionKey;
135
0
}
136
137
/** Algorithm 13: Validating the permissions */
138
std::vector<sal_uInt8> decryptPerms(std::vector<sal_uInt8>& rPermsEncrypted,
139
                                    std::vector<sal_uInt8>& rFileEncryptionKey)
140
0
{
141
0
    std::vector<sal_uInt8> aPermsDecrypted(rPermsEncrypted.size());
142
0
    std::vector<sal_uInt8> iv(IV_SIZE, 0);
143
0
    comphelper::Decrypt aDecryptor(rFileEncryptionKey, iv, comphelper::CryptoType::AES_256_ECB);
144
0
    aDecryptor.update(aPermsDecrypted, rPermsEncrypted);
145
0
    if (aPermsDecrypted[9] == 'a' && aPermsDecrypted[10] == 'd' && aPermsDecrypted[11] == 'b')
146
0
    {
147
0
        return aPermsDecrypted;
148
0
    }
149
0
    SAL_INFO("vcl.pdfwriter", "decryptPerms failed");
150
0
    return {};
151
0
}
152
153
/** Algorithm 10 step f) */
154
std::vector<sal_uInt8> encryptPerms(std::vector<sal_uInt8>& rPerms,
155
                                    std::vector<sal_uInt8>& rFileEncryptionKey)
156
0
{
157
0
    std::vector<sal_uInt8> aPermsEncrypted(rPerms.size());
158
0
    std::vector<sal_uInt8> iv(IV_SIZE, 0);
159
0
    comphelper::Encrypt aEncryptor(rFileEncryptionKey, iv, comphelper::CryptoType::AES_256_ECB);
160
0
    aEncryptor.update(aPermsEncrypted, rPerms);
161
0
    return aPermsEncrypted;
162
0
}
163
164
/** Algorithm 10 steps a) - e) */
165
std::vector<sal_uInt8> createPerms(sal_Int32 nAccessPermissions, bool bEncryptMetadata)
166
0
{
167
0
    std::vector<sal_uInt8> aPermsCreated;
168
0
    generateBytes(aPermsCreated, 16); // fill with random data - mainly for last 4 bytes
169
0
    aPermsCreated[0] = sal_uInt8(nAccessPermissions);
170
0
    aPermsCreated[1] = sal_uInt8(nAccessPermissions >> 8);
171
0
    aPermsCreated[2] = sal_uInt8(nAccessPermissions >> 16);
172
0
    aPermsCreated[3] = sal_uInt8(nAccessPermissions >> 24);
173
0
    aPermsCreated[4] = sal_uInt8(0xff);
174
0
    aPermsCreated[5] = sal_uInt8(0xff);
175
0
    aPermsCreated[6] = sal_uInt8(0xff);
176
0
    aPermsCreated[7] = sal_uInt8(0xff);
177
0
    aPermsCreated[8] = bEncryptMetadata ? 'T' : 'F'; // Encrypt metadata
178
0
    aPermsCreated[9] = 'a';
179
0
    aPermsCreated[10] = 'd';
180
0
    aPermsCreated[11] = 'b';
181
0
    return aPermsCreated;
182
0
}
183
184
/** Algorithm 2.B: Computing a hash (revision 6 and later) */
185
std::vector<sal_uInt8> computeHashR6(const sal_uInt8* pPassword, size_t nPasswordLength,
186
                                     std::vector<sal_uInt8> const& rValidationSalt,
187
                                     std::vector<sal_uInt8> const& rUserKey)
188
0
{
189
    // Round 0
190
0
    comphelper::Hash aHash(comphelper::HashType::SHA256);
191
0
    aHash.update(pPassword, nPasswordLength);
192
0
    aHash.update(rValidationSalt);
193
0
    if (!rUserKey.empty()) // if calculating owner key
194
0
        aHash.update(rUserKey);
195
196
0
    std::vector<sal_uInt8> K = aHash.finalize();
197
198
0
    std::vector<sal_uInt8> E;
199
200
    // Counts the repetitions of steps a) to d) done so far.
201
    // K is the input to the first repetition and is not counted.
202
0
    sal_Int32 nRound = 0;
203
0
    do
204
0
    {
205
        // Step a)
206
0
        std::vector<sal_uInt8> K1;
207
0
        for (sal_Int32 nRepetition = 0; nRepetition < 64; ++nRepetition)
208
0
        {
209
0
            K1.insert(K1.end(), pPassword, pPassword + nPasswordLength);
210
0
            K1.insert(K1.end(), K.begin(), K.end());
211
0
            if (!rUserKey.empty()) // if calculating owner key
212
0
                K1.insert(K1.end(), rUserKey.begin(), rUserKey.end());
213
0
        }
214
215
        // Step b)
216
0
        std::vector<sal_uInt8> aKey(K.begin(), K.begin() + 16);
217
0
        std::vector<sal_uInt8> aInitVector(K.begin() + 16, K.end());
218
219
0
        E = std::vector<sal_uInt8>(K1.size(), 0);
220
221
0
        comphelper::Encrypt aEncrypt(aKey, aInitVector, comphelper::CryptoType::AES_128_CBC);
222
0
        aEncrypt.update(E, K1);
223
224
        // Step c)
225
0
        sal_Int32 nModulo3Result = calculateModulo3(E);
226
227
        // Step d)
228
0
        comphelper::HashType eType;
229
0
        switch (nModulo3Result)
230
0
        {
231
0
            case 0:
232
0
                eType = comphelper::HashType::SHA256;
233
0
                break;
234
0
            case 1:
235
0
                eType = comphelper::HashType::SHA384;
236
0
                break;
237
0
            default:
238
0
                eType = comphelper::HashType::SHA512;
239
0
                break;
240
0
        }
241
0
        K = comphelper::Hash::calculateHash(E.data(), E.size(), eType);
242
243
0
        nRound++;
244
0
    }
245
    // Step e) and f)
246
    // We stop iteration if we do at least 64 rounds and (the last element of E <= round number - 32)
247
0
    while (nRound < 64 || E.back() > (nRound - 32));
248
249
    // Output - first 32 bytes
250
0
    return std::vector<sal_uInt8>(K.begin(), K.begin() + 32);
251
0
}
252
253
size_t addPaddingToVector(std::vector<sal_uInt8>& rVector, size_t nBlockSize)
254
0
{
255
    // RFC 8018 PKCS #5: Always add padding, between 1 and nBlockSize bytes.
256
0
    size_t nPaddingSize = nBlockSize - (rVector.size() % nBlockSize);
257
0
    rVector.resize(rVector.size() + nPaddingSize, sal_uInt8(nPaddingSize));
258
0
    return rVector.size();
259
0
}
260
261
class VCL_DLLPUBLIC EncryptionContext
262
{
263
private:
264
    std::vector<sal_uInt8> maKey;
265
266
public:
267
    EncryptionContext(std::vector<sal_uInt8> const& rKey)
268
0
        : maKey(rKey)
269
0
    {
270
0
    }
271
272
    /** Algorithm 1.A: Encryption of data using the AES algorithms */
273
    void encrypt(const void* pInput, sal_uInt64 nInputSize, std::vector<sal_uInt8>& rOutput,
274
                 std::vector<sal_uInt8>& rIV)
275
0
    {
276
0
        comphelper::Encrypt aEncrypt(maKey, rIV, comphelper::CryptoType::AES_256_CBC);
277
0
        const sal_uInt8* pInputBytes = static_cast<const sal_uInt8*>(pInput);
278
0
        std::vector<sal_uInt8> aInput(pInputBytes, pInputBytes + nInputSize);
279
0
        size_t nPaddedSize = addPaddingToVector(aInput, BLOCK_SIZE);
280
0
        std::vector<sal_uInt8> aOutput(nPaddedSize);
281
0
        aEncrypt.update(aOutput, aInput);
282
0
        rOutput.resize(nPaddedSize + IV_SIZE);
283
0
        std::copy(rIV.begin(), rIV.end(), rOutput.begin());
284
0
        std::copy(aOutput.begin(), aOutput.end(), rOutput.begin() + IV_SIZE);
285
0
    }
286
};
287
288
0
PDFEncryptorR6::PDFEncryptorR6() = default;
289
0
PDFEncryptorR6::~PDFEncryptorR6() = default;
290
291
std::vector<sal_uInt8> PDFEncryptorR6::getEncryptedAccessPermissions(std::vector<sal_uInt8>& rKey)
292
0
{
293
0
    std::vector<sal_uInt8> aPerms = createPerms(m_nAccessPermissions, /*bEncryptMetadata=*/true);
294
0
    return encryptPerms(aPerms, rKey);
295
0
}
296
297
void PDFEncryptorR6::initEncryption(EncryptionHashTransporter& rEncryptionHashTransporter,
298
                                    const OUString& rOwnerPassword, const OUString& rUserPassword)
299
0
{
300
0
    if (rUserPassword.isEmpty())
301
0
        return;
302
303
0
    std::vector<sal_uInt8> aEncryptionKey = vcl::pdf::generateKey();
304
0
    rEncryptionHashTransporter.setEncryptionKey(aEncryptionKey);
305
306
0
    std::vector<sal_uInt8> U;
307
0
    std::vector<sal_uInt8> UE;
308
309
0
    OString pUserPasswordUtf8 = OUStringToOString(rUserPassword, RTL_TEXTENCODING_UTF8);
310
0
    vcl::pdf::generateUandUE(reinterpret_cast<const sal_uInt8*>(pUserPasswordUtf8.getStr()),
311
0
                             pUserPasswordUtf8.getLength(), aEncryptionKey, U, UE);
312
313
0
    rEncryptionHashTransporter.setU(U);
314
0
    rEncryptionHashTransporter.setUE(UE);
315
316
0
    OUString aOwnerPasswordToUse = rOwnerPassword.isEmpty() ? rUserPassword : rOwnerPassword;
317
318
0
    std::vector<sal_uInt8> O;
319
0
    std::vector<sal_uInt8> OE;
320
321
0
    OString pOwnerPasswordUtf8 = OUStringToOString(aOwnerPasswordToUse, RTL_TEXTENCODING_UTF8);
322
0
    vcl::pdf::generateOandOE(reinterpret_cast<const sal_uInt8*>(pOwnerPasswordUtf8.getStr()),
323
0
                             pOwnerPasswordUtf8.getLength(), aEncryptionKey, U, O, OE);
324
325
0
    rEncryptionHashTransporter.setO(O);
326
0
    rEncryptionHashTransporter.setOE(OE);
327
0
}
328
329
bool PDFEncryptorR6::prepareEncryption(
330
    const uno::Reference<beans::XMaterialHolder>& xEncryptionMaterialHolder,
331
    vcl::PDFEncryptionProperties& rProperties)
332
0
{
333
0
    auto* pTransporter
334
0
        = EncryptionHashTransporter::getEncHashTransporter(xEncryptionMaterialHolder);
335
336
0
    if (!pTransporter)
337
0
    {
338
0
        rProperties.clear();
339
0
        return false;
340
0
    }
341
342
0
    rProperties.UValue = pTransporter->getU();
343
0
    rProperties.UE = pTransporter->getUE();
344
0
    rProperties.OValue = pTransporter->getO();
345
0
    rProperties.OE = pTransporter->getOE();
346
0
    rProperties.EncryptionKey = pTransporter->getEncryptionKey();
347
348
0
    return true;
349
0
}
350
351
void PDFEncryptorR6::setupKeysAndCheck(vcl::PDFEncryptionProperties& rProperties)
352
0
{
353
0
    m_nAccessPermissions = rProperties.getAccessPermissions();
354
0
}
355
356
sal_uInt64 PDFEncryptorR6::calculateSizeIncludingHeader(sal_uInt64 nSize)
357
0
{
358
    // IV goes before the data, and padding adds 1 to BLOCK_SIZE bytes.
359
0
    return IV_SIZE + nSize + (BLOCK_SIZE - nSize % BLOCK_SIZE);
360
0
}
361
362
void PDFEncryptorR6::setupEncryption(std::vector<sal_uInt8>& rEncryptionKey, sal_Int32 /*nObject*/)
363
0
{
364
0
    m_pEncryptionContext = std::make_unique<EncryptionContext>(rEncryptionKey);
365
0
}
366
367
void PDFEncryptorR6::encrypt(const void* pInput, sal_uInt64 nInputSize,
368
                             std::vector<sal_uInt8>& rOutput, sal_uInt64 /*nOutputSize*/)
369
0
{
370
0
    std::vector<sal_uInt8> aIV;
371
0
    generateBytes(aIV, IV_SIZE);
372
0
    m_pEncryptionContext->encrypt(pInput, nInputSize, rOutput, aIV);
373
0
}
374
375
void PDFEncryptorR6::encryptWithIV(const void* pInput, sal_uInt64 nInputSize,
376
                                   std::vector<sal_uInt8>& rOutput, std::vector<sal_uInt8>& rIV)
377
0
{
378
0
    m_pEncryptionContext->encrypt(pInput, nInputSize, rOutput, rIV);
379
0
}
380
381
} // end vcl::pdf
382
383
/* vim:set shiftwidth=4 softtabstop=4 expandtab: */