Coverage Report

Created: 2022-10-31 07:00

/src/ghostpdl/psi/zpdf_r6.c
Line
Count
Source (jump to first uncovered line)
1
/* Copyright (C) 2001-2021 Artifex Software, Inc.
2
   All Rights Reserved.
3
4
   This software is provided AS-IS with no warranty, either express or
5
   implied.
6
7
   This software is distributed under license and may not be copied,
8
   modified or distributed except as expressly authorized under the terms
9
   of the license contained in the file LICENSE in this distribution.
10
11
   Refer to licensing information at http://www.artifex.com or contact
12
   Artifex Software, Inc.,  1305 Grant Avenue - Suite 200, Novato,
13
   CA 94945, U.S.A., +1(415)492-9861, for further information.
14
*/
15
16
#include "memory_.h"
17
#include "ghost.h"
18
#include "oper.h"
19
#include "dstack.h" /* for systemdict */
20
#include "estack.h"
21
#include "ialloc.h"
22
#include "iutil.h"
23
#include "idict.h"
24
#include "iname.h"
25
#include "string_.h"  /* memcmp() */
26
#include "store.h"
27
#include "aes.h"
28
#include "sha2.h"
29
30
/* Implementation of the PDF security handler revision6 (PDF 1.7 ExtensionLevel 8 algorithm)
31
 *
32
 * Adobe/ISO has not yet released the details, so the algorithm reference is:
33
 * http://esec-lab.sogeti.com/post/The-undocumented-password-validation-algorithm-of-Adobe-Reader-X
34
 *
35
 * The code below is the same as (and copied from) the MuPDF implementation.
36
 */
37
38
static void
39
pdf_compute_hardened_hash_r6(unsigned char *password, int pwlen, unsigned char salt[16], unsigned char *ownerkey, unsigned char hash[32])
40
0
{
41
0
  unsigned char data[(128 + 64 + 48) * 64];
42
0
  unsigned char block[64];
43
0
  int block_size = 32;
44
0
  int data_len = 0;
45
0
  int i, j, sum;
46
47
0
    SHA256_CTX sha256;
48
0
    SHA384_CTX sha384;
49
0
    SHA512_CTX sha512;
50
0
    aes_context aes;
51
52
0
    pSHA256_Init(&sha256);
53
0
    pSHA256_Update(&sha256, password, pwlen);
54
0
    pSHA256_Update(&sha256, salt, 8);
55
0
    if (ownerkey)
56
0
        pSHA256_Update(&sha256, ownerkey, 48);
57
0
    pSHA256_Final((uint8_t *)block, &sha256);
58
59
0
  for (i = 0; i < 64 || i < data[data_len * 64 - 1] + 32; i++)
60
0
  {
61
    /* Step 2: repeat password and data block 64 times */
62
0
    memcpy(data, password, pwlen);
63
0
    memcpy(data + pwlen, block, block_size);
64
0
    if (ownerkey)
65
0
      memcpy(data + pwlen + block_size, ownerkey, 48);
66
0
    data_len = pwlen + block_size + (ownerkey ? 48 : 0);
67
0
    for (j = 1; j < 64; j++)
68
0
      memcpy(data + j * data_len, data, data_len);
69
70
    /* Step 3: encrypt data using data block as key and iv */
71
0
    aes_setkey_enc(&aes, block, 128);
72
0
    aes_crypt_cbc(&aes, AES_ENCRYPT, data_len * 64, block + 16, data, data);
73
74
    /* Step 4: determine SHA-2 hash size for this round */
75
0
    for (j = 0, sum = 0; j < 16; j++)
76
0
      sum += data[j];
77
78
    /* Step 5: calculate data block for next round */
79
0
    block_size = 32 + (sum % 3) * 16;
80
0
    switch (block_size)
81
0
    {
82
0
        case 32:
83
0
            pSHA256_Init(&sha256);
84
0
            pSHA256_Update(&sha256, data, data_len * 64);
85
0
            pSHA256_Final((uint8_t *)block, &sha256);
86
0
            break;
87
0
        case 48:
88
0
            pSHA384_Init(&sha384);
89
0
            pSHA384_Update(&sha384, data, data_len * 64);
90
0
            pSHA384_Final((uint8_t *)block, &sha384);
91
0
            break;
92
0
        case 64:
93
0
            pSHA512_Init(&sha512);
94
0
            pSHA512_Update(&sha512, data, data_len * 64);
95
0
            pSHA512_Final((uint8_t *)block, &sha512);
96
0
            break;
97
0
    }
98
0
  }
99
100
0
  memset(data, 0, sizeof(data));
101
0
  memcpy(hash, block, 32);
102
0
}
103
104
static void
105
pdf_compute_encryption_key_r6(unsigned char *password, int pwlen, unsigned char *O, unsigned char *OE, unsigned char *U, unsigned char *UE, int ownerkey, unsigned char *validationkey, unsigned char *output)
106
0
{
107
0
  unsigned char hash[32];
108
0
  unsigned char iv[16];
109
0
    aes_context aes;
110
111
0
  if (pwlen > 127)
112
0
    pwlen = 127;
113
114
0
  pdf_compute_hardened_hash_r6(password, pwlen,
115
0
    (ownerkey ? O : U) + 32,
116
0
    ownerkey ? U : NULL, validationkey);
117
0
  pdf_compute_hardened_hash_r6(password, pwlen,
118
0
        (ownerkey ? O : U) + 40,
119
0
        (ownerkey ? U : NULL), hash);
120
121
0
  memset(iv, 0, sizeof(iv));
122
0
    aes_setkey_dec(&aes, hash, 256);
123
0
  aes_crypt_cbc(&aes, AES_DECRYPT, 32, iv,
124
0
    ownerkey ? OE : UE, output);
125
0
}
126
127
/* (password) <encryption dict> check_r6_password (key) true|false */
128
static int
129
zcheck_r6_password(i_ctx_t * i_ctx_p)
130
0
{
131
0
    os_ptr  op = osp;
132
0
    ref *CryptDict, *Oref, *OEref, *Uref, *UEref, *Pref;
133
0
    int code, PWlen;
134
0
  unsigned char validation[32];
135
0
  unsigned char output[32];
136
0
    ref stref;
137
0
    byte *body;
138
139
0
    check_op(2);
140
141
0
    CryptDict = op--;
142
0
    Pref = op;
143
0
    if (!r_has_type(CryptDict, t_dictionary))
144
0
        return_error(gs_error_typecheck);
145
0
    if (!r_has_type(Pref, t_string))
146
0
        return_error(gs_error_typecheck);
147
148
0
    code = dict_find_string(CryptDict, "O", &Oref);
149
0
    if (code < 0)
150
0
        return code;
151
0
    if (code == 0)
152
0
        return_error(gs_error_undefined);
153
0
    if (!r_has_type(Oref, t_string))
154
0
      return_error(gs_error_typecheck);
155
0
    if (r_size(Oref) < 48)
156
0
        return_error(gs_error_invalidaccess);
157
158
0
    code = dict_find_string(CryptDict, "OE", &OEref);
159
0
    if (code < 0)
160
0
        return code;
161
0
    if (code == 0)
162
0
        return_error(gs_error_undefined);
163
0
    if (!r_has_type(OEref, t_string))
164
0
      return_error(gs_error_typecheck);
165
0
    if (r_size(OEref) < 32)
166
0
        return_error(gs_error_invalidaccess);
167
168
0
    code = dict_find_string(CryptDict, "U", &Uref);
169
0
    if (code < 0)
170
0
        return code;
171
0
    if (code == 0)
172
0
        return_error(gs_error_undefined);
173
0
    if (!r_has_type(Uref, t_string))
174
0
      return_error(gs_error_typecheck);
175
0
    if (r_size(Uref) < 48)
176
0
        return_error(gs_error_invalidaccess);
177
178
0
    code = dict_find_string(CryptDict, "UE", &UEref);
179
0
    if (code < 0)
180
0
        return code;
181
0
    if (code == 0)
182
0
        return_error(gs_error_undefined);
183
0
    if (!r_has_type(UEref, t_string))
184
0
      return_error(gs_error_typecheck);
185
0
    if (r_size(UEref) < 32)
186
0
        return_error(gs_error_invalidaccess);
187
188
0
    ref_stack_pop(&o_stack, 2);
189
0
    op = osp;
190
191
0
    PWlen = r_size(Pref);
192
193
    /* First, try the password as the user password */
194
0
    pdf_compute_encryption_key_r6((unsigned char *)Pref->value.const_bytes, PWlen, (unsigned char *)Oref->value.const_bytes,
195
0
        (unsigned char *)OEref->value.const_bytes, (unsigned char *)Uref->value.const_bytes, (unsigned char *)UEref->value.const_bytes, 0, validation, output);
196
197
0
    if (memcmp(validation, Uref->value.const_bytes, 32) != 0){
198
        /* It wasn't the user password, maybe its the owner password */
199
0
        pdf_compute_encryption_key_r6((unsigned char *)Pref->value.const_bytes, PWlen, (unsigned char *)Oref->value.const_bytes,
200
0
            (unsigned char *)OEref->value.const_bytes, (unsigned char *)Uref->value.const_bytes, (unsigned char *)UEref->value.const_bytes, 1, validation, output);
201
202
0
        if (memcmp(validation, Oref->value.const_bytes, 32) != 0){
203
            /* Doesn't seem to be a valid password.... */
204
0
            push(1);
205
0
            make_bool(op, 0);
206
0
            return 0;
207
0
        }
208
0
    }
209
210
0
    body = ialloc_string(32, "r6 encryption key");
211
0
    if (body == 0)
212
0
        return_error(gs_error_VMerror);
213
0
    push(1);
214
0
    memcpy(body, output, 32);
215
0
    make_string(&stref, a_all | icurrent_space, 32, body);
216
0
    ref_assign(op, &stref);
217
0
    push(1);
218
0
    make_bool(op, 1);
219
220
0
    return 0;
221
0
}
222
223
const op_def zpdf_r6_op_defs[] =
224
{
225
    { "2check_r6_password", zcheck_r6_password },
226
    op_def_end(0)
227
};