Coverage Report

Created: 2026-10-17 06:58

next uncovered line (L), next uncovered region (R), next uncovered branch (B)
/src/qpdf/libqpdf/QPDF_linearization.cc
Line
Count
Source
1
// See doc/linearization.
2
3
#include <qpdf/QPDF_private.hh>
4
5
#include <qpdf/BitStream.hh>
6
#include <qpdf/BitWriter.hh>
7
#include <qpdf/InputSource_private.hh>
8
#include <qpdf/Pipeline_private.hh>
9
#include <qpdf/Pl_Buffer.hh>
10
#include <qpdf/Pl_Flate.hh>
11
#include <qpdf/Pl_String.hh>
12
#include <qpdf/QPDFExc.hh>
13
#include <qpdf/QPDFObjectHandle_private.hh>
14
#include <qpdf/QPDFWriter_private.hh>
15
#include <qpdf/QTC.hh>
16
#include <qpdf/QUtil.hh>
17
#include <qpdf/Util.hh>
18
19
#include <algorithm>
20
#include <cmath>
21
#include <cstring>
22
#include <utility>
23
24
using namespace qpdf;
25
using namespace std::literals;
26
27
using Lin = QPDF::Doc::Linearization;
28
29
template <class T, class int_type>
30
static void
31
load_vector_int(
32
    BitStream& bit_stream, int nitems, std::vector<T>& vec, int bits_wanted, int_type T::* field)
33
0
{
34
0
    bool append = vec.empty();
35
    // nitems times, read bits_wanted from the given bit stream, storing results in the ith vector
36
    // entry.
37
38
0
    for (size_t i = 0; i < QIntC::to_size(nitems); ++i) {
39
0
        if (append) {
40
0
            vec.push_back(T());
41
0
        }
42
0
        vec.at(i).*field = bit_stream.getBitsInt(QIntC::to_size(bits_wanted));
43
0
    }
44
0
    util::assertion(
45
0
        std::cmp_equal(vec.size(), nitems), "vector has wrong size in load_vector_int" //
46
0
    );
47
    // The PDF spec says that each hint table starts at a byte boundary.  Each "row" actually must
48
    // start on a byte boundary.
49
0
    bit_stream.skipToNextByte();
50
0
}
Unexecuted instantiation: QPDF_linearization.cc:void load_vector_int<QPDF::Doc::Linearization::HPageOffsetEntry, int>(BitStream&, int, std::__1::vector<QPDF::Doc::Linearization::HPageOffsetEntry, std::__1::allocator<QPDF::Doc::Linearization::HPageOffsetEntry> >&, int, int QPDF::Doc::Linearization::HPageOffsetEntry::*)
Unexecuted instantiation: QPDF_linearization.cc:void load_vector_int<QPDF::Doc::Linearization::HPageOffsetEntry, long long>(BitStream&, int, std::__1::vector<QPDF::Doc::Linearization::HPageOffsetEntry, std::__1::allocator<QPDF::Doc::Linearization::HPageOffsetEntry> >&, int, long long QPDF::Doc::Linearization::HPageOffsetEntry::*)
Unexecuted instantiation: QPDF_linearization.cc:void load_vector_int<QPDF::Doc::Linearization::HSharedObjectEntry, int>(BitStream&, int, std::__1::vector<QPDF::Doc::Linearization::HSharedObjectEntry, std::__1::allocator<QPDF::Doc::Linearization::HSharedObjectEntry> >&, int, int QPDF::Doc::Linearization::HSharedObjectEntry::*)
51
52
template <class T>
53
static void
54
load_vector_vector(
55
    BitStream& bit_stream,
56
    int nitems1,
57
    std::vector<T>& vec1,
58
    int T::* nitems2,
59
    int bits_wanted,
60
    std::vector<int> T::* vec2)
61
0
{
62
    // nitems1 times, read nitems2 (from the ith element of vec1) items into the vec2 vector field
63
    // of the ith item of vec1.
64
0
    for (size_t i1 = 0; i1 < QIntC::to_size(nitems1); ++i1) {
65
0
        for (int i2 = 0; i2 < vec1.at(i1).*nitems2; ++i2) {
66
0
            (vec1.at(i1).*vec2).push_back(bit_stream.getBitsInt(QIntC::to_size(bits_wanted)));
67
0
        }
68
0
    }
69
0
    bit_stream.skipToNextByte();
70
0
}
71
72
template <class T>
73
static void
74
append_vec(std::vector<T>& v1, std::vector<T>&& v2)
75
0
{
76
0
    v1.insert(v1.end(), std::make_move_iterator(v2.begin()), std::make_move_iterator(v2.end()));
77
0
}
78
79
class Lin::ObjUser
80
{
81
  public:
82
    enum user_e { ou_page = 1, ou_thumb, ou_trailer_key, ou_root_key, ou_root };
83
84
    ObjUser() = delete;
85
86
    // type must be ou_root
87
    ObjUser(user_e type) :
88
        // ou_type(type),
89
0
        category(c_root)
90
0
    {
91
0
        qpdf_expect(type == ou_root);
92
0
    }
93
94
    // type must be one of ou_page or ou_thumb
95
    ObjUser(user_e type, size_t pageno) :
96
        category(
97
0
            type == ou_page ? (pageno == 0 ? c_first_page_private : c_other_page_private)
98
0
                            : c_thumbnail_private),
99
0
        pageno(pageno)
100
0
    {
101
0
        qpdf_expect(type == ou_page || type == ou_thumb);
102
0
    }
103
104
    // type must be one of ou_trailer_key or ou_root_key
105
    ObjUser(user_e type, std::string const& key) :
106
0
        category(category_from_key(type, key))
107
0
    {
108
0
    }
109
110
    bool
111
    page() const
112
0
    {
113
0
        return category == c_first_page_private || category == c_other_page_private;
114
0
    }
115
116
    /// @brief Return the linearization category this object would have if it were only referenced
117
    /// in one place.
118
    ObjCategory
119
    obj_category(bool top) const
120
0
    {
121
0
        if (page() || category == c_thumbnail_private) {
122
0
            return {category, util::to<int>(pageno)};
123
0
        }
124
0
        if (top && category == c_outlines) {
125
0
            return {c_outlines_obj, -1};
126
0
        }
127
0
        return {category, -1};
128
0
    }
129
130
    const obj_category_e category;
131
    const size_t pageno{0}; // if ou_page or ou_thumb
132
133
  private:
134
    static obj_category_e
135
    category_from_key(user_e type, std::string const& key)
136
0
    {
137
0
        qpdf_expect(type == ou_trailer_key || type == ou_root_key);
138
0
        if (type == ou_root_key) {
139
0
            if (key == "/ViewerPreferences" || key == "/PageMode" || key == "/Threads" ||
140
0
                key == "/OpenAction" || key == "/AcroForm") {
141
0
                return c_open_document;
142
0
            }
143
0
            if (key == "/Outlines") {
144
0
                return c_outlines;
145
0
            }
146
0
            if (key == "/Pages") {
147
0
                return c_pages_obj;
148
0
            }
149
0
            return c_other;
150
0
        }
151
0
        if (key == "/Encrypt") {
152
0
            return c_open_document;
153
0
        }
154
0
        return c_other;
155
0
    }
156
};
157
158
struct Lin::UpdateObjectMapsFrame
159
{
160
    UpdateObjectMapsFrame(ObjUser const& ou, QPDFObjectHandle oh, bool top) :
161
0
        ou(ou),
162
0
        oh(oh),
163
0
        top(top)
164
0
    {
165
0
    }
166
167
    ObjUser const& ou;
168
    QPDFObjectHandle oh;
169
    bool top;
170
};
171
172
void
173
QPDF::optimize(
174
    std::map<int, int> const& object_stream_data,
175
    bool allow_changes,
176
    std::function<int(QPDFObjectHandle&)> skip_stream_parameters)
177
0
{
178
0
    m->lin.prepare(object_stream_data, allow_changes, skip_stream_parameters);
179
0
}
180
181
template <typename T>
182
void
183
Lin::prepare(
184
    T const& object_stream_data,
185
    bool allow_changes,
186
    std::function<int(QPDFObjectHandle&)> skip_stream_parameters)
187
0
{
188
0
    if (!obj_categories_.empty()) {
189
        // already prepared
190
0
        return;
191
0
    }
192
193
    // includes some extra for objects that have incorrectly been excluded from xref table, such as
194
    // xref streams, lin dictionaries, int streams etc.
195
0
    max_obj_ = qpdf.getObjectCount() + 10;
196
197
    // The PDF specification indicates that /Outlines is supposed to be an indirect reference. Force
198
    // it to be so if it exists and is direct.  (This has been seen in the wild.)
199
0
    QPDFObjectHandle root = qpdf.getRoot();
200
0
    if (root.getKey("/Outlines").isDictionary()) {
201
0
        QPDFObjectHandle outlines = root.getKey("/Outlines");
202
0
        if (!outlines.isIndirect()) {
203
0
            root.replaceKey("/Outlines", qpdf.makeIndirectObject(outlines));
204
0
        }
205
0
    }
206
207
    // Traverse pages tree pushing all inherited resources down to the page level.  This also
208
    // initializes m->all_pages.
209
0
    m->pages.pushInheritedAttributesToPage(allow_changes, false);
210
211
    // Traverse pages to assign each object to a linearization category and to record which
212
    // objects each page references. This operation dominates the time spent linearizing. To
213
    // support more accurate progress reporting, we report progress through this operation when a
214
    // callback is registered. Throttle to one event per integer-percent change to minimize
215
    // overhead.
216
0
    size_t const total = m->pages.size();
217
0
    page_objs_.resize(total);
218
0
    int last_pct = -1;
219
0
    size_t n = 0;
220
0
    for (auto const& page: m->pages) {
221
0
        updateObjectMaps(
222
0
            ObjUser(ObjUser::ou_page, n), page, object_stream_data, skip_stream_parameters);
223
0
        ++n;
224
0
        if (progress_callback_ && total > 0) {
225
0
            int pct = static_cast<int>((100ULL * n) / total);
226
0
            if (pct != last_pct) {
227
0
                progress_callback_(pct);
228
0
                last_pct = pct;
229
0
            }
230
0
        }
231
0
    }
232
233
    // Traverse document-level items
234
0
    for (auto const& [key, value]: m->trailer.as_dictionary()) {
235
0
        if (key == "/Root") {
236
            // handled separately
237
0
        } else {
238
0
            if (!value.null()) {
239
0
                updateObjectMaps(
240
0
                    ObjUser(ObjUser::ou_trailer_key, key),
241
0
                    value,
242
0
                    object_stream_data,
243
0
                    skip_stream_parameters);
244
0
            }
245
0
        }
246
0
    }
247
248
0
    for (auto const& [key, value]: root.as_dictionary()) {
249
        // Technically, /I keys from /Thread dictionaries are supposed to be handled separately, but
250
        // we are going to disregard that specification for now.  There is loads of evidence that
251
        // pdlin and Acrobat both disregard things like this from time to time, so this is almost
252
        // certain not to cause any problems.
253
0
        if (!value.null()) {
254
0
            updateObjectMaps(
255
0
                ObjUser(ObjUser::ou_root_key, key),
256
0
                value,
257
0
                object_stream_data,
258
0
                skip_stream_parameters);
259
0
        }
260
0
    }
261
262
0
    ObjUser root_ou = ObjUser(ObjUser::ou_root);
263
0
    auto root_og = root.id_gen();
264
0
    resolveCompressedObject(root_og, object_stream_data);
265
0
    obj_categories_[root_og].update(root_ou, true);
266
267
0
    for (auto& objs: page_objs_) {
268
0
        std::sort(objs.begin(), objs.end());
269
0
        objs.erase(std::unique(objs.begin(), objs.end()), objs.end());
270
0
    }
271
0
}
Unexecuted instantiation: void QPDF::Doc::Linearization::prepare<std::__1::map<int, int, std::__1::less<int>, std::__1::allocator<std::__1::pair<int const, int> > > >(std::__1::map<int, int, std::__1::less<int>, std::__1::allocator<std::__1::pair<int const, int> > > const&, bool, std::__1::function<int (QPDFObjectHandle&)>)
Unexecuted instantiation: void QPDF::Doc::Linearization::prepare<QPDFWriter::ObjTable>(QPDFWriter::ObjTable const&, bool, std::__1::function<int (QPDFObjectHandle&)>)
272
273
void
274
Lin::ObjCategory::update(ObjUser const& other_ou, bool top)
275
0
{
276
0
    auto other = other_ou.obj_category(top);
277
    // Figure out what category this should be when combined with `other`. Earlier parts take
278
    // precedence over later parts. This logic is non-trivial for objects that are referenced from
279
    // more than one category. For example, if an object is referenced by the first page and the
280
    // root, it goes in the root category. If it's accessed from a non-first page and a first page,
281
    // it belongs in a first page category. This is only part of the logic. See remaining comments.
282
0
    auto min_category = std::min(lin_category_, other.lin_category_);
283
0
    auto max_category = std::max(lin_category_, other.lin_category_);
284
0
    auto this_pageno = pageno_;
285
0
    if (this_pageno <= 0 || other.pageno_ <= 0) {
286
0
        pageno_ = std::max(this_pageno, other.pageno_);
287
0
    } else {
288
        // If an object is referenced by more than one page, count it as being on the first one
289
        // for purposes of placement.
290
0
        pageno_ = std::min(this_pageno, other.pageno_);
291
0
    }
292
0
    if (min_category <= c_first_page_shared) {
293
        // The object is in part 4, or is already known to be referenced from the first page and at
294
        // least one other page. Nothing we can learn later changes this, so keep the highest
295
        // priority of the two values.
296
0
        lin_category_ = min_category;
297
0
        return;
298
0
    }
299
0
    if (min_category == c_first_page_private) {
300
0
        if (max_category == c_other_page_private || max_category == c_other_page_shared) {
301
            // We know this is referenced by the first page and at least one other page.
302
0
            lin_category_ = c_first_page_shared;
303
0
            return;
304
0
        }
305
        // Whatever the max category is, if the object is referenced by the first page, we need to
306
        // ensure it ends up in part 6 (unless it was already placed in part 4).
307
0
        lin_category_ = c_first_page_private;
308
0
        return;
309
0
    }
310
0
    if (min_category == c_other_page_shared) {
311
        // This is already known to be accessed by multiple non-first pages.
312
0
        lin_category_ = c_other_page_shared;
313
0
        return;
314
0
    }
315
0
    if (min_category == c_other_page_private) {
316
0
        if (max_category == c_other_page_private && this_pageno != other.pageno_) {
317
            // This is now known to be referenced from multiple pages.
318
0
            lin_category_ = c_other_page_shared;
319
0
        } else {
320
0
            lin_category_ = c_other_page_private;
321
0
        }
322
0
        return;
323
0
    }
324
    // Thumbnail logic mirrors page logic
325
0
    if (min_category == c_thumbnail_shared) {
326
0
        lin_category_ = c_thumbnail_shared;
327
0
        return;
328
0
    }
329
0
    if (min_category == c_thumbnail_private) {
330
0
        if (max_category == c_thumbnail_private && this_pageno != other.pageno_) {
331
0
            lin_category_ = c_thumbnail_shared;
332
0
        } else {
333
0
            lin_category_ = c_thumbnail_private;
334
0
        }
335
0
        return;
336
0
    }
337
    // All that remains is outlines and other. Keep whichever is highest priority.
338
0
    lin_category_ = min_category;
339
0
}
340
341
template <typename T>
342
void
343
Lin::updateObjectMaps(
344
    ObjUser const& first_ou,
345
    QPDFObjectHandle first_oh,
346
    T const& object_stream_data,
347
    std::function<int(QPDFObjectHandle&)> skip_stream_parameters)
348
0
{
349
0
    visited_.clear();
350
0
    visited_.resize(max_obj_, false);
351
352
0
    std::vector<UpdateObjectMapsFrame> pending;
353
0
    pending.emplace_back(first_ou, first_oh, true);
354
    // Traverse the object tree from this point taking care to avoid crossing page boundaries.
355
0
    std::unique_ptr<ObjUser> thumb_ou;
356
0
    while (!pending.empty()) {
357
0
        auto cur = pending.back();
358
0
        pending.pop_back();
359
360
0
        bool is_page_node = false;
361
0
        bool is_stream = false;
362
0
        int ssp = 0;
363
364
0
        if (cur.oh.indirect()) {
365
0
            auto og = cur.oh.id_gen();
366
0
            const int int_id = og.getObj();
367
0
            if (int_id <= 0 || std::cmp_greater_equal(int_id, visited_.size())) {
368
0
                no_ci_lin_warn_if(!cur.oh.null(), "ignoring unexpected object " + og.unparse(' '));
369
0
                continue;
370
0
            }
371
0
            const size_t id = static_cast<size_t>(int_id);
372
0
            if (visited_[id]) {
373
0
                QTC::TC("qpdf", "QPDF opt loop detected");
374
0
                continue;
375
0
            }
376
0
            visited_[id] = true;
377
378
0
            if (cur.oh.isStream()) {
379
0
                is_stream = true;
380
0
                if (skip_stream_parameters) {
381
0
                    ssp = skip_stream_parameters(cur.oh);
382
0
                }
383
0
            } else {
384
0
                if (Name type = cur.oh["/Type"]) {
385
                    // In prepare we called pushInheritedAttributesToPage which ensures that all
386
                    // genuine page tree nodes are indirect.  We therefore don't need to test direct
387
                    // objects.
388
0
                    if (type == "/Page") {
389
0
                        is_page_node = true;
390
0
                        if (!cur.top) {
391
0
                            continue;
392
0
                        }
393
0
                    } else if (first_ou.page() && type == "/Pages") {
394
0
                        continue;
395
0
                    }
396
0
                }
397
398
0
                resolveCompressedObject(og, object_stream_data);
399
0
            }
400
0
            obj_categories_[og].update(cur.ou, cur.top);
401
0
            if (cur.ou.page()) {
402
0
                page_objs_.at(cur.ou.pageno).push_back(og.getObj());
403
0
            }
404
0
        }
405
406
0
        if (cur.oh.isArray()) {
407
0
            for (auto const& item: cur.oh.as_array()) {
408
0
                pending.emplace_back(cur.ou, item, false);
409
0
            }
410
0
        } else if (Dictionary dict = is_stream ? cur.oh.getDict() : cur.oh) {
411
0
            for (auto& [key, value]: dict) {
412
0
                if (value.null()) {
413
0
                    continue;
414
0
                }
415
416
0
                if (is_page_node && key == "/Thumb") {
417
                    // Traverse page thumbnail dictionaries as a special case. There can only ever
418
                    // be one /Thumb key on a page, and we see at most one page node per call.
419
0
                    thumb_ou = std::make_unique<ObjUser>(ObjUser::ou_thumb, cur.ou.pageno);
420
0
                    pending.emplace_back(*thumb_ou, value, false);
421
0
                } else if (is_page_node && (key == "/Parent")) {
422
                    // Don't traverse back up the page tree
423
0
                } else if (
424
0
                    ssp >= 1 &&
425
0
                    (key == "/Length" ||
426
0
                     (ssp >= 2 && (key == "/Filter" || key == "/DecodeParms")))) {
427
                    // Don't traverse into stream parameters that we are not going to write.
428
0
                } else {
429
0
                    pending.emplace_back(cur.ou, value, false);
430
0
                }
431
0
            }
432
0
        }
433
0
    }
434
0
}
Unexecuted instantiation: void QPDF::Doc::Linearization::updateObjectMaps<std::__1::map<int, int, std::__1::less<int>, std::__1::allocator<std::__1::pair<int const, int> > > >(QPDF::Doc::Linearization::ObjUser const&, QPDFObjectHandle, std::__1::map<int, int, std::__1::less<int>, std::__1::allocator<std::__1::pair<int const, int> > > const&, std::__1::function<int (QPDFObjectHandle&)>)
Unexecuted instantiation: void QPDF::Doc::Linearization::updateObjectMaps<QPDFWriter::ObjTable>(QPDF::Doc::Linearization::ObjUser const&, QPDFObjectHandle, QPDFWriter::ObjTable const&, std::__1::function<int (QPDFObjectHandle&)>)
435
436
void
437
Lin::resolveCompressedObject(QPDFObjGen& og, std::map<int, int> const& object_stream_data)
438
0
{
439
0
    auto owner = object_stream_data.find(og.getObj());
440
0
    if (owner != object_stream_data.end()) {
441
0
        og = {owner->second, 0};
442
0
    }
443
0
}
444
445
void
446
Lin::resolveCompressedObject(QPDFObjGen& og, QPDFWriter::ObjTable const& obj)
447
0
{
448
0
    if (auto owner = obj.contains(og) ? obj[og].object_stream : 0; owner > 0) {
449
0
        og = {owner, 0};
450
0
    }
451
0
}
452
453
void
454
Lin::linearizationWarning(std::string_view msg)
455
0
{
456
0
    linearization_warnings_ = true;
457
0
    warn(qpdf_e_linearization, "", 0, std::string(msg));
458
0
}
459
460
void
461
Lin::no_ci_lin_warn_if(bool condition, std::string_view msg)
462
0
{
463
0
    if (condition) {
464
0
        linearizationWarning(msg);
465
0
    }
466
0
}
467
468
bool
469
QPDF::checkLinearization()
470
0
{
471
0
    return m->lin.check();
472
0
}
473
474
bool
475
Lin::check()
476
0
{
477
0
    try {
478
0
        readLinearizationData();
479
0
        checkLinearizationInternal();
480
0
        return !linearization_warnings_;
481
0
    } catch (std::runtime_error& e) {
482
0
        linearizationWarning(
483
0
            "error encountered while checking linearization data: " + std::string(e.what()));
484
0
        return false;
485
0
    }
486
0
}
487
488
bool
489
QPDF::isLinearized()
490
0
{
491
0
    return m->lin.linearized();
492
0
}
493
494
bool
495
Lin::linearized()
496
0
{
497
    // If the first object in the file is a dictionary with a suitable /Linearized key and has an /L
498
    // key that accurately indicates the file size, initialize m->lindict and return true.
499
500
    // A linearized PDF spec's first object will be contained within the first 1024 bytes of the
501
    // file and will be a dictionary with a valid /Linearized key.  This routine looks for that and
502
    // does no additional validation.
503
504
    // The PDF spec says the linearization dictionary must be completely contained within the first
505
    // 1024 bytes of the file. Add a byte for a null terminator.
506
0
    auto buffer = m->file->read(1024, 0);
507
0
    size_t pos = 0;
508
0
    while (true) {
509
        // Find a digit or end of buffer
510
0
        pos = buffer.find_first_of("0123456789"sv, pos);
511
0
        if (pos == std::string::npos) {
512
0
            return false;
513
0
        }
514
        // Seek to the digit. Then skip over digits for a potential
515
        // next iteration.
516
0
        m->file->seek(toO(pos), SEEK_SET);
517
518
0
        auto t1 = m->objects.readToken(*m->file, 20);
519
0
        if (!(t1.isInteger() && m->objects.readToken(*m->file, 6).isInteger() &&
520
0
              m->objects.readToken(*m->file, 4).isWord("obj"))) {
521
0
            pos = buffer.find_first_not_of("0123456789"sv, pos);
522
0
            if (pos == std::string::npos) {
523
0
                return false;
524
0
            }
525
0
            continue;
526
0
        }
527
528
0
        Dictionary candidate = qpdf.getObject(toI(QUtil::string_to_ll(t1.getValue().data())), 0);
529
0
        auto linkey = candidate["/Linearized"];
530
0
        if (!(linkey.isNumber() && toI(floor(linkey.getNumericValue())) == 1)) {
531
0
            return false;
532
0
        }
533
534
0
        m->file->seek(0, SEEK_END);
535
0
        Integer L = candidate["/L"];
536
0
        if (L != m->file->tell()) {
537
0
            return false;
538
0
        }
539
0
        linp_.file_size = L;
540
0
        lindict_ = candidate;
541
0
        return true;
542
0
    }
543
0
}
544
545
void
546
Lin::readLinearizationData()
547
0
{
548
0
    util::assertion(
549
0
        linearized(), "called readLinearizationData for file that is not linearized" //
550
0
    );
551
552
    // This function throws an exception (which is trapped by checkLinearization()) for any errors
553
    // that prevent loading.
554
555
    // /L is read and stored in linp by isLinearized()
556
0
    Array H = lindict_["/H"]; // hint table offset/length for primary and overflow hint tables
557
0
    auto H_size = H.size();
558
0
    Integer H_0 = H[0]; // hint table offset
559
0
    Integer H_1 = H[1]; // hint table length
560
0
    Integer H_2 = H[2]; // hint table offset for overflow hint table
561
0
    Integer H_3 = H[3]; // hint table length for overflow hint table
562
0
    Integer O = lindict_["/O"];
563
0
    Integer E = lindict_["/E"];
564
0
    Integer N = lindict_["/N"];
565
0
    Integer T = lindict_["/T"];
566
0
    auto P_oh = lindict_["/P"];
567
0
    Integer P = P_oh; // first page number
568
0
    QTC::TC("qpdf", "QPDF P absent in lindict", P ? 0 : 1);
569
570
0
    no_ci_stop_if(
571
0
        !(H && O && E && N && T && (P || P_oh.null())),
572
0
        "some keys in linearization dictionary are of the wrong type",
573
0
        "linearization dictionary" //
574
0
    );
575
576
0
    no_ci_stop_if(
577
0
        !(H_size == 2 || H_size == 4),
578
0
        "H has the wrong number of items",
579
0
        "linearization dictionary" //
580
0
    );
581
582
0
    no_ci_stop_if(
583
0
        !(H_0 && H_1 && (H_size == 2 || (H_2 && H_3))),
584
0
        "some H items are of the wrong type",
585
0
        "linearization dictionary" //
586
0
    );
587
588
    // Store linearization parameter data
589
590
    // Various places in the code use linp.npages, which is initialized from N, to pre-allocate
591
    // memory, so make sure it's accurate and bail right now if it's not.
592
0
    no_ci_stop_if(
593
0
        N != pages.size(),
594
0
        "/N does not match number of pages",
595
0
        "linearization dictionary" //
596
0
    );
597
598
    // file_size initialized by isLinearized()
599
0
    linp_.first_page_object = O.value<int>();
600
0
    linp_.first_page_end = E;
601
0
    linp_.npages = N.value<size_t>();
602
0
    linp_.xref_zero_offset = T;
603
0
    linp_.first_page = P ? P.value<int>() : 0;
604
0
    linp_.H_offset = H_0;
605
0
    linp_.H_length = H_1;
606
607
    // Read hint streams
608
609
0
    Pl_Buffer pb("hint buffer");
610
0
    auto H0 = readHintStream(pb, H_0, H_1.value<size_t>());
611
0
    if (H_2) {
612
0
        (void)readHintStream(pb, H_2, H_3.value<size_t>());
613
0
    }
614
615
    // PDF 1.4 hint tables that we ignore:
616
617
    //  /T    thumbnail
618
    //  /A    thread information
619
    //  /E    named destination
620
    //  /V    interactive form
621
    //  /I    information dictionary
622
    //  /C    logical structure
623
    //  /L    page label
624
625
    // Individual hint table offsets
626
0
    Integer HS = H0["/S"]; // shared object
627
0
    Integer HO = H0["/O"]; // outline
628
629
0
    auto hbp = pb.getBufferSharedPointer();
630
0
    Buffer* hb = hbp.get();
631
0
    unsigned char const* h_buf = hb->getBuffer();
632
0
    size_t h_size = hb->getSize();
633
634
0
    readHPageOffset(BitStream(h_buf, h_size));
635
636
0
    size_t HSi = HS.value<size_t>();
637
0
    if (HSi < 0 || HSi >= h_size) {
638
0
        throw damagedPDF("linearization hint table", "/S (shared object) offset is out of bounds");
639
0
    }
640
0
    readHSharedObject(BitStream(h_buf + HSi, h_size - HSi));
641
642
0
    if (HO) {
643
0
        no_ci_stop_if(
644
0
            HO < 0 || HO >= h_size,
645
0
            "/O (outline) offset is out of bounds",
646
0
            "linearization dictionary" //
647
0
        );
648
0
        size_t HOi = HO.value<size_t>();
649
0
        readHGeneric(BitStream(h_buf + HO, h_size - HOi), outline_hints_);
650
0
    }
651
0
}
652
653
Dictionary
654
Lin::readHintStream(Pipeline& pl, qpdf_offset_t offset, size_t length)
655
0
{
656
0
    auto H = m->objects.readObjectAtOffset(offset, "linearization hint stream", false);
657
0
    no_ci_stop_if(
658
0
        !H.isStream(), "hint table is not a stream", "linearization dictionary" //
659
0
    );
660
0
    ObjCache& oc = m->obj_cache[H];
661
0
    qpdf_offset_t min_end_offset = oc.end_before_space;
662
0
    qpdf_offset_t max_end_offset = oc.end_after_space;
663
664
0
    Dictionary Hdict = H.getDict();
665
666
    // Some versions of Acrobat make /Length indirect and place it immediately after the stream,
667
    // increasing length to cover it, even though the specification says all objects in the
668
    // linearization parameter dictionary must be direct.  We have to get the file position of the
669
    // end of length in this case.
670
0
    if (Hdict["/Length"].indirect()) {
671
0
        ObjCache& oc2 = m->obj_cache[Hdict["/Length"]];
672
0
        min_end_offset = oc2.end_before_space;
673
0
        max_end_offset = oc2.end_after_space;
674
0
    } else {
675
0
        QTC::TC("qpdf", "QPDF hint table length direct");
676
0
    }
677
0
    qpdf_offset_t computed_end = offset + toO(length);
678
0
    no_ci_stop_if(
679
0
        computed_end < min_end_offset || computed_end > max_end_offset,
680
0
        "hint table length mismatch (expected = " + std::to_string(computed_end) + "; actual = " +
681
0
            std::to_string(min_end_offset) + ".." + std::to_string(max_end_offset) + ")",
682
0
        "linearization dictionary" //
683
0
    );
684
0
    H.pipeStreamData(&pl, 0, qpdf_dl_specialized);
685
0
    return Hdict;
686
0
}
687
688
void
689
Lin::readHPageOffset(BitStream h)
690
0
{
691
    // All comments referring to the PDF spec refer to the spec for version 1.4.
692
693
0
    HPageOffset& t = page_offset_hints_;
694
695
0
    t.min_nobjects = h.getBitsInt(32);               // 1
696
0
    t.first_page_offset = h.getBitsInt(32);          // 2
697
0
    t.nbits_delta_nobjects = h.getBitsInt(16);       // 3
698
0
    t.min_page_length = h.getBitsInt(32);            // 4
699
0
    t.nbits_delta_page_length = h.getBitsInt(16);    // 5
700
0
    t.min_content_offset = h.getBitsInt(32);         // 6
701
0
    t.nbits_delta_content_offset = h.getBitsInt(16); // 7
702
0
    t.min_content_length = h.getBitsInt(32);         // 8
703
0
    t.nbits_delta_content_length = h.getBitsInt(16); // 9
704
0
    t.nbits_nshared_objects = h.getBitsInt(16);      // 10
705
0
    t.nbits_shared_identifier = h.getBitsInt(16);    // 11
706
0
    t.nbits_shared_numerator = h.getBitsInt(16);     // 12
707
0
    t.shared_denominator = h.getBitsInt(16);         // 13
708
709
0
    std::vector<HPageOffsetEntry>& entries = t.entries;
710
0
    entries.clear();
711
0
    int nitems = toI(linp_.npages);
712
0
    load_vector_int(h, nitems, entries, t.nbits_delta_nobjects, &HPageOffsetEntry::delta_nobjects);
713
0
    load_vector_int(
714
0
        h, nitems, entries, t.nbits_delta_page_length, &HPageOffsetEntry::delta_page_length);
715
0
    load_vector_int(
716
0
        h, nitems, entries, t.nbits_nshared_objects, &HPageOffsetEntry::nshared_objects);
717
0
    load_vector_vector(
718
0
        h,
719
0
        nitems,
720
0
        entries,
721
0
        &HPageOffsetEntry::nshared_objects,
722
0
        t.nbits_shared_identifier,
723
0
        &HPageOffsetEntry::shared_identifiers);
724
0
    load_vector_vector(
725
0
        h,
726
0
        nitems,
727
0
        entries,
728
0
        &HPageOffsetEntry::nshared_objects,
729
0
        t.nbits_shared_numerator,
730
0
        &HPageOffsetEntry::shared_numerators);
731
0
    load_vector_int(
732
0
        h, nitems, entries, t.nbits_delta_content_offset, &HPageOffsetEntry::delta_content_offset);
733
0
    load_vector_int(
734
0
        h, nitems, entries, t.nbits_delta_content_length, &HPageOffsetEntry::delta_content_length);
735
0
}
736
737
void
738
Lin::readHSharedObject(BitStream h)
739
0
{
740
0
    HSharedObject& t = shared_object_hints_;
741
742
0
    t.first_shared_obj = h.getBitsInt(32);         // 1
743
0
    t.first_shared_offset = h.getBitsInt(32);      // 2
744
0
    t.nshared_first_page = h.getBitsInt(32);       // 3
745
0
    t.nshared_total = h.getBitsInt(32);            // 4
746
0
    t.nbits_nobjects = h.getBitsInt(16);           // 5
747
0
    t.min_group_length = h.getBitsInt(32);         // 6
748
0
    t.nbits_delta_group_length = h.getBitsInt(16); // 7
749
750
0
    QTC::TC(
751
0
        "qpdf",
752
0
        "QPDF lin nshared_total > nshared_first_page",
753
0
        (t.nshared_total > t.nshared_first_page) ? 1 : 0);
754
755
0
    std::vector<HSharedObjectEntry>& entries = t.entries;
756
0
    entries.clear();
757
0
    int nitems = t.nshared_total;
758
0
    load_vector_int(
759
0
        h, nitems, entries, t.nbits_delta_group_length, &HSharedObjectEntry::delta_group_length);
760
0
    load_vector_int(h, nitems, entries, 1, &HSharedObjectEntry::signature_present);
761
0
    for (size_t i = 0; i < toS(nitems); ++i) {
762
0
        if (entries.at(i).signature_present) {
763
            // Skip 128-bit MD5 hash.  These are not supported by acrobat, so they should probably
764
            // never be there.  We have no test case for this.
765
0
            for (int j = 0; j < 4; ++j) {
766
0
                (void)h.getBits(32);
767
0
            }
768
0
        }
769
0
    }
770
0
    load_vector_int(h, nitems, entries, t.nbits_nobjects, &HSharedObjectEntry::nobjects_minus_one);
771
0
}
772
773
void
774
Lin::readHGeneric(BitStream h, HGeneric& t)
775
0
{
776
0
    t.first_object = h.getBitsInt(32);        // 1
777
0
    t.first_object_offset = h.getBitsInt(32); // 2
778
0
    t.nobjects = h.getBitsInt(32);            // 3
779
0
    t.group_length = h.getBitsInt(32);        // 4
780
0
}
781
782
void
783
Lin::checkLinearizationInternal()
784
0
{
785
    // All comments referring to the PDF spec refer to the spec for version 1.4.
786
787
    // Check all values in linearization parameter dictionary
788
789
0
    LinParameters& p = linp_;
790
791
    // L: file size in bytes -- checked by isLinearized
792
793
    // O: object number of first page
794
0
    auto const& all_pages = pages.all();
795
0
    if (p.first_page_object != all_pages.at(0).getObjectID()) {
796
0
        linearizationWarning("first page object (/O) mismatch");
797
0
    }
798
799
    // N: number of pages
800
0
    size_t npages = all_pages.size();
801
0
    if (std::cmp_not_equal(p.npages, npages)) {
802
        // Not tested in the test suite
803
0
        linearizationWarning("page count (/N) mismatch");
804
0
    }
805
806
0
    int i = 0;
807
0
    for (auto const& page: all_pages) {
808
0
        if (m->xref_table[page].getType() == 2) {
809
0
            linearizationWarning(
810
0
                "page dictionary for page " + std::to_string(i) + " is compressed");
811
0
        }
812
0
        ++i;
813
0
    }
814
815
    // T: offset of whitespace character preceding xref entry for object 0
816
0
    m->file->seek(p.xref_zero_offset, SEEK_SET);
817
0
    while (true) {
818
0
        char ch;
819
0
        m->file->read(&ch, 1);
820
0
        if (!(ch == ' ' || ch == '\r' || ch == '\n')) {
821
0
            m->file->seek(-1, SEEK_CUR);
822
0
            break;
823
0
        }
824
0
    }
825
0
    if (m->file->tell() != objects.first_xref_item_offset()) {
826
0
        linearizationWarning(
827
0
            "space before first xref item (/T) mismatch (computed = " +
828
0
            std::to_string(objects.first_xref_item_offset()) +
829
0
            "; file = " + std::to_string(m->file->tell()));
830
0
    }
831
832
    // P: first page number -- Implementation note 124 says Acrobat ignores this value, so we will
833
    // too.
834
835
    // Check numbering of compressed objects in each xref section. For linearized files, all
836
    // compressed objects are supposed to be at the end of the containing xref section if any object
837
    // streams are in use.
838
839
0
    if (objects.uncompressed_after_compressed()) {
840
0
        linearizationWarning(
841
0
            "linearized file contains an uncompressed object after a compressed "
842
0
            "one in a cross-reference stream");
843
0
    }
844
845
    // Further checking requires preparation and order calculation. Don't allow preparation to
846
    // make changes.  If it has to, then the file is not properly linearized.  We use the xref table
847
    // to figure out which objects are compressed and which are uncompressed.
848
0
    { // local scope
849
0
        std::map<int, int> object_stream_data;
850
0
        for (auto const& [og, entry]: m->xref_table) {
851
0
            if (entry.getType() == 2) {
852
0
                object_stream_data[og.getObj()] = entry.getObjStreamNumber();
853
0
            }
854
0
        }
855
0
        prepare(object_stream_data, false, nullptr);
856
0
        calculateLinearizationData(object_stream_data);
857
0
    }
858
859
    // E: offset of end of first page -- Implementation note 123 says Acrobat includes on extra
860
    // object here by mistake.  pdlin fails to place thumbnail images in section 9, so when
861
    // thumbnails are present, it also gets the wrong value for /E.  It also doesn't count outlines
862
    // here when it should even though it places them in part 6.  This code fails to put thread
863
    // information dictionaries in part 9, so it actually gets the wrong value for E when threads
864
    // are present.  In that case, it would probably agree with pdlin.  As of this writing, the test
865
    // suite doesn't contain any files with threads.
866
867
0
    no_ci_stop_if(
868
0
        part6_.empty(), "linearization part 6 unexpectedly empty" //
869
0
    );
870
0
    qpdf_offset_t min_E = -1;
871
0
    qpdf_offset_t max_E = -1;
872
0
    for (auto const& oh: part6_) {
873
0
        QPDFObjGen og(oh.getObjGen());
874
        // All objects have to have been dereferenced to be classified.
875
0
        util::assertion(m->obj_cache.contains(og), "linearization part6 object not in cache");
876
0
        ObjCache const& oc = m->obj_cache[og];
877
0
        min_E = std::max(min_E, oc.end_before_space);
878
0
        max_E = std::max(max_E, oc.end_after_space);
879
0
    }
880
0
    if (p.first_page_end < min_E || p.first_page_end > max_E) {
881
0
        linearizationWarning(
882
0
            "end of first page section (/E) mismatch: /E = " + std::to_string(p.first_page_end) +
883
0
            "; computed = " + std::to_string(min_E) + ".." + std::to_string(max_E));
884
0
    }
885
886
    // Check hint tables
887
888
0
    std::map<int, int> shared_idx_to_obj;
889
0
    checkHSharedObject(all_pages, shared_idx_to_obj);
890
0
    checkHPageOffset(all_pages, shared_idx_to_obj);
891
0
    checkHOutlines();
892
0
}
893
894
qpdf_offset_t
895
Lin::objectEnd(QPDFObjGen og) const
896
0
{
897
0
    auto oc = m->obj_cache.find(og);
898
0
    return oc == m->obj_cache.end() ? 0 : oc->second.end_after_space;
899
0
}
900
901
qpdf_offset_t
902
Lin::getLinearizationOffset(QPDFObjGen og, bool require_type_1)
903
0
{
904
0
    QPDFXRefEntry const& entry = m->xref_table[og];
905
0
    auto typ = entry.getType();
906
0
    if (typ == 1) {
907
0
        return entry.getOffset();
908
0
    }
909
0
    no_ci_stop_if(typ != 2, "getLinearizationOffset called for xref entry not of type 1 or 2");
910
0
    if (require_type_1) {
911
0
        stopOnError(
912
0
            "the object stream containing an object is itself contained in an object stream");
913
0
    }
914
915
    // For compressed objects, return the offset of the object stream that contains them.
916
0
    return getLinearizationOffset({entry.getObjStreamNumber(), 0}, true);
917
0
}
918
919
QPDFObjectHandle
920
Lin::getUncompressedObject(QPDFObjectHandle& obj, std::map<int, int> const& object_stream_data)
921
0
{
922
0
    if (obj.null() || !object_stream_data.contains(obj.getObjectID())) {
923
0
        return obj;
924
0
    }
925
0
    return qpdf.getObject((*(object_stream_data.find(obj.getObjectID()))).second, 0);
926
0
}
927
928
QPDFObjectHandle
929
Lin::getUncompressedObject(QPDFObjectHandle& oh, QPDFWriter::ObjTable const& obj)
930
0
{
931
0
    if (obj.contains(oh)) {
932
0
        if (auto id = obj[oh].object_stream; id > 0) {
933
0
            return oh.null() ? oh : qpdf.getObject(id, 0);
934
0
        }
935
0
    }
936
0
    return oh;
937
0
}
938
939
int
940
Lin::lengthNextN(int first_object, int n)
941
0
{
942
0
    int length = 0;
943
0
    for (int i = 0; i < n; ++i) {
944
0
        QPDFObjGen og(first_object + i, 0);
945
0
        if (m->xref_table.contains(og)) {
946
0
            no_ci_stop_if(
947
0
                !m->obj_cache.contains(og),
948
0
                "found unknown object while calculating length for linearization data" //
949
0
            );
950
951
0
            length += toI(m->obj_cache[og].end_after_space - getLinearizationOffset(og));
952
0
        } else {
953
0
            linearizationWarning(
954
0
                "no xref table entry for " + std::to_string(first_object + i) + " 0");
955
0
        }
956
0
    }
957
0
    return length;
958
0
}
959
960
void
961
Lin::checkHPageOffset(
962
    std::vector<QPDFObjectHandle> const& pages, std::map<int, int>& shared_idx_to_obj)
963
0
{
964
    // Implementation note 126 says Acrobat always sets delta_content_offset and
965
    // delta_content_length in the page offset header dictionary to 0.  It also states that
966
    // min_content_offset in the per-page information is always 0, which is an incorrect value.
967
968
    // Implementation note 127 explains that Acrobat always sets item 8 (min_content_length) to
969
    // zero, item 9 (nbits_delta_content_length) to the value of item 5 (nbits_delta_page_length),
970
    // and item 7 of each per-page hint table (delta_content_length) to item 2 (delta_page_length)
971
    // of that entry.  Acrobat ignores these values when reading files.
972
973
    // Empirically, it also seems that Acrobat sometimes puts items under a page's /Resources
974
    // dictionary in with shared objects even when they are private.
975
976
0
    size_t npages = pages.size();
977
0
    qpdf_offset_t table_offset = adjusted_offset(page_offset_hints_.first_page_offset);
978
0
    QPDFObjGen first_page_og(pages.at(0).getObjGen());
979
0
    if (!m->xref_table.contains(first_page_og)) {
980
0
        stopOnError("supposed first page object is not known");
981
0
    }
982
0
    qpdf_offset_t offset = getLinearizationOffset(first_page_og);
983
0
    if (table_offset != offset) {
984
0
        linearizationWarning("first page object offset mismatch");
985
0
    }
986
987
0
    for (size_t pageno = 0; pageno < npages; ++pageno) {
988
0
        QPDFObjGen page_og(pages.at(pageno).getObjGen());
989
0
        int first_object = page_og.getObj();
990
0
        if (!m->xref_table.contains(page_og)) {
991
0
            stopOnError("unknown object in page offset hint table");
992
0
        }
993
0
        offset = getLinearizationOffset(page_og);
994
995
0
        HPageOffsetEntry& he = page_offset_hints_.entries.at(pageno);
996
0
        CHPageOffsetEntry& ce = c_page_offset_data_.entries.at(pageno);
997
0
        int h_nobjects = he.delta_nobjects + page_offset_hints_.min_nobjects;
998
0
        if (h_nobjects != ce.nobjects) {
999
            // This happens with pdlin when there are thumbnails.
1000
0
            linearizationWarning(
1001
0
                "object count mismatch for page " + std::to_string(pageno) + ": hint table = " +
1002
0
                std::to_string(h_nobjects) + "; computed = " + std::to_string(ce.nobjects));
1003
0
        }
1004
1005
        // Use value for number of objects in hint table rather than computed value if there is a
1006
        // discrepancy.
1007
0
        int length = lengthNextN(first_object, h_nobjects);
1008
0
        int h_length = toI(he.delta_page_length + page_offset_hints_.min_page_length);
1009
0
        if (length != h_length) {
1010
            // This condition almost certainly indicates a bad hint table or a bug in this code.
1011
0
            linearizationWarning(
1012
0
                "page length mismatch for page " + std::to_string(pageno) + ": hint table = " +
1013
0
                std::to_string(h_length) + "; computed length = " + std::to_string(length) +
1014
0
                " (offset = " + std::to_string(offset) + ")");
1015
0
        }
1016
1017
0
        offset += h_length;
1018
1019
        // Translate shared object indexes to object numbers.
1020
0
        std::set<int> hint_shared;
1021
0
        std::set<int> computed_shared;
1022
1023
0
        if (pageno == 0 && he.nshared_objects > 0) {
1024
            // pdlin and Acrobat both do this even though the spec states clearly and unambiguously
1025
            // that they should not.
1026
0
            linearizationWarning("page 0 has shared identifier entries");
1027
0
        }
1028
1029
0
        for (size_t i = 0; i < toS(he.nshared_objects); ++i) {
1030
0
            int idx = he.shared_identifiers.at(i);
1031
0
            no_ci_stop_if(
1032
0
                !shared_idx_to_obj.contains(idx),
1033
0
                "unable to get object " + std::to_string(idx) +
1034
0
                    " for item in shared objects hint table",
1035
0
                "",
1036
0
                false //
1037
0
            );
1038
1039
0
            hint_shared.insert(shared_idx_to_obj[idx]);
1040
0
        }
1041
1042
0
        for (size_t i = 0; i < toS(ce.nshared_objects); ++i) {
1043
0
            int idx = ce.shared_identifiers.at(i);
1044
0
            no_ci_stop_if(
1045
0
                idx >= c_shared_object_data_.nshared_total,
1046
0
                "index out of bounds for shared object hint table" //
1047
0
            );
1048
1049
0
            int obj = c_shared_object_data_.entries.at(toS(idx)).object;
1050
0
            computed_shared.insert(obj);
1051
0
        }
1052
1053
0
        for (int iter: hint_shared) {
1054
0
            if (!computed_shared.contains(iter)) {
1055
                // pdlin puts thumbnails here even though it shouldn't
1056
0
                linearizationWarning(
1057
0
                    "page " + std::to_string(pageno) + ": shared object " + std::to_string(iter) +
1058
0
                    ": in hint table but not computed list");
1059
0
            }
1060
0
        }
1061
1062
0
        for (int iter: computed_shared) {
1063
0
            if (!hint_shared.contains(iter)) {
1064
                // Acrobat does not put some things including at least built-in fonts and procsets
1065
                // here, at least in some cases.
1066
0
                linearizationWarning(
1067
0
                    ("page " + std::to_string(pageno) + ": shared object " + std::to_string(iter) +
1068
0
                     ": in computed list but not hint table"));
1069
0
            }
1070
0
        }
1071
0
    }
1072
0
}
1073
1074
void
1075
Lin::checkHSharedObject(std::vector<QPDFObjectHandle> const& pages, std::map<int, int>& idx_to_obj)
1076
0
{
1077
    // Implementation note 125 says shared object groups always contain only one object.
1078
    // Implementation note 128 says that Acrobat always nbits_nobjects to zero.  Implementation note
1079
    // 130 says that Acrobat does not support more than one shared object per group.  These are all
1080
    // consistent.
1081
1082
    // Implementation note 129 states that MD5 signatures are not implemented in Acrobat, so
1083
    // signature_present must always be zero.
1084
1085
    // Implementation note 131 states that first_shared_obj and first_shared_offset have meaningless
1086
    // values for single-page files.
1087
1088
    // Empirically, Acrobat and pdlin generate incorrect values for these whenever there are no
1089
    // shared objects not referenced by the first page (i.e., nshared_total == nshared_first_page).
1090
1091
0
    HSharedObject& so = shared_object_hints_;
1092
0
    if (so.nshared_total < so.nshared_first_page) {
1093
0
        linearizationWarning("shared object hint table: ntotal < nfirst_page");
1094
0
    } else {
1095
        // The first nshared_first_page objects are consecutive objects starting with the first page
1096
        // object.  The rest are consecutive starting from the first_shared_obj object.
1097
0
        int cur_object = pages.at(0).getObjectID();
1098
0
        for (int i = 0; i < so.nshared_total; ++i) {
1099
0
            if (i == so.nshared_first_page) {
1100
0
                QTC::TC("qpdf", "QPDF lin check shared past first page");
1101
0
                if (part8_.empty()) {
1102
0
                    linearizationWarning("part 8 is empty but nshared_total > nshared_first_page");
1103
0
                } else {
1104
0
                    int obj = part8_.at(0).getObjectID();
1105
0
                    if (obj != so.first_shared_obj) {
1106
0
                        linearizationWarning(
1107
0
                            "first shared object number mismatch: hint table = " +
1108
0
                            std::to_string(so.first_shared_obj) +
1109
0
                            "; computed = " + std::to_string(obj));
1110
0
                    }
1111
0
                }
1112
1113
0
                cur_object = so.first_shared_obj;
1114
1115
0
                QPDFObjGen og(cur_object, 0);
1116
0
                if (!m->xref_table.contains(og)) {
1117
0
                    stopOnError("unknown object in shared object hint table");
1118
0
                }
1119
0
                qpdf_offset_t offset = getLinearizationOffset(og);
1120
0
                qpdf_offset_t h_offset = adjusted_offset(so.first_shared_offset);
1121
0
                if (offset != h_offset) {
1122
0
                    linearizationWarning(
1123
0
                        "first shared object offset mismatch: hint table = " +
1124
0
                        std::to_string(h_offset) + "; computed = " + std::to_string(offset));
1125
0
                }
1126
0
            }
1127
1128
0
            idx_to_obj[i] = cur_object;
1129
0
            HSharedObjectEntry& se = so.entries.at(toS(i));
1130
0
            int nobjects = se.nobjects_minus_one + 1;
1131
0
            int length = lengthNextN(cur_object, nobjects);
1132
0
            int h_length = so.min_group_length + se.delta_group_length;
1133
0
            if (length != h_length) {
1134
0
                linearizationWarning(
1135
0
                    "shared object " + std::to_string(i) + " length mismatch: hint table = " +
1136
0
                    std::to_string(h_length) + "; computed = " + std::to_string(length));
1137
0
            }
1138
0
            cur_object += nobjects;
1139
0
        }
1140
0
    }
1141
0
}
1142
1143
void
1144
Lin::checkHOutlines()
1145
0
{
1146
    // Empirically, Acrobat generates the correct value for the object number but incorrectly stores
1147
    // the next object number's offset as the offset, at least when outlines appear in part 6.  It
1148
    // also generates an incorrect value for length (specifically, the length that would cover the
1149
    // correct number of objects from the wrong starting place).  pdlin appears to generate correct
1150
    // values in those cases.
1151
1152
0
    if (c_outline_data_.nobjects == outline_hints_.nobjects) {
1153
0
        if (c_outline_data_.nobjects == 0) {
1154
0
            return;
1155
0
        }
1156
1157
0
        if (c_outline_data_.first_object == outline_hints_.first_object) {
1158
            // Check length and offset.  Acrobat gets these wrong.
1159
0
            QPDFObjectHandle outlines = qpdf.getRoot().getKey("/Outlines");
1160
0
            if (!outlines.isIndirect()) {
1161
                // This case is not exercised in test suite since not permitted by the spec, but if
1162
                // this does occur, the code below would fail.
1163
0
                linearizationWarning("/Outlines key of root dictionary is not indirect");
1164
0
                return;
1165
0
            }
1166
0
            QPDFObjGen og(outlines.getObjGen());
1167
0
            no_ci_stop_if(
1168
0
                !m->xref_table.contains(og), "unknown object in outlines hint table" //
1169
0
            );
1170
0
            qpdf_offset_t offset = getLinearizationOffset(og);
1171
0
            int length = toI(outlines_max_end_ - offset);
1172
0
            qpdf_offset_t table_offset = adjusted_offset(outline_hints_.first_object_offset);
1173
0
            if (offset != table_offset) {
1174
0
                linearizationWarning(
1175
0
                    "incorrect offset in outlines table: hint table = " +
1176
0
                    std::to_string(table_offset) + "; computed = " + std::to_string(offset));
1177
0
            }
1178
0
            int table_length = outline_hints_.group_length;
1179
0
            if (length != table_length) {
1180
0
                linearizationWarning(
1181
0
                    "incorrect length in outlines table: hint table = " +
1182
0
                    std::to_string(table_length) + "; computed = " + std::to_string(length));
1183
0
            }
1184
0
        } else {
1185
0
            linearizationWarning("incorrect first object number in outline hints table.");
1186
0
        }
1187
0
    } else {
1188
0
        linearizationWarning("incorrect object count in outline hint table");
1189
0
    }
1190
0
}
1191
1192
void
1193
QPDF::showLinearizationData()
1194
0
{
1195
0
    m->lin.show_data();
1196
0
}
1197
1198
void
1199
Lin::show_data()
1200
0
{
1201
0
    try {
1202
0
        readLinearizationData();
1203
0
        checkLinearizationInternal();
1204
0
    } catch (QPDFExc const& e) {
1205
0
        linearizationWarning(e.what());
1206
0
    }
1207
0
    try {
1208
0
        dumpLinearizationDataInternal();
1209
0
    } catch (QPDFExc const& e) {
1210
0
        linearizationWarning(e.what());
1211
0
    }
1212
0
}
1213
1214
void
1215
Lin::dumpLinearizationDataInternal()
1216
0
{
1217
0
    auto& info = *cf.log()->getInfo();
1218
1219
0
    info << m->file->getName() << ": linearization data:\n\n";
1220
1221
0
    info << "file_size: " << linp_.file_size << "\n"
1222
0
         << "first_page_object: " << linp_.first_page_object << "\n"
1223
0
         << "first_page_end: " << linp_.first_page_end << "\n"
1224
0
         << "npages: " << linp_.npages << "\n"
1225
0
         << "xref_zero_offset: " << linp_.xref_zero_offset << "\n"
1226
0
         << "first_page: " << linp_.first_page << "\n"
1227
0
         << "H_offset: " << linp_.H_offset << "\n"
1228
0
         << "H_length: " << linp_.H_length << "\n"
1229
0
         << "\n";
1230
1231
0
    info << "Page Offsets Hint Table\n\n";
1232
0
    dumpHPageOffset();
1233
0
    info << "\nShared Objects Hint Table\n\n";
1234
0
    dumpHSharedObject();
1235
1236
0
    if (outline_hints_.nobjects > 0) {
1237
0
        info << "\nOutlines Hint Table\n\n";
1238
0
        dumpHGeneric(outline_hints_);
1239
0
    }
1240
1241
0
    info << "\nComputed Linearization Part Assignments\n\n";
1242
0
    dumpPart("4 (root, document open)", part4_);
1243
0
    dumpPart("6 (first page)", part6_);
1244
0
    dumpPart("7 (other pages, private)", part7_);
1245
0
    dumpPart("8 (other pages, shared)", part8_);
1246
0
    dumpPart("9 (everything else)", part9_);
1247
0
}
1248
1249
void
1250
Lin::dumpPart(std::string_view label, std::vector<QPDFObjectHandle> const& part)
1251
0
{
1252
0
    auto& info = *cf.log()->getInfo();
1253
0
    info << "part " << std::string(label) << ":";
1254
0
    if (part.empty()) {
1255
0
        info << " none\n";
1256
0
        return;
1257
0
    }
1258
0
    size_t n = 0;
1259
0
    for (size_t i = 0; i < part.size(); ++i) {
1260
0
        int first = part[i].getObjectID();
1261
0
        int last = first;
1262
0
        while (i + 1 < part.size() && part[i + 1].getObjectID() == last + 1) {
1263
0
            ++i;
1264
0
            last = part[i].getObjectID();
1265
0
        }
1266
0
        info << (n % 10 == 0 ? "\n " : ", ");
1267
0
        ++n;
1268
0
        info << first;
1269
0
        if (last > first) {
1270
0
            info << "-" << last;
1271
0
        }
1272
0
    }
1273
0
    info << "\n";
1274
0
}
1275
1276
qpdf_offset_t
1277
Lin::adjusted_offset(qpdf_offset_t offset)
1278
0
{
1279
    // All offsets >= H_offset have to be increased by H_length since all hint table location values
1280
    // disregard the hint table itself.
1281
0
    if (offset >= linp_.H_offset) {
1282
0
        return offset + linp_.H_length;
1283
0
    }
1284
0
    return offset;
1285
0
}
1286
1287
void
1288
Lin::dumpHPageOffset()
1289
0
{
1290
0
    auto& info = *cf.log()->getInfo();
1291
0
    HPageOffset& t = page_offset_hints_;
1292
0
    info << "min_nobjects: " << t.min_nobjects << "\n"
1293
0
         << "first_page_offset: " << adjusted_offset(t.first_page_offset) << "\n"
1294
0
         << "nbits_delta_nobjects: " << t.nbits_delta_nobjects << "\n"
1295
0
         << "min_page_length: " << t.min_page_length << "\n"
1296
0
         << "nbits_delta_page_length: " << t.nbits_delta_page_length << "\n"
1297
0
         << "min_content_offset: " << t.min_content_offset << "\n"
1298
0
         << "nbits_delta_content_offset: " << t.nbits_delta_content_offset << "\n"
1299
0
         << "min_content_length: " << t.min_content_length << "\n"
1300
0
         << "nbits_delta_content_length: " << t.nbits_delta_content_length << "\n"
1301
0
         << "nbits_nshared_objects: " << t.nbits_nshared_objects << "\n"
1302
0
         << "nbits_shared_identifier: " << t.nbits_shared_identifier << "\n"
1303
0
         << "nbits_shared_numerator: " << t.nbits_shared_numerator << "\n"
1304
0
         << "shared_denominator: " << t.shared_denominator << "\n";
1305
1306
0
    for (size_t i1 = 0; i1 < linp_.npages; ++i1) {
1307
0
        HPageOffsetEntry& pe = t.entries.at(i1);
1308
0
        info << "Page " << i1 << ":\n"
1309
0
             << "  nobjects: " << pe.delta_nobjects + t.min_nobjects << "\n"
1310
0
             << "  length: " << pe.delta_page_length + t.min_page_length
1311
0
             << "\n"
1312
             // content offset is relative to page, not file
1313
0
             << "  content_offset: " << pe.delta_content_offset + t.min_content_offset << "\n"
1314
0
             << "  content_length: " << pe.delta_content_length + t.min_content_length << "\n"
1315
0
             << "  nshared_objects: " << pe.nshared_objects << "\n";
1316
0
        for (size_t i2 = 0; i2 < toS(pe.nshared_objects); ++i2) {
1317
0
            info << "    identifier " << i2 << ": " << pe.shared_identifiers.at(i2) << "\n";
1318
0
            info << "    numerator " << i2 << ": " << pe.shared_numerators.at(i2) << "\n";
1319
0
        }
1320
0
    }
1321
0
}
1322
1323
void
1324
Lin::dumpHSharedObject()
1325
0
{
1326
0
    auto& info = *cf.log()->getInfo();
1327
0
    HSharedObject& t = shared_object_hints_;
1328
0
    info << "first_shared_obj: " << t.first_shared_obj << "\n"
1329
0
         << "first_shared_offset: " << adjusted_offset(t.first_shared_offset) << "\n"
1330
0
         << "nshared_first_page: " << t.nshared_first_page << "\n"
1331
0
         << "nshared_total: " << t.nshared_total << "\n"
1332
0
         << "nbits_nobjects: " << t.nbits_nobjects << "\n"
1333
0
         << "min_group_length: " << t.min_group_length << "\n"
1334
0
         << "nbits_delta_group_length: " << t.nbits_delta_group_length << "\n";
1335
1336
0
    for (size_t i = 0; i < toS(t.nshared_total); ++i) {
1337
0
        HSharedObjectEntry& se = t.entries.at(i);
1338
0
        info << "Shared Object " << i << ":\n"
1339
0
             << "  group length: " << se.delta_group_length + t.min_group_length << "\n";
1340
        // PDF spec says signature present nobjects_minus_one are always 0, so print them only if
1341
        // they have a non-zero value.
1342
0
        if (se.signature_present) {
1343
0
            info << "  signature present\n";
1344
0
        }
1345
0
        if (se.nobjects_minus_one != 0) {
1346
0
            info << "  nobjects: " << se.nobjects_minus_one + 1 << "\n";
1347
0
        }
1348
0
    }
1349
0
}
1350
1351
void
1352
Lin::dumpHGeneric(HGeneric& t)
1353
0
{
1354
0
    *cf.log()->getInfo() << "first_object: " << t.first_object << "\n"
1355
0
                         << "first_object_offset: " << adjusted_offset(t.first_object_offset)
1356
0
                         << "\n"
1357
0
                         << "nobjects: " << t.nobjects << "\n"
1358
0
                         << "group_length: " << t.group_length << "\n";
1359
0
}
1360
1361
template <typename T>
1362
void
1363
Lin::calculateLinearizationData(T const& object_stream_data)
1364
0
{
1365
    // This function calculates the ordering of objects, divides them into the appropriate parts,
1366
    // and computes some values for the linearization parameter dictionary and hint tables.  The
1367
    // file must be prepared (via calling prepare()) prior to calling this function.  Note that
1368
    // actual offsets and lengths are not computed here, but anything related to object ordering is.
1369
1370
0
    util::assertion(
1371
0
        !obj_categories_.empty(),
1372
0
        "INTERNAL ERROR: QPDF::calculateLinearizationData called before prepare()" //
1373
0
    );
1374
    // Note that we can't call prepare here because we don't know whether it should be called
1375
    // with or without allow changes.
1376
1377
    // Separate objects into the categories sufficient for us to determine which part of the
1378
    // linearized file should contain the object.  This categorization is useful for other purposes
1379
    // as well.  Part numbers refer to version 1.4 of the PDF spec.
1380
1381
    // Parts 1, 3, 5, 10, and 11 don't contain any objects from the original file (except the
1382
    // trailer dictionary in part 11).
1383
1384
    // Part 4 is the document catalog (root) and the following root keys: /ViewerPreferences,
1385
    // /PageMode, /Threads, /OpenAction, /AcroForm, /Encrypt.  Note that Thread information
1386
    // dictionaries are supposed to appear in part 9, but we are disregarding that recommendation
1387
    // for now.
1388
1389
    // Part 6 is the first page section.  It includes all remaining objects referenced by the first
1390
    // page including shared objects but not including thumbnails.  Additionally, if /PageMode is
1391
    // /Outlines, then information from /Outlines also appears here.
1392
1393
    // Part 7 contains remaining objects private to pages other than the first page.
1394
1395
    // Part 8 contains all remaining shared objects except those that are shared only within
1396
    // thumbnails. "Shared objects" means objects referenced by more than one page.
1397
1398
    // Part 9 contains all remaining objects.
1399
1400
    // We sort objects into the following categories:
1401
    //   * root (the document catalog): part 4
1402
    //   * open_document: part 4
1403
    //   * first_page_shared: part 6
1404
    //   * first_page_private: part 6
1405
    //   * other_page_shared: part 8
1406
    //   * other_page_private: part 7
1407
    //   * thumbnail_shared: part 9
1408
    //   * thumbnail_private: part 9
1409
    //   * pages_obj (the pages tree): part 9
1410
    //   * outlines_obj (the outlines dictionary): part 6 or 9
1411
    //   * outlines: part 6 or 9
1412
    //   * other: part 9
1413
1414
    // Each part is further subdivided into regions. Create a separate vector for each subpart,
1415
    // then merge. Directly add the things that go to the beginning of the part in the final vector
1416
    // to avoid any extra moves.
1417
1418
0
    std::vector<QPDFObjectHandle> uc_pages;
1419
0
    std::vector<QPDFObjGen> uc_page_ogs;
1420
0
    std::vector<QPDFObjectHandle> uc_thumbs;
1421
0
    std::vector<QPDFObjGen> thumb_ogs;
1422
0
    { // local scope
1423
        // Map all page objects to the containing object stream.  This should be a no-op in a
1424
        // properly linearized file.
1425
0
        for (auto oh: pages) {
1426
0
            auto uc = getUncompressedObject(oh, object_stream_data);
1427
0
            uc_page_ogs.emplace_back(uc.getObjGen());
1428
0
            QPDFObjectHandle thumb = uc.getKey("/Thumb");
1429
0
            thumb = getUncompressedObject(thumb, object_stream_data);
1430
0
            thumb_ogs.emplace_back(thumb.getObjGen());
1431
0
            uc_pages.emplace_back(uc);
1432
0
            uc_thumbs.emplace_back(thumb);
1433
0
        }
1434
0
    }
1435
0
    size_t npages = pages.size();
1436
1437
0
    part4_.clear(); // root
1438
0
    std::vector<QPDFObjectHandle> part4a_open_document;
1439
0
    part6_.clear(); // first page
1440
0
    std::vector<QPDFObjectHandle> part6a_first_page_private;
1441
0
    std::vector<QPDFObjectHandle> part6b_first_page_shared;
1442
0
    part7_.clear(); // other page private
1443
0
    std::vector<std::vector<QPDFObjectHandle>> part7a_page_objects(npages);
1444
0
    std::vector<std::vector<QPDFObjectHandle>> part7b_other_page_private(npages);
1445
0
    part8_.clear(); // other page shared
1446
0
    part9_.clear(); // starts with pages objects
1447
0
    std::vector<std::vector<QPDFObjectHandle>> part9a_thumbnail_objects(npages);
1448
0
    std::vector<std::vector<QPDFObjectHandle>> part9b_thumbnail_private(npages);
1449
0
    std::vector<QPDFObjectHandle> part9c_thumbnail_shared;
1450
0
    std::vector<QPDFObjectHandle> part9d_other;
1451
0
    std::vector<QPDFObjectHandle> part_outlines_obj;
1452
0
    std::vector<QPDFObjectHandle> part_outlines;
1453
0
    c_linp_ = LinParameters();
1454
0
    c_page_offset_data_ = CHPageOffset();
1455
0
    c_shared_object_data_ = CHSharedObject();
1456
0
    c_outline_data_ = HGeneric();
1457
0
    outlines_max_end_ = 0;
1458
1459
0
    QPDFObjectHandle root = qpdf.getRoot();
1460
0
    bool outlines_in_first_page = false;
1461
0
    QPDFObjectHandle pagemode = root.getKey("/PageMode");
1462
0
    QTC::TC("qpdf", "QPDF categorize pagemode present", pagemode.isName() ? 1 : 0);
1463
0
    if (pagemode.isName()) {
1464
0
        if (pagemode.getName() == "/UseOutlines") {
1465
0
            if (root.hasKey("/Outlines")) {
1466
0
                outlines_in_first_page = true;
1467
0
            } else {
1468
0
                QTC::TC("qpdf", "QPDF UseOutlines but no Outlines");
1469
0
            }
1470
0
        }
1471
0
        QTC::TC("qpdf", "QPDF categorize pagemode outlines", outlines_in_first_page ? 1 : 0);
1472
0
    }
1473
1474
    // Generate ordering for objects in the output file.  Note that obj_categories_ is keyed by
1475
    // QPDFObjGen, so iterating it visits objects in object number order within each category.  In a
1476
    // linearized file, objects appear in sequence with the possible exception of hints tables which
1477
    // we won't see here anyway.  That means that running calculateLinearizationData() on a
1478
    // linearized file should give results identical to the original file ordering.
1479
1480
0
    ObjTable<int8_t> shared_objects;
1481
0
    for (auto& [og, category]: obj_categories_) {
1482
0
        auto oh = qpdf.getObject(og);
1483
0
        switch (category.category()) {
1484
0
        case c_root:
1485
0
            part4_.emplace_back(oh);
1486
0
            break;
1487
0
        case c_open_document:
1488
0
            part4a_open_document.emplace_back(oh);
1489
0
            break;
1490
0
        case c_first_page_shared:
1491
0
            shared_objects[og.getObj()] = true;
1492
0
            part6b_first_page_shared.emplace_back(oh);
1493
0
            break;
1494
0
        case c_first_page_private:
1495
0
            if (og == uc_page_ogs.at(0)) {
1496
0
                part6_.emplace_back(uc_pages.at(0));
1497
0
                c_linp_.first_page_object = uc_pages.at(0).getObjectID();
1498
0
            } else {
1499
0
                part6a_first_page_private.emplace_back(oh);
1500
0
            }
1501
0
            break;
1502
0
        case c_other_page_shared:
1503
0
            shared_objects[og.getObj()] = true;
1504
0
            part8_.emplace_back(oh);
1505
0
            break;
1506
0
        case c_other_page_private:
1507
0
            {
1508
0
                auto pageno = util::to<size_t>(category.pageno());
1509
0
                if (og == uc_page_ogs.at(pageno)) {
1510
0
                    part7a_page_objects.at(pageno).emplace_back(uc_pages.at(pageno));
1511
0
                } else {
1512
0
                    part7b_other_page_private.at(pageno).emplace_back(oh);
1513
0
                }
1514
0
            }
1515
0
            break;
1516
0
        case c_thumbnail_private:
1517
0
            {
1518
0
                auto pageno = util::to<size_t>(category.pageno());
1519
0
                if (og == thumb_ogs.at(pageno)) {
1520
0
                    part9a_thumbnail_objects.at(pageno).emplace_back(uc_thumbs.at(pageno));
1521
0
                } else {
1522
0
                    part9b_thumbnail_private.at(pageno).emplace_back(oh);
1523
0
                }
1524
0
            }
1525
0
            break;
1526
0
        case c_thumbnail_shared:
1527
0
            part9c_thumbnail_shared.emplace_back(oh);
1528
0
            break;
1529
0
        case c_outlines_obj:
1530
0
            part_outlines_obj.emplace_back(oh);
1531
0
            if constexpr (std::is_same_v<T, std::map<int, int>>) {
1532
                // Only used for linearization checks.
1533
0
                outlines_max_end_ = std::max(outlines_max_end_, objectEnd(og));
1534
0
            }
1535
0
            break;
1536
0
        case c_outlines:
1537
0
            part_outlines.emplace_back(oh);
1538
0
            if constexpr (std::is_same_v<T, std::map<int, int>>) {
1539
                // Only used for linearization checks.
1540
0
                outlines_max_end_ = std::max(outlines_max_end_, objectEnd(og));
1541
0
            }
1542
0
            break;
1543
0
        case c_pages_obj:
1544
0
            part9_.emplace_back(oh);
1545
0
            break;
1546
0
        case c_other:
1547
0
            part9d_other.emplace_back(oh);
1548
0
            break;
1549
0
        }
1550
0
    }
1551
1552
0
    if (!part_outlines_obj.empty()) {
1553
0
        c_outline_data_.first_object = part_outlines_obj.at(0).getObjectID();
1554
0
        c_outline_data_.nobjects = 1;
1555
0
    } else if (!part_outlines.empty()) {
1556
        // This case is probably not reachable. There can't be outlines without an outlines obj, and
1557
        // if any part of the outlines are swallowed up by a higher priority part, the entire
1558
        // structure will come along for the ride.
1559
0
        c_outline_data_.first_object = part_outlines.at(0).getObjectID();
1560
0
    }
1561
0
    c_outline_data_.nobjects += toI(part_outlines.size());
1562
1563
    // We will be initializing some values of the computed hint tables.  Specifically, we can
1564
    // initialize any items that deal with object numbers or counts but not any items that deal with
1565
    // lengths or offsets.  The code that writes linearized files will have to fill in these values
1566
    // during the first pass.  The validation code can compute them relatively easily given the rest
1567
    // of the information.
1568
1569
    // npages is the size of the existing pages vector, which has been created by traversing the
1570
    // pages tree, and as such is a reasonable size.
1571
0
    c_linp_.npages = npages;
1572
0
    c_page_offset_data_.entries = std::vector<CHPageOffsetEntry>(npages);
1573
1574
    // Part 4: open document objects.  We don't care about the order.
1575
1576
0
    no_ci_stop_if(
1577
0
        part4_.size() != 1, "found other than one root while calculating linearization data" //
1578
0
    );
1579
0
    append_vec(part4_, std::move(part4a_open_document));
1580
1581
    // Part 6: first page objects.  Note: implementation note 124 states that Acrobat always treats
1582
    // page 0 as the first page for linearization regardless of /OpenAction.  pdlin doesn't provide
1583
    // any option to set this and also disregards /OpenAction.  We will do the same.
1584
0
    no_ci_stop_if(
1585
0
        part6_.size() != 1, "part 6 has other than exactly one object (first page dictionary)" //
1586
0
    );
1587
    // The PDF spec "recommends" an order for the rest of the objects, but we are going to disregard
1588
    // it except to the extent that it groups private and shared objects contiguously for the sake
1589
    // of hint tables.
1590
0
    append_vec(part6_, std::move(part6a_first_page_private));
1591
0
    append_vec(part6_, std::move(part6b_first_page_shared));
1592
1593
    // Place the outline dictionary if it goes in the first page section.
1594
0
    if (outlines_in_first_page) {
1595
0
        append_vec(part6_, std::move(part_outlines_obj));
1596
0
        append_vec(part6_, std::move(part_outlines));
1597
0
    }
1598
1599
    // Fill in page offset hint table information for the first page. The PDF spec says that
1600
    // nshared_objects should be zero for the first page.  pdlin does not appear to obey this, but
1601
    // it fills in garbage values for all the shared object identifiers on the first page.
1602
1603
0
    c_page_offset_data_.entries.at(0).nobjects = toI(part6_.size());
1604
1605
    // Part 7: other pages' private objects
1606
1607
    // For each page in order:
1608
0
    for (size_t i = 1; i < npages; ++i) {
1609
        // Place this page's page object
1610
0
        no_ci_stop_if(
1611
0
            part7a_page_objects.at(i).size() != 1,
1612
0
            "unable to linearize page " + std::to_string(i) //
1613
0
        );
1614
0
        append_vec(part7_, std::move(part7a_page_objects.at(i)));
1615
1616
        // Place all non-shared objects referenced by this page, updating the page object count for
1617
        // the hint table.
1618
0
        c_page_offset_data_.entries.at(i).nobjects =
1619
0
            util::to<int>(1 + part7b_other_page_private.at(i).size());
1620
0
        append_vec(part7_, std::move(part7b_other_page_private.at(i)));
1621
0
    }
1622
1623
    // Part 8: other pages' shared objects -- these were already appended
1624
1625
    // Part 9: other objects
1626
1627
    // The PDF specification makes recommendations on ordering here. We follow them only to a
1628
    // limited extent.  Specifically, we put the pages tree first, then private thumbnail objects in
1629
    // page order, then shared thumbnail objects, and then outlines (unless in part 6).  After that,
1630
    // we throw all remaining objects in arbitrary order.
1631
1632
    // Place private thumbnail images in page order.  Slightly more information would be required if
1633
    // we were going to bother with thumbnail hint tables.
1634
0
    for (size_t i = 0; i < npages; ++i) {
1635
        // Output the thumbnail itself
1636
0
        append_vec(part9_, std::move(part9a_thumbnail_objects.at(i)));
1637
0
        append_vec(part9_, std::move(part9b_thumbnail_private.at(i)));
1638
0
    }
1639
1640
    // Place shared thumbnail objects
1641
0
    append_vec(part9_, std::move(part9c_thumbnail_shared));
1642
1643
    // Place outlines unless in first page
1644
0
    if (!outlines_in_first_page) {
1645
0
        append_vec(part9_, std::move(part_outlines_obj));
1646
0
        append_vec(part9_, std::move(part_outlines));
1647
0
    }
1648
1649
    // Place all remaining objects
1650
0
    append_vec(part9_, std::move(part9d_other));
1651
1652
    // Make sure we got everything exactly once.
1653
0
    size_t num_placed =
1654
0
        part4_.size() + part6_.size() + part7_.size() + part8_.size() + part9_.size();
1655
0
    size_t num_wanted = obj_categories_.size();
1656
0
    no_ci_stop_if(
1657
        // This can happen with damaged files, e.g. if the root is part of the the pages tree.
1658
0
        num_placed != num_wanted,
1659
0
        "QPDF::calculateLinearizationData: wrong number of objects placed (num_placed = " +
1660
0
            std::to_string(num_placed) + "; number of objects: " + std::to_string(num_wanted) +
1661
0
            "\nIf the file did not generate any other warnings please report this as a bug." //
1662
0
    );
1663
1664
    // Calculate shared object hint table information including references to shared objects from
1665
    // page offset hint data.
1666
1667
    // The shared object hint table consists of all part 6 (whether shared or not) in order followed
1668
    // by all part 8 objects in order.  Add the objects to shared object data keeping a map of
1669
    // object number to index.  Then populate the shared object information for the pages.
1670
1671
    // Note that two objects never have the same object number, so we can map from object number
1672
    // only without regards to generation.
1673
0
    ObjTable<int> obj_to_index;
1674
1675
0
    c_shared_object_data_.nshared_first_page = toI(part6_.size());
1676
0
    c_shared_object_data_.nshared_total =
1677
0
        c_shared_object_data_.nshared_first_page + toI(part8_.size());
1678
1679
0
    std::vector<CHSharedObjectEntry>& shared = c_shared_object_data_.entries;
1680
0
    for (auto& oh: part6_) {
1681
0
        int obj = oh.getObjectID();
1682
0
        obj_to_index[obj] = toI(shared.size());
1683
0
        shared.emplace_back(obj);
1684
0
    }
1685
0
    QTC::TC("qpdf", "QPDF lin part 8 empty", part8_.empty() ? 1 : 0);
1686
0
    if (!part8_.empty()) {
1687
0
        c_shared_object_data_.first_shared_obj = part8_.at(0).getObjectID();
1688
0
        for (auto& oh: part8_) {
1689
0
            int obj = oh.getObjectID();
1690
0
            obj_to_index[obj] = toI(shared.size());
1691
0
            shared.emplace_back(obj);
1692
0
        }
1693
0
    }
1694
0
    no_ci_stop_if(
1695
0
        std::cmp_not_equal(
1696
0
            c_shared_object_data_.nshared_total, c_shared_object_data_.entries.size()),
1697
0
        "shared object hint table has wrong number of entries" //
1698
0
    );
1699
1700
    // Now compute the list of shared objects for each page after the first page.
1701
0
    for (size_t i = 1; i < npages; ++i) {
1702
0
        CHPageOffsetEntry& pe = c_page_offset_data_.entries.at(i);
1703
0
        for (auto obj: page_objs_.at(i)) {
1704
0
            if (shared_objects[obj]) {
1705
0
                ++pe.nshared_objects;
1706
0
                pe.shared_identifiers.push_back(obj_to_index[obj]);
1707
0
            }
1708
0
        }
1709
0
    }
1710
0
}
Unexecuted instantiation: void QPDF::Doc::Linearization::calculateLinearizationData<std::__1::map<int, int, std::__1::less<int>, std::__1::allocator<std::__1::pair<int const, int> > > >(std::__1::map<int, int, std::__1::less<int>, std::__1::allocator<std::__1::pair<int const, int> > > const&)
Unexecuted instantiation: void QPDF::Doc::Linearization::calculateLinearizationData<QPDFWriter::ObjTable>(QPDFWriter::ObjTable const&)
1711
1712
void
1713
Lin::parts(
1714
    QPDFWriter::ObjTable const& obj,
1715
    std::function<int(QPDFObjectHandle&)> skip_stream_parameters,
1716
    std::vector<QPDFObjectHandle>& part4,
1717
    std::vector<QPDFObjectHandle>& part6,
1718
    std::vector<QPDFObjectHandle>& part7,
1719
    std::vector<QPDFObjectHandle>& part8,
1720
    std::vector<QPDFObjectHandle>& part9)
1721
0
{
1722
0
    prepare(obj, true, skip_stream_parameters);
1723
0
    calculateLinearizationData(obj);
1724
0
    part4 = part4_;
1725
0
    part6 = part6_;
1726
0
    part7 = part7_;
1727
0
    part8 = part8_;
1728
0
    part9 = part9_;
1729
0
}
1730
1731
static inline int
1732
nbits(int val)
1733
0
{
1734
0
    return (val == 0 ? 0 : (1 + nbits(val >> 1)));
1735
0
}
1736
1737
int
1738
Lin::outputLengthNextN(
1739
    int in_object, int n, QPDFWriter::NewObjTable const& new_obj, QPDFWriter::ObjTable const& obj)
1740
0
{
1741
    // Figure out the length of a series of n consecutive objects in the output file starting with
1742
    // whatever object in_object from the input file mapped to.
1743
1744
0
    int first = obj[in_object].renumber;
1745
0
    int last = first + n;
1746
0
    no_ci_stop_if(
1747
0
        first <= 0, "found object that is not renumbered while writing linearization data");
1748
0
    qpdf_offset_t length = 0;
1749
0
    for (int i = first; i < last; ++i) {
1750
0
        auto l = new_obj[i].length;
1751
0
        no_ci_stop_if(
1752
0
            l == 0, "found item with unknown length while writing linearization data" //
1753
0
        );
1754
0
        length += l;
1755
0
    }
1756
0
    return toI(length);
1757
0
}
1758
1759
void
1760
Lin::calculateHPageOffset(QPDFWriter::NewObjTable const& new_obj, QPDFWriter::ObjTable const& obj)
1761
0
{
1762
    // Page Offset Hint Table
1763
1764
    // We are purposely leaving some values set to their initial zero values.
1765
1766
0
    auto const& all_pages = pages.all();
1767
0
    size_t npages = all_pages.size();
1768
0
    CHPageOffset& cph = c_page_offset_data_;
1769
0
    std::vector<CHPageOffsetEntry>& cphe = cph.entries;
1770
1771
    // Calculate minimum and maximum values for number of objects per page and page length.
1772
1773
0
    int min_nobjects = std::numeric_limits<int>::max();
1774
0
    int max_nobjects = 0;
1775
0
    int min_length = std::numeric_limits<int>::max();
1776
0
    int max_length = 0;
1777
0
    int max_shared = 0;
1778
1779
0
    HPageOffset& ph = page_offset_hints_;
1780
0
    std::vector<HPageOffsetEntry>& phe = ph.entries;
1781
    // npages is the size of the existing pages array.
1782
0
    phe = std::vector<HPageOffsetEntry>(npages);
1783
1784
0
    size_t i = 0;
1785
0
    for (auto& phe_i: phe) {
1786
        // Calculate values for each page, assigning full values to the delta items.  They will be
1787
        // adjusted later.
1788
1789
        // Repeat calculations for page 0 so we can assign to phe[i] without duplicating those
1790
        // assignments.
1791
1792
0
        int nobjects = cphe.at(i).nobjects;
1793
0
        int length = outputLengthNextN(all_pages.at(i).getObjectID(), nobjects, new_obj, obj);
1794
0
        int nshared = cphe.at(i).nshared_objects;
1795
1796
0
        min_nobjects = std::min(min_nobjects, nobjects);
1797
0
        max_nobjects = std::max(max_nobjects, nobjects);
1798
0
        min_length = std::min(min_length, length);
1799
0
        max_length = std::max(max_length, length);
1800
0
        max_shared = std::max(max_shared, nshared);
1801
1802
0
        phe_i.delta_nobjects = nobjects;
1803
0
        phe_i.delta_page_length = length;
1804
0
        phe_i.nshared_objects = nshared;
1805
0
        ++i;
1806
0
    }
1807
1808
0
    ph.min_nobjects = min_nobjects;
1809
0
    ph.first_page_offset = new_obj[obj[all_pages.at(0)].renumber].xref.getOffset();
1810
0
    ph.nbits_delta_nobjects = nbits(max_nobjects - min_nobjects);
1811
0
    ph.min_page_length = min_length;
1812
0
    ph.nbits_delta_page_length = nbits(max_length - min_length);
1813
0
    ph.nbits_nshared_objects = nbits(max_shared);
1814
0
    ph.nbits_shared_identifier = nbits(c_shared_object_data_.nshared_total);
1815
0
    ph.shared_denominator = 4; // doesn't matter
1816
1817
    // It isn't clear how to compute content offset and content length.  Since we are not
1818
    // interleaving page objects with the content stream, we'll use the same values for content
1819
    // length as page length.  We will use 0 as content offset because this is what Adobe does
1820
    // (implementation note 127) and pdlin as well.
1821
0
    ph.nbits_delta_content_length = ph.nbits_delta_page_length;
1822
0
    ph.min_content_length = ph.min_page_length;
1823
1824
0
    i = 0;
1825
0
    for (auto& phe_i: phe) {
1826
        // Adjust delta entries
1827
0
        if (phe_i.delta_nobjects < min_nobjects || phe_i.delta_page_length < min_length) {
1828
0
            stopOnError(
1829
0
                "found too small delta nobjects or delta page length while writing "
1830
0
                "linearization data");
1831
0
        }
1832
0
        phe_i.delta_nobjects -= min_nobjects;
1833
0
        phe_i.delta_page_length -= min_length;
1834
0
        phe_i.delta_content_length = phe_i.delta_page_length;
1835
1836
0
        auto& si = cphe.at(i).shared_identifiers;
1837
0
        phe_i.shared_identifiers.insert(phe_i.shared_identifiers.end(), si.begin(), si.end());
1838
0
        phe_i.shared_numerators.insert(phe_i.shared_numerators.end(), si.size(), 0);
1839
0
        ++i;
1840
0
    }
1841
0
}
1842
1843
void
1844
Lin::calculateHSharedObject(QPDFWriter::NewObjTable const& new_obj, QPDFWriter::ObjTable const& obj)
1845
0
{
1846
0
    CHSharedObject& cso = c_shared_object_data_;
1847
0
    std::vector<CHSharedObjectEntry>& csoe = cso.entries;
1848
0
    HSharedObject& so = shared_object_hints_;
1849
0
    std::vector<HSharedObjectEntry>& soe = so.entries;
1850
0
    soe.clear();
1851
1852
0
    int min_length = outputLengthNextN(csoe.at(0).object, 1, new_obj, obj);
1853
0
    int max_length = min_length;
1854
1855
0
    for (size_t i = 0; i < toS(cso.nshared_total); ++i) {
1856
        // Assign absolute numbers to deltas; adjust later
1857
0
        int length = outputLengthNextN(csoe.at(i).object, 1, new_obj, obj);
1858
0
        min_length = std::min(min_length, length);
1859
0
        max_length = std::max(max_length, length);
1860
0
        soe.emplace_back();
1861
0
        soe.at(i).delta_group_length = length;
1862
0
    }
1863
0
    no_ci_stop_if(
1864
0
        soe.size() != toS(cso.nshared_total), "soe has wrong size after initialization" //
1865
0
    );
1866
1867
0
    so.nshared_total = cso.nshared_total;
1868
0
    so.nshared_first_page = cso.nshared_first_page;
1869
0
    if (so.nshared_total > so.nshared_first_page) {
1870
0
        so.first_shared_obj = obj[cso.first_shared_obj].renumber;
1871
0
        so.min_group_length = min_length;
1872
0
        so.first_shared_offset = new_obj[so.first_shared_obj].xref.getOffset();
1873
0
    }
1874
0
    so.min_group_length = min_length;
1875
0
    so.nbits_delta_group_length = nbits(max_length - min_length);
1876
1877
0
    for (size_t i = 0; i < toS(cso.nshared_total); ++i) {
1878
        // Adjust deltas
1879
0
        no_ci_stop_if(
1880
0
            soe.at(i).delta_group_length < min_length,
1881
0
            "found too small group length while writing linearization data" //
1882
0
        );
1883
1884
0
        soe.at(i).delta_group_length -= min_length;
1885
0
    }
1886
0
}
1887
1888
void
1889
Lin::calculateHOutline(QPDFWriter::NewObjTable const& new_obj, QPDFWriter::ObjTable const& obj)
1890
0
{
1891
0
    HGeneric& cho = c_outline_data_;
1892
1893
0
    if (cho.nobjects == 0) {
1894
0
        return;
1895
0
    }
1896
1897
0
    HGeneric& ho = outline_hints_;
1898
1899
0
    ho.first_object = obj[cho.first_object].renumber;
1900
0
    ho.first_object_offset = new_obj[ho.first_object].xref.getOffset();
1901
0
    ho.nobjects = cho.nobjects;
1902
0
    ho.group_length = outputLengthNextN(cho.first_object, ho.nobjects, new_obj, obj);
1903
0
}
1904
1905
template <class T, class int_type>
1906
static void
1907
write_vector_int(BitWriter& w, int nitems, std::vector<T>& vec, int bits, int_type T::* field)
1908
0
{
1909
    // nitems times, write bits bits from the given field of the ith vector to the given bit writer.
1910
1911
0
    for (size_t i = 0; i < QIntC::to_size(nitems); ++i) {
1912
0
        w.writeBits(QIntC::to_ulonglong(vec.at(i).*field), QIntC::to_size(bits));
1913
0
    }
1914
    // The PDF spec says that each hint table starts at a byte boundary.  Each "row" actually must
1915
    // start on a byte boundary.
1916
0
    w.flush();
1917
0
}
Unexecuted instantiation: QPDF_linearization.cc:void write_vector_int<QPDF::Doc::Linearization::HPageOffsetEntry, int>(BitWriter&, int, std::__1::vector<QPDF::Doc::Linearization::HPageOffsetEntry, std::__1::allocator<QPDF::Doc::Linearization::HPageOffsetEntry> >&, int, int QPDF::Doc::Linearization::HPageOffsetEntry::*)
Unexecuted instantiation: QPDF_linearization.cc:void write_vector_int<QPDF::Doc::Linearization::HPageOffsetEntry, long long>(BitWriter&, int, std::__1::vector<QPDF::Doc::Linearization::HPageOffsetEntry, std::__1::allocator<QPDF::Doc::Linearization::HPageOffsetEntry> >&, int, long long QPDF::Doc::Linearization::HPageOffsetEntry::*)
Unexecuted instantiation: QPDF_linearization.cc:void write_vector_int<QPDF::Doc::Linearization::HSharedObjectEntry, int>(BitWriter&, int, std::__1::vector<QPDF::Doc::Linearization::HSharedObjectEntry, std::__1::allocator<QPDF::Doc::Linearization::HSharedObjectEntry> >&, int, int QPDF::Doc::Linearization::HSharedObjectEntry::*)
1918
1919
template <class T>
1920
static void
1921
write_vector_vector(
1922
    BitWriter& w,
1923
    int nitems1,
1924
    std::vector<T>& vec1,
1925
    int T::* nitems2,
1926
    int bits,
1927
    std::vector<int> T::* vec2)
1928
0
{
1929
    // nitems1 times, write nitems2 (from the ith element of vec1) items from the vec2 vector field
1930
    // of the ith item of vec1.
1931
0
    for (size_t i1 = 0; i1 < QIntC::to_size(nitems1); ++i1) {
1932
0
        for (size_t i2 = 0; i2 < QIntC::to_size(vec1.at(i1).*nitems2); ++i2) {
1933
0
            w.writeBits(QIntC::to_ulonglong((vec1.at(i1).*vec2).at(i2)), QIntC::to_size(bits));
1934
0
        }
1935
0
    }
1936
0
    w.flush();
1937
0
}
1938
1939
void
1940
Lin::writeHPageOffset(BitWriter& w)
1941
0
{
1942
0
    HPageOffset& t = page_offset_hints_;
1943
1944
0
    w.writeBitsInt(t.min_nobjects, 32);               // 1
1945
0
    w.writeBits(toULL(t.first_page_offset), 32);      // 2
1946
0
    w.writeBitsInt(t.nbits_delta_nobjects, 16);       // 3
1947
0
    w.writeBitsInt(t.min_page_length, 32);            // 4
1948
0
    w.writeBitsInt(t.nbits_delta_page_length, 16);    // 5
1949
0
    w.writeBits(toULL(t.min_content_offset), 32);     // 6
1950
0
    w.writeBitsInt(t.nbits_delta_content_offset, 16); // 7
1951
0
    w.writeBitsInt(t.min_content_length, 32);         // 8
1952
0
    w.writeBitsInt(t.nbits_delta_content_length, 16); // 9
1953
0
    w.writeBitsInt(t.nbits_nshared_objects, 16);      // 10
1954
0
    w.writeBitsInt(t.nbits_shared_identifier, 16);    // 11
1955
0
    w.writeBitsInt(t.nbits_shared_numerator, 16);     // 12
1956
0
    w.writeBitsInt(t.shared_denominator, 16);         // 13
1957
1958
0
    int nitems = toI(pages.size());
1959
0
    std::vector<HPageOffsetEntry>& entries = t.entries;
1960
1961
0
    write_vector_int(w, nitems, entries, t.nbits_delta_nobjects, &HPageOffsetEntry::delta_nobjects);
1962
0
    write_vector_int(
1963
0
        w, nitems, entries, t.nbits_delta_page_length, &HPageOffsetEntry::delta_page_length);
1964
0
    write_vector_int(
1965
0
        w, nitems, entries, t.nbits_nshared_objects, &HPageOffsetEntry::nshared_objects);
1966
0
    write_vector_vector(
1967
0
        w,
1968
0
        nitems,
1969
0
        entries,
1970
0
        &HPageOffsetEntry::nshared_objects,
1971
0
        t.nbits_shared_identifier,
1972
0
        &HPageOffsetEntry::shared_identifiers);
1973
0
    write_vector_vector(
1974
0
        w,
1975
0
        nitems,
1976
0
        entries,
1977
0
        &HPageOffsetEntry::nshared_objects,
1978
0
        t.nbits_shared_numerator,
1979
0
        &HPageOffsetEntry::shared_numerators);
1980
0
    write_vector_int(
1981
0
        w, nitems, entries, t.nbits_delta_content_offset, &HPageOffsetEntry::delta_content_offset);
1982
0
    write_vector_int(
1983
0
        w, nitems, entries, t.nbits_delta_content_length, &HPageOffsetEntry::delta_content_length);
1984
0
}
1985
1986
void
1987
Lin::writeHSharedObject(BitWriter& w)
1988
0
{
1989
0
    HSharedObject& t = shared_object_hints_;
1990
1991
0
    w.writeBitsInt(t.first_shared_obj, 32);         // 1
1992
0
    w.writeBits(toULL(t.first_shared_offset), 32);  // 2
1993
0
    w.writeBitsInt(t.nshared_first_page, 32);       // 3
1994
0
    w.writeBitsInt(t.nshared_total, 32);            // 4
1995
0
    w.writeBitsInt(t.nbits_nobjects, 16);           // 5
1996
0
    w.writeBitsInt(t.min_group_length, 32);         // 6
1997
0
    w.writeBitsInt(t.nbits_delta_group_length, 16); // 7
1998
1999
0
    QTC::TC(
2000
0
        "qpdf",
2001
0
        "QPDF lin write nshared_total > nshared_first_page",
2002
0
        (t.nshared_total > t.nshared_first_page) ? 1 : 0);
2003
2004
0
    int nitems = t.nshared_total;
2005
0
    std::vector<HSharedObjectEntry>& entries = t.entries;
2006
2007
0
    write_vector_int(
2008
0
        w, nitems, entries, t.nbits_delta_group_length, &HSharedObjectEntry::delta_group_length);
2009
0
    write_vector_int(w, nitems, entries, 1, &HSharedObjectEntry::signature_present);
2010
0
    for (size_t i = 0; i < toS(nitems); ++i) {
2011
        // If signature were present, we'd have to write a 128-bit hash.
2012
0
        if (entries.at(i).signature_present != 0) {
2013
0
            stopOnError("found unexpected signature present while writing linearization data");
2014
0
        }
2015
0
    }
2016
0
    write_vector_int(w, nitems, entries, t.nbits_nobjects, &HSharedObjectEntry::nobjects_minus_one);
2017
0
}
2018
2019
void
2020
Lin::writeHGeneric(BitWriter& w, HGeneric& t)
2021
0
{
2022
0
    w.writeBitsInt(t.first_object, 32);            // 1
2023
0
    w.writeBits(toULL(t.first_object_offset), 32); // 2
2024
0
    w.writeBitsInt(t.nobjects, 32);                // 3
2025
0
    w.writeBitsInt(t.group_length, 32);            // 4
2026
0
}
2027
2028
void
2029
Lin::generateHintStream(
2030
    QPDFWriter::NewObjTable const& new_obj,
2031
    QPDFWriter::ObjTable const& obj,
2032
    std::string& hint_buffer,
2033
    int& S,
2034
    int& O,
2035
    bool compressed)
2036
0
{
2037
    // Populate actual hint table values
2038
0
    calculateHPageOffset(new_obj, obj);
2039
0
    calculateHSharedObject(new_obj, obj);
2040
0
    calculateHOutline(new_obj, obj);
2041
2042
    // Write the hint stream itself into a compressed memory buffer. Write through a counter so we
2043
    // can get offsets.
2044
0
    pl::Count c(0, hint_buffer);
2045
0
    BitWriter w(&c);
2046
2047
0
    writeHPageOffset(w);
2048
0
    S = toI(c.getCount());
2049
0
    writeHSharedObject(w);
2050
0
    O = 0;
2051
0
    if (outline_hints_.nobjects > 0) {
2052
0
        O = toI(c.getCount());
2053
0
        writeHGeneric(w, outline_hints_);
2054
0
    }
2055
0
    if (compressed) {
2056
0
        hint_buffer = pl::pipe<Pl_Flate>(hint_buffer, Pl_Flate::a_deflate);
2057
0
    }
2058
0
}