/src/libjxl/lib/jpegli/memory_manager.cc
Line | Count | Source (jump to first uncovered line) |
1 | | // Copyright (c) the JPEG XL Project Authors. All rights reserved. |
2 | | // |
3 | | // Use of this source code is governed by a BSD-style |
4 | | // license that can be found in the LICENSE file. |
5 | | |
6 | | #include "lib/jpegli/memory_manager.h" |
7 | | |
8 | | #include <string.h> |
9 | | |
10 | | #include <hwy/aligned_allocator.h> |
11 | | #include <vector> |
12 | | |
13 | | #include "lib/jpegli/common_internal.h" |
14 | | #include "lib/jpegli/error.h" |
15 | | |
16 | | struct jvirt_sarray_control { |
17 | | JSAMPARRAY full_buffer; |
18 | | size_t numrows; |
19 | | JDIMENSION maxaccess; |
20 | | }; |
21 | | |
22 | | struct jvirt_barray_control { |
23 | | JBLOCKARRAY full_buffer; |
24 | | size_t numrows; |
25 | | JDIMENSION maxaccess; |
26 | | }; |
27 | | |
28 | | namespace jpegli { |
29 | | |
30 | | namespace { |
31 | | |
32 | | struct MemoryManager { |
33 | | struct jpeg_memory_mgr pub; |
34 | | std::vector<void*> owned_ptrs[2 * JPOOL_NUMPOOLS]; |
35 | | uint64_t pool_memory_usage[2 * JPOOL_NUMPOOLS]; |
36 | | uint64_t total_memory_usage; |
37 | | uint64_t peak_memory_usage; |
38 | | }; |
39 | | |
40 | 90.1k | void* Alloc(j_common_ptr cinfo, int pool_id, size_t sizeofobject) { |
41 | 90.1k | MemoryManager* mem = reinterpret_cast<MemoryManager*>(cinfo->mem); |
42 | 90.1k | if (pool_id < 0 || pool_id >= 2 * JPOOL_NUMPOOLS) { |
43 | 0 | JPEGLI_ERROR("Invalid pool id %d", pool_id); |
44 | 0 | } |
45 | 90.1k | if (mem->pub.max_memory_to_use > 0 && |
46 | 90.1k | mem->total_memory_usage + static_cast<uint64_t>(sizeofobject) > |
47 | 0 | static_cast<uint64_t>(mem->pub.max_memory_to_use)) { |
48 | 0 | JPEGLI_ERROR("Total memory usage exceeding %ld", |
49 | 0 | mem->pub.max_memory_to_use); |
50 | 0 | } |
51 | 90.1k | void* p; |
52 | 90.1k | if (pool_id < JPOOL_NUMPOOLS) { |
53 | 49.1k | p = malloc(sizeofobject); |
54 | 49.1k | } else { |
55 | 41.0k | p = hwy::AllocateAlignedBytes(sizeofobject, nullptr, nullptr); |
56 | 41.0k | } |
57 | 90.1k | if (p == nullptr) { |
58 | 0 | JPEGLI_ERROR("Out of memory"); |
59 | 0 | } |
60 | 90.1k | mem->owned_ptrs[pool_id].push_back(p); |
61 | 90.1k | mem->pool_memory_usage[pool_id] += sizeofobject; |
62 | 90.1k | mem->total_memory_usage += sizeofobject; |
63 | 90.1k | mem->peak_memory_usage = |
64 | 90.1k | std::max(mem->peak_memory_usage, mem->total_memory_usage); |
65 | 90.1k | return p; |
66 | 90.1k | } |
67 | | |
68 | 12.0k | constexpr size_t gcd(size_t a, size_t b) { return b == 0 ? a : gcd(b, a % b); } |
69 | 6.04k | constexpr size_t lcm(size_t a, size_t b) { return (a * b) / gcd(a, b); } |
70 | | |
71 | | template <typename T> |
72 | | T** Alloc2dArray(j_common_ptr cinfo, int pool_id, JDIMENSION samplesperrow, |
73 | 6.04k | JDIMENSION numrows) { |
74 | 6.04k | T** array = Allocate<T*>(cinfo, numrows, pool_id); |
75 | | // Always use aligned allocator for large 2d arrays. |
76 | 6.04k | if (pool_id < JPOOL_NUMPOOLS) { |
77 | 6.04k | pool_id += JPOOL_NUMPOOLS; |
78 | 6.04k | } |
79 | 6.04k | size_t alignment = lcm(sizeof(T), HWY_ALIGNMENT); |
80 | 6.04k | size_t memstride = RoundUpTo(samplesperrow * sizeof(T), alignment); |
81 | 6.04k | size_t stride = memstride / sizeof(T); |
82 | 6.04k | T* buffer = Allocate<T>(cinfo, numrows * stride, pool_id); |
83 | 2.79M | for (size_t i = 0; i < numrows; ++i) { |
84 | 2.79M | array[i] = &buffer[i * stride]; |
85 | 2.79M | } |
86 | 6.04k | return array; |
87 | 6.04k | } Unexecuted instantiation: memory_manager.cc:unsigned char** jpegli::(anonymous namespace)::Alloc2dArray<unsigned char>(jpeg_common_struct*, int, unsigned int, unsigned int) memory_manager.cc:short (**jpegli::(anonymous namespace)::Alloc2dArray<short [64]>(jpeg_common_struct*, int, unsigned int, unsigned int)) [64] Line | Count | Source | 73 | 6.04k | JDIMENSION numrows) { | 74 | 6.04k | T** array = Allocate<T*>(cinfo, numrows, pool_id); | 75 | | // Always use aligned allocator for large 2d arrays. | 76 | 6.04k | if (pool_id < JPOOL_NUMPOOLS) { | 77 | 6.04k | pool_id += JPOOL_NUMPOOLS; | 78 | 6.04k | } | 79 | 6.04k | size_t alignment = lcm(sizeof(T), HWY_ALIGNMENT); | 80 | 6.04k | size_t memstride = RoundUpTo(samplesperrow * sizeof(T), alignment); | 81 | 6.04k | size_t stride = memstride / sizeof(T); | 82 | 6.04k | T* buffer = Allocate<T>(cinfo, numrows * stride, pool_id); | 83 | 2.79M | for (size_t i = 0; i < numrows; ++i) { | 84 | 2.79M | array[i] = &buffer[i * stride]; | 85 | 2.79M | } | 86 | 6.04k | return array; | 87 | 6.04k | } |
|
88 | | |
89 | | template <typename Control, typename T> |
90 | | Control* RequestVirtualArray(j_common_ptr cinfo, int pool_id, boolean pre_zero, |
91 | | JDIMENSION samplesperrow, JDIMENSION numrows, |
92 | 6.04k | JDIMENSION maxaccess) { |
93 | 6.04k | if (pool_id != JPOOL_IMAGE) { |
94 | 0 | JPEGLI_ERROR("Only image lifetime virtual arrays are supported."); |
95 | 0 | } |
96 | 6.04k | Control* p = Allocate<Control>(cinfo, 1, pool_id); |
97 | 6.04k | p->full_buffer = Alloc2dArray<T>(cinfo, pool_id, samplesperrow, numrows); |
98 | 6.04k | p->numrows = numrows; |
99 | 6.04k | p->maxaccess = maxaccess; |
100 | 6.04k | if (pre_zero) { |
101 | 2.79M | for (size_t i = 0; i < numrows; ++i) { |
102 | 2.79M | memset(p->full_buffer[i], 0, samplesperrow * sizeof(T)); |
103 | 2.79M | } |
104 | 6.04k | } |
105 | 6.04k | return p; |
106 | 6.04k | } Unexecuted instantiation: memory_manager.cc:jvirt_sarray_control* jpegli::(anonymous namespace)::RequestVirtualArray<jvirt_sarray_control, unsigned char>(jpeg_common_struct*, int, int, unsigned int, unsigned int, unsigned int) memory_manager.cc:jvirt_barray_control* jpegli::(anonymous namespace)::RequestVirtualArray<jvirt_barray_control, short [64]>(jpeg_common_struct*, int, int, unsigned int, unsigned int, unsigned int) Line | Count | Source | 92 | 6.04k | JDIMENSION maxaccess) { | 93 | 6.04k | if (pool_id != JPOOL_IMAGE) { | 94 | 0 | JPEGLI_ERROR("Only image lifetime virtual arrays are supported."); | 95 | 0 | } | 96 | 6.04k | Control* p = Allocate<Control>(cinfo, 1, pool_id); | 97 | 6.04k | p->full_buffer = Alloc2dArray<T>(cinfo, pool_id, samplesperrow, numrows); | 98 | 6.04k | p->numrows = numrows; | 99 | 6.04k | p->maxaccess = maxaccess; | 100 | 6.04k | if (pre_zero) { | 101 | 2.79M | for (size_t i = 0; i < numrows; ++i) { | 102 | 2.79M | memset(p->full_buffer[i], 0, samplesperrow * sizeof(T)); | 103 | 2.79M | } | 104 | 6.04k | } | 105 | 6.04k | return p; | 106 | 6.04k | } |
|
107 | | |
108 | 3.16k | void RealizeVirtualArrays(j_common_ptr cinfo) { |
109 | | // Nothing to do, the full arrays were realized at request time already. |
110 | 3.16k | } |
111 | | |
112 | | template <typename Control, typename T> |
113 | | T** AccessVirtualArray(j_common_ptr cinfo, Control* ptr, JDIMENSION start_row, |
114 | 1.85M | JDIMENSION num_rows, boolean writable) { |
115 | 1.85M | if (num_rows > ptr->maxaccess) { |
116 | 0 | JPEGLI_ERROR("Invalid virtual array access, num rows %u vs max rows %u", |
117 | 0 | num_rows, ptr->maxaccess); |
118 | 0 | } |
119 | 1.85M | if (start_row + num_rows > ptr->numrows) { |
120 | 0 | JPEGLI_ERROR("Invalid virtual array access, %u vs %u total rows", |
121 | 0 | start_row + num_rows, ptr->numrows); |
122 | 0 | } |
123 | 1.85M | if (ptr->full_buffer == nullptr) { |
124 | 0 | JPEGLI_ERROR("Invalid virtual array access, array not realized."); |
125 | 0 | } |
126 | 1.85M | return ptr->full_buffer + start_row; |
127 | 1.85M | } Unexecuted instantiation: memory_manager.cc:unsigned char** jpegli::(anonymous namespace)::AccessVirtualArray<jvirt_sarray_control, unsigned char>(jpeg_common_struct*, jvirt_sarray_control*, unsigned int, unsigned int, int) memory_manager.cc:short (**jpegli::(anonymous namespace)::AccessVirtualArray<jvirt_barray_control, short [64]>(jpeg_common_struct*, jvirt_barray_control*, unsigned int, unsigned int, int)) [64] Line | Count | Source | 114 | 1.85M | JDIMENSION num_rows, boolean writable) { | 115 | 1.85M | if (num_rows > ptr->maxaccess) { | 116 | 0 | JPEGLI_ERROR("Invalid virtual array access, num rows %u vs max rows %u", | 117 | 0 | num_rows, ptr->maxaccess); | 118 | 0 | } | 119 | 1.85M | if (start_row + num_rows > ptr->numrows) { | 120 | 0 | JPEGLI_ERROR("Invalid virtual array access, %u vs %u total rows", | 121 | 0 | start_row + num_rows, ptr->numrows); | 122 | 0 | } | 123 | 1.85M | if (ptr->full_buffer == nullptr) { | 124 | 0 | JPEGLI_ERROR("Invalid virtual array access, array not realized."); | 125 | 0 | } | 126 | 1.85M | return ptr->full_buffer + start_row; | 127 | 1.85M | } |
|
128 | | |
129 | 22.7k | void ClearPool(j_common_ptr cinfo, int pool_id) { |
130 | 22.7k | MemoryManager* mem = reinterpret_cast<MemoryManager*>(cinfo->mem); |
131 | 22.7k | mem->owned_ptrs[pool_id].clear(); |
132 | 22.7k | mem->total_memory_usage -= mem->pool_memory_usage[pool_id]; |
133 | 22.7k | mem->pool_memory_usage[pool_id] = 0; |
134 | 22.7k | } |
135 | | |
136 | 11.3k | void FreePool(j_common_ptr cinfo, int pool_id) { |
137 | 11.3k | MemoryManager* mem = reinterpret_cast<MemoryManager*>(cinfo->mem); |
138 | 11.3k | if (pool_id < 0 || pool_id >= JPOOL_NUMPOOLS) { |
139 | 0 | JPEGLI_ERROR("Invalid pool id %d", pool_id); |
140 | 0 | } |
141 | 49.1k | for (void* ptr : mem->owned_ptrs[pool_id]) { |
142 | 49.1k | free(ptr); |
143 | 49.1k | } |
144 | 11.3k | ClearPool(cinfo, pool_id); |
145 | 41.0k | for (void* ptr : mem->owned_ptrs[JPOOL_NUMPOOLS + pool_id]) { |
146 | 41.0k | hwy::FreeAlignedBytes(ptr, nullptr, nullptr); |
147 | 41.0k | } |
148 | 11.3k | ClearPool(cinfo, JPOOL_NUMPOOLS + pool_id); |
149 | 11.3k | } |
150 | | |
151 | 4.43k | void SelfDestruct(j_common_ptr cinfo) { |
152 | 4.43k | MemoryManager* mem = reinterpret_cast<MemoryManager*>(cinfo->mem); |
153 | 13.2k | for (int pool_id = 0; pool_id < JPOOL_NUMPOOLS; ++pool_id) { |
154 | 8.86k | FreePool(cinfo, pool_id); |
155 | 8.86k | } |
156 | 4.43k | delete mem; |
157 | 4.43k | cinfo->mem = nullptr; |
158 | 4.43k | } |
159 | | |
160 | | } // namespace |
161 | | |
162 | 4.43k | void InitMemoryManager(j_common_ptr cinfo) { |
163 | 4.43k | MemoryManager* mem = new MemoryManager; |
164 | 4.43k | mem->pub.alloc_small = jpegli::Alloc; |
165 | 4.43k | mem->pub.alloc_large = jpegli::Alloc; |
166 | 4.43k | mem->pub.alloc_sarray = jpegli::Alloc2dArray<JSAMPLE>; |
167 | 4.43k | mem->pub.alloc_barray = jpegli::Alloc2dArray<JBLOCK>; |
168 | 4.43k | mem->pub.request_virt_sarray = |
169 | 4.43k | jpegli::RequestVirtualArray<jvirt_sarray_control, JSAMPLE>; |
170 | 4.43k | mem->pub.request_virt_barray = |
171 | 4.43k | jpegli::RequestVirtualArray<jvirt_barray_control, JBLOCK>; |
172 | 4.43k | mem->pub.realize_virt_arrays = jpegli::RealizeVirtualArrays; |
173 | 4.43k | mem->pub.access_virt_sarray = |
174 | 4.43k | jpegli::AccessVirtualArray<jvirt_sarray_control, JSAMPLE>; |
175 | 4.43k | mem->pub.access_virt_barray = |
176 | 4.43k | jpegli::AccessVirtualArray<jvirt_barray_control, JBLOCK>; |
177 | 4.43k | mem->pub.free_pool = jpegli::FreePool; |
178 | 4.43k | mem->pub.self_destruct = jpegli::SelfDestruct; |
179 | 4.43k | mem->pub.max_memory_to_use = 0; |
180 | 4.43k | mem->total_memory_usage = 0; |
181 | 4.43k | mem->peak_memory_usage = 0; |
182 | 4.43k | memset(mem->pool_memory_usage, 0, sizeof(mem->pool_memory_usage)); |
183 | 4.43k | cinfo->mem = reinterpret_cast<struct jpeg_memory_mgr*>(mem); |
184 | 4.43k | } |
185 | | |
186 | | } // namespace jpegli |