Line | Count | Source (jump to first uncovered line) |
1 | | #ifndef PACK_OBJECTS_H |
2 | | #define PACK_OBJECTS_H |
3 | | |
4 | | #include "object-store-ll.h" |
5 | | #include "thread-utils.h" |
6 | | #include "pack.h" |
7 | | |
8 | | struct repository; |
9 | | |
10 | 0 | #define DEFAULT_DELTA_CACHE_SIZE (256 * 1024 * 1024) |
11 | | |
12 | 0 | #define OE_DFS_STATE_BITS 2 |
13 | 0 | #define OE_DEPTH_BITS 12 |
14 | 0 | #define OE_IN_PACK_BITS 10 |
15 | 0 | #define OE_Z_DELTA_BITS 20 |
16 | | /* |
17 | | * Note that oe_set_size() becomes expensive when the given size is |
18 | | * above this limit. Don't lower it too much. |
19 | | */ |
20 | 0 | #define OE_SIZE_BITS 31 |
21 | 0 | #define OE_DELTA_SIZE_BITS 23 |
22 | | |
23 | | /* |
24 | | * State flags for depth-first search used for analyzing delta cycles. |
25 | | * |
26 | | * The depth is measured in delta-links to the base (so if A is a delta |
27 | | * against B, then A has a depth of 1, and B a depth of 0). |
28 | | */ |
29 | | enum dfs_state { |
30 | | DFS_NONE = 0, |
31 | | DFS_ACTIVE, |
32 | | DFS_DONE, |
33 | | DFS_NUM_STATES |
34 | | }; |
35 | | |
36 | | /* |
37 | | * The size of struct nearly determines pack-objects's memory |
38 | | * consumption. This struct is packed tight for that reason. When you |
39 | | * add or reorder something in this struct, think a bit about this. |
40 | | * |
41 | | * basic object info |
42 | | * ----------------- |
43 | | * idx.oid is filled up before delta searching starts. idx.crc32 is |
44 | | * only valid after the object is written out and will be used for |
45 | | * generating the index. idx.offset will be both gradually set and |
46 | | * used in writing phase (base objects get offset first, then deltas |
47 | | * refer to them) |
48 | | * |
49 | | * "size" is the uncompressed object size. Compressed size of the raw |
50 | | * data for an object in a pack is not stored anywhere but is computed |
51 | | * and made available when reverse .idx is made. Note that when a |
52 | | * delta is reused, "size" is the uncompressed _delta_ size, not the |
53 | | * canonical one after the delta has been applied. |
54 | | * |
55 | | * "hash" contains a path name hash which is used for sorting the |
56 | | * delta list and also during delta searching. Once prepare_pack() |
57 | | * returns it's no longer needed. |
58 | | * |
59 | | * source pack info |
60 | | * ---------------- |
61 | | * The (in_pack, in_pack_offset) tuple contains the location of the |
62 | | * object in the source pack. in_pack_header_size allows quickly |
63 | | * skipping the header and going straight to the zlib stream. |
64 | | * |
65 | | * "type" and "in_pack_type" both describe object type. in_pack_type |
66 | | * may contain a delta type, while type is always the canonical type. |
67 | | * |
68 | | * deltas |
69 | | * ------ |
70 | | * Delta links (delta, delta_child and delta_sibling) are created to |
71 | | * reflect that delta graph from the source pack then updated or added |
72 | | * during delta searching phase when we find better deltas. |
73 | | * |
74 | | * delta_child and delta_sibling are last needed in |
75 | | * compute_write_order(). "delta" and "delta_size" must remain valid |
76 | | * at object writing phase in case the delta is not cached. |
77 | | * |
78 | | * If a delta is cached in memory and is compressed, delta_data points |
79 | | * to the data and z_delta_size contains the compressed size. If it's |
80 | | * uncompressed [1], z_delta_size must be zero. delta_size is always |
81 | | * the uncompressed size and must be valid even if the delta is not |
82 | | * cached. |
83 | | * |
84 | | * [1] during try_delta phase we don't bother with compressing because |
85 | | * the delta could be quickly replaced with a better one. |
86 | | */ |
87 | | struct object_entry { |
88 | | struct pack_idx_entry idx; |
89 | | void *delta_data; /* cached delta (uncompressed) */ |
90 | | off_t in_pack_offset; |
91 | | uint32_t hash; /* name hint hash */ |
92 | | unsigned size_:OE_SIZE_BITS; |
93 | | unsigned size_valid:1; |
94 | | uint32_t delta_idx; /* delta base object */ |
95 | | uint32_t delta_child_idx; /* deltified objects who bases me */ |
96 | | uint32_t delta_sibling_idx; /* other deltified objects who |
97 | | * uses the same base as me |
98 | | */ |
99 | | unsigned delta_size_:OE_DELTA_SIZE_BITS; /* delta data size (uncompressed) */ |
100 | | unsigned delta_size_valid:1; |
101 | | unsigned char in_pack_header_size; |
102 | | unsigned in_pack_idx:OE_IN_PACK_BITS; /* already in pack */ |
103 | | unsigned z_delta_size:OE_Z_DELTA_BITS; |
104 | | unsigned type_valid:1; |
105 | | unsigned no_try_delta:1; |
106 | | unsigned type_:TYPE_BITS; |
107 | | unsigned in_pack_type:TYPE_BITS; /* could be delta */ |
108 | | |
109 | | unsigned preferred_base:1; /* |
110 | | * we do not pack this, but is available |
111 | | * to be used as the base object to delta |
112 | | * objects against. |
113 | | */ |
114 | | unsigned tagged:1; /* near the very tip of refs */ |
115 | | unsigned filled:1; /* assigned write-order */ |
116 | | unsigned dfs_state:OE_DFS_STATE_BITS; |
117 | | unsigned depth:OE_DEPTH_BITS; |
118 | | unsigned ext_base:1; /* delta_idx points outside packlist */ |
119 | | }; |
120 | | |
121 | | struct packing_data { |
122 | | struct repository *repo; |
123 | | struct object_entry *objects; |
124 | | uint32_t nr_objects, nr_alloc; |
125 | | |
126 | | int32_t *index; |
127 | | uint32_t index_size; |
128 | | |
129 | | unsigned int *in_pack_pos; |
130 | | unsigned long *delta_size; |
131 | | |
132 | | /* |
133 | | * Only one of these can be non-NULL and they have different |
134 | | * sizes. if in_pack_by_idx is allocated, oe_in_pack() returns |
135 | | * the pack of an object using in_pack_idx field. If not, |
136 | | * in_pack[] array is used the same way as in_pack_pos[] |
137 | | */ |
138 | | struct packed_git **in_pack_by_idx; |
139 | | struct packed_git **in_pack; |
140 | | |
141 | | /* |
142 | | * During packing with multiple threads, protect the in-core |
143 | | * object database from concurrent accesses. |
144 | | */ |
145 | | pthread_mutex_t odb_lock; |
146 | | |
147 | | /* |
148 | | * This list contains entries for bases which we know the other side |
149 | | * has (e.g., via reachability bitmaps), but which aren't in our |
150 | | * "objects" list. |
151 | | */ |
152 | | struct object_entry *ext_bases; |
153 | | uint32_t nr_ext, alloc_ext; |
154 | | |
155 | | uintmax_t oe_size_limit; |
156 | | uintmax_t oe_delta_size_limit; |
157 | | |
158 | | /* delta islands */ |
159 | | unsigned int *tree_depth; |
160 | | unsigned char *layer; |
161 | | |
162 | | /* |
163 | | * Used when writing cruft packs. |
164 | | * |
165 | | * Object mtimes are stored in pack order when writing, but |
166 | | * written out in lexicographic (index) order. |
167 | | */ |
168 | | uint32_t *cruft_mtime; |
169 | | }; |
170 | | |
171 | | void prepare_packing_data(struct repository *r, struct packing_data *pdata); |
172 | | void clear_packing_data(struct packing_data *pdata); |
173 | | |
174 | | /* Protect access to object database */ |
175 | | static inline void packing_data_lock(struct packing_data *pdata) |
176 | 0 | { |
177 | 0 | pthread_mutex_lock(&pdata->odb_lock); |
178 | 0 | } Unexecuted instantiation: fsck.c:packing_data_lock Unexecuted instantiation: gc.c:packing_data_lock Unexecuted instantiation: pack-objects.c:packing_data_lock Unexecuted instantiation: repack.c:packing_data_lock Unexecuted instantiation: rev-list.c:packing_data_lock Unexecuted instantiation: delta-islands.c:packing_data_lock Unexecuted instantiation: midx.c:packing_data_lock Unexecuted instantiation: midx-write.c:packing_data_lock Unexecuted instantiation: pack-bitmap-write.c:packing_data_lock Unexecuted instantiation: pack-bitmap.c:packing_data_lock Unexecuted instantiation: pack-write.c:packing_data_lock Unexecuted instantiation: pseudo-merge.c:packing_data_lock Unexecuted instantiation: reachable.c:packing_data_lock |
179 | | static inline void packing_data_unlock(struct packing_data *pdata) |
180 | 0 | { |
181 | 0 | pthread_mutex_unlock(&pdata->odb_lock); |
182 | 0 | } Unexecuted instantiation: fsck.c:packing_data_unlock Unexecuted instantiation: gc.c:packing_data_unlock Unexecuted instantiation: pack-objects.c:packing_data_unlock Unexecuted instantiation: repack.c:packing_data_unlock Unexecuted instantiation: rev-list.c:packing_data_unlock Unexecuted instantiation: delta-islands.c:packing_data_unlock Unexecuted instantiation: midx.c:packing_data_unlock Unexecuted instantiation: midx-write.c:packing_data_unlock Unexecuted instantiation: pack-bitmap-write.c:packing_data_unlock Unexecuted instantiation: pack-bitmap.c:packing_data_unlock Unexecuted instantiation: pack-write.c:packing_data_unlock Unexecuted instantiation: pseudo-merge.c:packing_data_unlock Unexecuted instantiation: reachable.c:packing_data_unlock |
183 | | |
184 | | struct object_entry *packlist_alloc(struct packing_data *pdata, |
185 | | const struct object_id *oid); |
186 | | |
187 | | struct object_entry *packlist_find(struct packing_data *pdata, |
188 | | const struct object_id *oid); |
189 | | |
190 | | static inline uint32_t pack_name_hash(const char *name) |
191 | 0 | { |
192 | 0 | uint32_t c, hash = 0; |
193 | |
|
194 | 0 | if (!name) |
195 | 0 | return 0; |
196 | | |
197 | | /* |
198 | | * This effectively just creates a sortable number from the |
199 | | * last sixteen non-whitespace characters. Last characters |
200 | | * count "most", so things that end in ".c" sort together. |
201 | | */ |
202 | 0 | while ((c = *name++) != 0) { |
203 | 0 | if (isspace(c)) |
204 | 0 | continue; |
205 | 0 | hash = (hash >> 2) + (c << 24); |
206 | 0 | } |
207 | 0 | return hash; |
208 | 0 | } Unexecuted instantiation: fsck.c:pack_name_hash Unexecuted instantiation: gc.c:pack_name_hash Unexecuted instantiation: pack-objects.c:pack_name_hash Unexecuted instantiation: repack.c:pack_name_hash Unexecuted instantiation: rev-list.c:pack_name_hash Unexecuted instantiation: delta-islands.c:pack_name_hash Unexecuted instantiation: midx.c:pack_name_hash Unexecuted instantiation: midx-write.c:pack_name_hash Unexecuted instantiation: pack-bitmap-write.c:pack_name_hash Unexecuted instantiation: pack-bitmap.c:pack_name_hash Unexecuted instantiation: pack-write.c:pack_name_hash Unexecuted instantiation: pseudo-merge.c:pack_name_hash Unexecuted instantiation: reachable.c:pack_name_hash |
209 | | |
210 | | static inline enum object_type oe_type(const struct object_entry *e) |
211 | 0 | { |
212 | 0 | return e->type_valid ? e->type_ : OBJ_BAD; |
213 | 0 | } Unexecuted instantiation: fsck.c:oe_type Unexecuted instantiation: gc.c:oe_type Unexecuted instantiation: pack-objects.c:oe_type Unexecuted instantiation: repack.c:oe_type Unexecuted instantiation: rev-list.c:oe_type Unexecuted instantiation: delta-islands.c:oe_type Unexecuted instantiation: midx.c:oe_type Unexecuted instantiation: midx-write.c:oe_type Unexecuted instantiation: pack-bitmap-write.c:oe_type Unexecuted instantiation: pack-bitmap.c:oe_type Unexecuted instantiation: pack-write.c:oe_type Unexecuted instantiation: pseudo-merge.c:oe_type Unexecuted instantiation: reachable.c:oe_type |
214 | | |
215 | | static inline void oe_set_type(struct object_entry *e, |
216 | | enum object_type type) |
217 | 0 | { |
218 | 0 | if (type >= OBJ_ANY) |
219 | 0 | BUG("OBJ_ANY cannot be set in pack-objects code"); |
220 | | |
221 | 0 | e->type_valid = type >= OBJ_NONE; |
222 | 0 | e->type_ = (unsigned)type; |
223 | 0 | } Unexecuted instantiation: fsck.c:oe_set_type Unexecuted instantiation: gc.c:oe_set_type Unexecuted instantiation: pack-objects.c:oe_set_type Unexecuted instantiation: repack.c:oe_set_type Unexecuted instantiation: rev-list.c:oe_set_type Unexecuted instantiation: delta-islands.c:oe_set_type Unexecuted instantiation: midx.c:oe_set_type Unexecuted instantiation: midx-write.c:oe_set_type Unexecuted instantiation: pack-bitmap-write.c:oe_set_type Unexecuted instantiation: pack-bitmap.c:oe_set_type Unexecuted instantiation: pack-write.c:oe_set_type Unexecuted instantiation: pseudo-merge.c:oe_set_type Unexecuted instantiation: reachable.c:oe_set_type |
224 | | |
225 | | static inline unsigned int oe_in_pack_pos(const struct packing_data *pack, |
226 | | const struct object_entry *e) |
227 | 0 | { |
228 | 0 | return pack->in_pack_pos[e - pack->objects]; |
229 | 0 | } Unexecuted instantiation: fsck.c:oe_in_pack_pos Unexecuted instantiation: gc.c:oe_in_pack_pos Unexecuted instantiation: pack-objects.c:oe_in_pack_pos Unexecuted instantiation: repack.c:oe_in_pack_pos Unexecuted instantiation: rev-list.c:oe_in_pack_pos Unexecuted instantiation: delta-islands.c:oe_in_pack_pos Unexecuted instantiation: midx.c:oe_in_pack_pos Unexecuted instantiation: midx-write.c:oe_in_pack_pos Unexecuted instantiation: pack-bitmap-write.c:oe_in_pack_pos Unexecuted instantiation: pack-bitmap.c:oe_in_pack_pos Unexecuted instantiation: pack-write.c:oe_in_pack_pos Unexecuted instantiation: pseudo-merge.c:oe_in_pack_pos Unexecuted instantiation: reachable.c:oe_in_pack_pos |
230 | | |
231 | | static inline void oe_set_in_pack_pos(const struct packing_data *pack, |
232 | | const struct object_entry *e, |
233 | | unsigned int pos) |
234 | 0 | { |
235 | 0 | pack->in_pack_pos[e - pack->objects] = pos; |
236 | 0 | } Unexecuted instantiation: fsck.c:oe_set_in_pack_pos Unexecuted instantiation: gc.c:oe_set_in_pack_pos Unexecuted instantiation: pack-objects.c:oe_set_in_pack_pos Unexecuted instantiation: repack.c:oe_set_in_pack_pos Unexecuted instantiation: rev-list.c:oe_set_in_pack_pos Unexecuted instantiation: delta-islands.c:oe_set_in_pack_pos Unexecuted instantiation: midx.c:oe_set_in_pack_pos Unexecuted instantiation: midx-write.c:oe_set_in_pack_pos Unexecuted instantiation: pack-bitmap-write.c:oe_set_in_pack_pos Unexecuted instantiation: pack-bitmap.c:oe_set_in_pack_pos Unexecuted instantiation: pack-write.c:oe_set_in_pack_pos Unexecuted instantiation: pseudo-merge.c:oe_set_in_pack_pos Unexecuted instantiation: reachable.c:oe_set_in_pack_pos |
237 | | |
238 | | static inline struct packed_git *oe_in_pack(const struct packing_data *pack, |
239 | | const struct object_entry *e) |
240 | 0 | { |
241 | 0 | if (pack->in_pack_by_idx) |
242 | 0 | return pack->in_pack_by_idx[e->in_pack_idx]; |
243 | 0 | else |
244 | 0 | return pack->in_pack[e - pack->objects]; |
245 | 0 | } Unexecuted instantiation: fsck.c:oe_in_pack Unexecuted instantiation: gc.c:oe_in_pack Unexecuted instantiation: pack-objects.c:oe_in_pack Unexecuted instantiation: repack.c:oe_in_pack Unexecuted instantiation: rev-list.c:oe_in_pack Unexecuted instantiation: delta-islands.c:oe_in_pack Unexecuted instantiation: midx.c:oe_in_pack Unexecuted instantiation: midx-write.c:oe_in_pack Unexecuted instantiation: pack-bitmap-write.c:oe_in_pack Unexecuted instantiation: pack-bitmap.c:oe_in_pack Unexecuted instantiation: pack-write.c:oe_in_pack Unexecuted instantiation: pseudo-merge.c:oe_in_pack Unexecuted instantiation: reachable.c:oe_in_pack |
246 | | |
247 | | void oe_map_new_pack(struct packing_data *pack); |
248 | | |
249 | | static inline void oe_set_in_pack(struct packing_data *pack, |
250 | | struct object_entry *e, |
251 | | struct packed_git *p) |
252 | 0 | { |
253 | 0 | if (pack->in_pack_by_idx) { |
254 | 0 | if (p->index) { |
255 | 0 | e->in_pack_idx = p->index; |
256 | 0 | return; |
257 | 0 | } |
258 | | /* |
259 | | * We're accessing packs by index, but this pack doesn't have |
260 | | * an index (e.g., because it was added since we created the |
261 | | * in_pack_by_idx array). Bail to oe_map_new_pack(), which |
262 | | * will convert us to using the full in_pack array, and then |
263 | | * fall through to our in_pack handling. |
264 | | */ |
265 | 0 | oe_map_new_pack(pack); |
266 | 0 | } |
267 | 0 | pack->in_pack[e - pack->objects] = p; |
268 | 0 | } Unexecuted instantiation: fsck.c:oe_set_in_pack Unexecuted instantiation: gc.c:oe_set_in_pack Unexecuted instantiation: pack-objects.c:oe_set_in_pack Unexecuted instantiation: repack.c:oe_set_in_pack Unexecuted instantiation: rev-list.c:oe_set_in_pack Unexecuted instantiation: delta-islands.c:oe_set_in_pack Unexecuted instantiation: midx.c:oe_set_in_pack Unexecuted instantiation: midx-write.c:oe_set_in_pack Unexecuted instantiation: pack-bitmap-write.c:oe_set_in_pack Unexecuted instantiation: pack-bitmap.c:oe_set_in_pack Unexecuted instantiation: pack-write.c:oe_set_in_pack Unexecuted instantiation: pseudo-merge.c:oe_set_in_pack Unexecuted instantiation: reachable.c:oe_set_in_pack |
269 | | |
270 | | void oe_set_delta_ext(struct packing_data *pack, |
271 | | struct object_entry *e, |
272 | | const struct object_id *oid); |
273 | | |
274 | | static inline unsigned int oe_tree_depth(struct packing_data *pack, |
275 | | struct object_entry *e) |
276 | 0 | { |
277 | 0 | if (!pack->tree_depth) |
278 | 0 | return 0; |
279 | 0 | return pack->tree_depth[e - pack->objects]; |
280 | 0 | } Unexecuted instantiation: fsck.c:oe_tree_depth Unexecuted instantiation: gc.c:oe_tree_depth Unexecuted instantiation: pack-objects.c:oe_tree_depth Unexecuted instantiation: repack.c:oe_tree_depth Unexecuted instantiation: rev-list.c:oe_tree_depth Unexecuted instantiation: delta-islands.c:oe_tree_depth Unexecuted instantiation: midx.c:oe_tree_depth Unexecuted instantiation: midx-write.c:oe_tree_depth Unexecuted instantiation: pack-bitmap-write.c:oe_tree_depth Unexecuted instantiation: pack-bitmap.c:oe_tree_depth Unexecuted instantiation: pack-write.c:oe_tree_depth Unexecuted instantiation: pseudo-merge.c:oe_tree_depth Unexecuted instantiation: reachable.c:oe_tree_depth |
281 | | |
282 | | static inline void oe_set_layer(struct packing_data *pack, |
283 | | struct object_entry *e, |
284 | | unsigned char layer) |
285 | 0 | { |
286 | 0 | if (!pack->layer) |
287 | 0 | CALLOC_ARRAY(pack->layer, pack->nr_alloc); |
288 | 0 | pack->layer[e - pack->objects] = layer; |
289 | 0 | } Unexecuted instantiation: fsck.c:oe_set_layer Unexecuted instantiation: gc.c:oe_set_layer Unexecuted instantiation: pack-objects.c:oe_set_layer Unexecuted instantiation: repack.c:oe_set_layer Unexecuted instantiation: rev-list.c:oe_set_layer Unexecuted instantiation: delta-islands.c:oe_set_layer Unexecuted instantiation: midx.c:oe_set_layer Unexecuted instantiation: midx-write.c:oe_set_layer Unexecuted instantiation: pack-bitmap-write.c:oe_set_layer Unexecuted instantiation: pack-bitmap.c:oe_set_layer Unexecuted instantiation: pack-write.c:oe_set_layer Unexecuted instantiation: pseudo-merge.c:oe_set_layer Unexecuted instantiation: reachable.c:oe_set_layer |
290 | | |
291 | | static inline uint32_t oe_cruft_mtime(struct packing_data *pack, |
292 | | struct object_entry *e) |
293 | 0 | { |
294 | 0 | if (!pack->cruft_mtime) |
295 | 0 | return 0; |
296 | 0 | return pack->cruft_mtime[e - pack->objects]; |
297 | 0 | } Unexecuted instantiation: fsck.c:oe_cruft_mtime Unexecuted instantiation: gc.c:oe_cruft_mtime Unexecuted instantiation: pack-objects.c:oe_cruft_mtime Unexecuted instantiation: repack.c:oe_cruft_mtime Unexecuted instantiation: rev-list.c:oe_cruft_mtime Unexecuted instantiation: delta-islands.c:oe_cruft_mtime Unexecuted instantiation: midx.c:oe_cruft_mtime Unexecuted instantiation: midx-write.c:oe_cruft_mtime Unexecuted instantiation: pack-bitmap-write.c:oe_cruft_mtime Unexecuted instantiation: pack-bitmap.c:oe_cruft_mtime Unexecuted instantiation: pack-write.c:oe_cruft_mtime Unexecuted instantiation: pseudo-merge.c:oe_cruft_mtime Unexecuted instantiation: reachable.c:oe_cruft_mtime |
298 | | |
299 | | static inline void oe_set_cruft_mtime(struct packing_data *pack, |
300 | | struct object_entry *e, |
301 | | uint32_t mtime) |
302 | 0 | { |
303 | 0 | if (!pack->cruft_mtime) |
304 | 0 | CALLOC_ARRAY(pack->cruft_mtime, pack->nr_alloc); |
305 | 0 | pack->cruft_mtime[e - pack->objects] = mtime; |
306 | 0 | } Unexecuted instantiation: fsck.c:oe_set_cruft_mtime Unexecuted instantiation: gc.c:oe_set_cruft_mtime Unexecuted instantiation: pack-objects.c:oe_set_cruft_mtime Unexecuted instantiation: repack.c:oe_set_cruft_mtime Unexecuted instantiation: rev-list.c:oe_set_cruft_mtime Unexecuted instantiation: delta-islands.c:oe_set_cruft_mtime Unexecuted instantiation: midx.c:oe_set_cruft_mtime Unexecuted instantiation: midx-write.c:oe_set_cruft_mtime Unexecuted instantiation: pack-bitmap-write.c:oe_set_cruft_mtime Unexecuted instantiation: pack-bitmap.c:oe_set_cruft_mtime Unexecuted instantiation: pack-write.c:oe_set_cruft_mtime Unexecuted instantiation: pseudo-merge.c:oe_set_cruft_mtime Unexecuted instantiation: reachable.c:oe_set_cruft_mtime |
307 | | |
308 | | #endif |