Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/pack.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# pack.py -- For dealing with packed git objects.
2# Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
3# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
23"""Classes for dealing with packed git objects.
25A pack is a compact representation of a bunch of objects, stored
26using deltas where possible.
28They have two parts, the pack file, which stores the data, and an index
29that tells you where the data is.
31To find an object you look in all of the index files 'til you find a
32match for the object name. You then use the pointer got from this as
33a pointer in to the corresponding packfile.
34"""
36__all__ = [
37 "DEFAULT_DELTA_BASE_CACHE_LIMIT",
38 "DEFAULT_PACK_DELTA_WINDOW_SIZE",
39 "DEFAULT_PACK_INDEX_VERSION",
40 "DELTA_TYPES",
41 "OFS_DELTA",
42 "PACK_SPOOL_FILE_MAX_SIZE",
43 "REF_DELTA",
44 "DeltaChainIterator",
45 "DeltaCycle",
46 "FilePackIndex",
47 "HashWriter",
48 "MemoryPackIndex",
49 "ObjectContainer",
50 "OldUnpackedObject",
51 "Pack",
52 "PackChunkGenerator",
53 "PackData",
54 "PackFileDisappeared",
55 "PackHint",
56 "PackIndex",
57 "PackIndex1",
58 "PackIndex2",
59 "PackIndex3",
60 "PackIndexEntry",
61 "PackIndexer",
62 "PackInflater",
63 "PackStreamCopier",
64 "PackStreamReader",
65 "PackedObjectContainer",
66 "ProgressFn",
67 "ResolveExtRefFn",
68 "SHA1Reader",
69 "SHA1Writer",
70 "UnpackedObject",
71 "UnpackedObjectIterator",
72 "UnpackedObjectStream",
73 "UnresolvedDeltas",
74 "apply_delta",
75 "bisect_find_sha",
76 "chunks_length",
77 "compute_buffer_sha",
78 "compute_file_sha",
79 "create_delta",
80 "deltas_from_sorted_objects",
81 "deltify_pack_objects",
82 "extend_pack",
83 "find_reusable_deltas",
84 "full_unpacked_object",
85 "generate_unpacked_objects",
86 "has_mmap",
87 "iter_sha1",
88 "load_pack_index",
89 "load_pack_index_file",
90 "obj_sha",
91 "pack_header_chunks",
92 "pack_object_chunks",
93 "pack_object_header",
94 "pack_objects_to_data",
95 "read_pack_header",
96 "read_pack_header_at",
97 "read_zlib_chunks",
98 "read_zlib_chunks_at",
99 "sort_objects_for_delta",
100 "take_msb_bytes",
101 "take_msb_bytes_at",
102 "unpack_object",
103 "unpack_object_at",
104 "verify_and_read",
105 "write_pack",
106 "write_pack_data",
107 "write_pack_from_container",
108 "write_pack_header",
109 "write_pack_index",
110 "write_pack_index_v1",
111 "write_pack_index_v2",
112 "write_pack_index_v3",
113 "write_pack_object",
114 "write_pack_objects",
115]
117import binascii
118from collections import defaultdict, deque
119from contextlib import suppress
120from io import BytesIO, UnsupportedOperation
122try:
123 from cdifflib import CSequenceMatcher as SequenceMatcher
124except ModuleNotFoundError:
125 from difflib import SequenceMatcher
127import logging
128import os
129import struct
130import sys
131import threading
132import warnings
133import zlib
134from collections.abc import Callable, Iterable, Iterator, Sequence, Set
135from hashlib import sha1, sha256
136from itertools import chain
137from os import SEEK_END
138from struct import unpack_from
139from types import TracebackType
140from typing import (
141 IO,
142 TYPE_CHECKING,
143 Any,
144 BinaryIO,
145 Generic,
146 Protocol,
147 TypeVar,
148)
150if sys.version_info >= (3, 11):
151 from typing import Self
152else:
153 from typing_extensions import Self
155import mmap
157from .errors import ApplyDeltaError, ChecksumMismatch
158from .file import GitFile, _GitFile
159from .lru_cache import LRUSizeCache
160from .object_format import OBJECT_FORMAT_TYPE_NUMS, SHA1, ObjectFormat
161from .objects import (
162 ObjectID,
163 RawObjectID,
164 ShaFile,
165 hex_to_sha,
166 object_header,
167 sha_to_hex,
168)
170if TYPE_CHECKING:
171 from _hashlib import HASH as HashObject
173 from .bitmap import PackBitmap
174 from .commit_graph import CommitGraph
175 from .object_store import BaseObjectStore
176 from .refs import Ref
178logger = logging.getLogger(__name__)
180# Some platforms (e.g. plan9) don't support mmap properly
181has_mmap = sys.platform != "Plan9"
183OFS_DELTA = 6
184REF_DELTA = 7
186DELTA_TYPES = (OFS_DELTA, REF_DELTA)
189DEFAULT_PACK_DELTA_WINDOW_SIZE = 10
191# Keep pack files under 16Mb in memory, otherwise write them out to disk
192PACK_SPOOL_FILE_MAX_SIZE = 16 * 1024 * 1024
194# Default pack index version to use when none is specified
195DEFAULT_PACK_INDEX_VERSION = 2
198OldUnpackedObject = tuple[bytes | int, list[bytes]] | list[bytes] | bytes
199ResolveExtRefFn = Callable[[RawObjectID | ObjectID], tuple[int, bytes | list[bytes]]]
200ProgressFn = Callable[[int, str], None]
201PackHint = tuple[int, bytes | None]
204def verify_and_read(
205 read_func: Callable[[int], bytes],
206 expected_hash: bytes,
207 hash_algo: str,
208 progress: Callable[[bytes], None] | None = None,
209) -> Iterator[bytes]:
210 """Read from stream, verify hash, then yield verified chunks.
212 This function downloads data to a temporary file (in-memory for small files,
213 on-disk for large ones) while computing its hash. Only after the hash is
214 verified to match expected_hash will it yield any data. This prevents
215 corrupted or malicious data from reaching the caller.
217 Args:
218 read_func: Function to read bytes (like file.read or HTTP response reader)
219 expected_hash: Expected hash as hex string bytes (e.g., b'a3b2c1...')
220 hash_algo: Hash algorithm name ('sha1' or 'sha256')
221 progress: Optional progress callback
223 Yields:
224 Chunks of verified data (only after hash verification succeeds)
226 Raises:
227 ValueError: If hash doesn't match or algorithm unsupported
228 """
229 from tempfile import SpooledTemporaryFile
231 from .object_format import OBJECT_FORMATS
233 # Get the hash function for this algorithm
234 obj_format = OBJECT_FORMATS.get(hash_algo)
235 if obj_format is None:
236 raise ValueError(f"Unsupported hash algorithm: {hash_algo}")
238 hasher = obj_format.new_hash()
240 # Download to temporary file (memory or disk) while computing hash
241 with SpooledTemporaryFile(
242 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="dulwich-verify-"
243 ) as temp_file:
244 # Read data, hash it, and write to temp file
245 while True:
246 chunk = read_func(65536) # Read in 64KB chunks
247 if not chunk:
248 break
249 hasher.update(chunk)
250 temp_file.write(chunk)
252 # Verify hash BEFORE yielding any data
253 computed_hash = hasher.hexdigest().encode("ascii")
254 if computed_hash != expected_hash:
255 raise ValueError(
256 f"hash mismatch: expected {expected_hash.decode('ascii')}, "
257 f"got {computed_hash.decode('ascii')}"
258 )
260 # Hash verified! Now read from temp file and yield chunks
261 if progress:
262 progress(b"Hash verified, processing data\n")
264 temp_file.seek(0)
265 while True:
266 chunk = temp_file.read(65536)
267 if not chunk:
268 break
269 yield chunk
272class UnresolvedDeltas(Exception):
273 """Delta objects could not be resolved."""
275 def __init__(self, shas: list[bytes]) -> None:
276 """Initialize UnresolvedDeltas exception.
278 Args:
279 shas: List of SHA hashes for unresolved delta objects
280 """
281 self.shas = shas
284class DeltaCycle(UnresolvedDeltas):
285 """A pack's delta chain references itself and cannot be resolved."""
288class ObjectContainer(Protocol):
289 """Protocol for objects that can contain git objects."""
291 def add_object(self, obj: ShaFile) -> None:
292 """Add a single object to this object store."""
294 def add_objects(
295 self,
296 objects: Sequence[tuple[ShaFile, str | None]],
297 progress: Callable[..., None] | None = None,
298 ) -> "Pack | None":
299 """Add a set of objects to this object store.
301 Args:
302 objects: Iterable over a list of (object, path) tuples
303 progress: Progress callback for object insertion
304 Returns: Optional Pack object of the objects written.
305 """
307 def __contains__(self, sha1: "ObjectID") -> bool:
308 """Check if a hex sha is present."""
309 ...
311 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
312 """Retrieve an object."""
313 ...
315 def peel(self, sha: "ObjectID | RawObjectID") -> tuple[ShaFile, ShaFile]:
316 """Peel all tags from a SHA.
318 Args:
319 sha: The object SHA to peel.
320 Returns: Tuple of (unpeeled, peeled) objects.
321 """
322 from .object_store import peel_sha
324 return peel_sha(self, sha)
326 def get_commit_graph(self) -> "CommitGraph | None":
327 """Get the commit graph for this object store.
329 Returns:
330 CommitGraph object if available, None otherwise
331 """
332 return None
335class PackedObjectContainer(ObjectContainer):
336 """Container for objects packed in a pack file."""
338 def get_unpacked_object(
339 self, sha1: "ObjectID | RawObjectID", *, include_comp: bool = False
340 ) -> "UnpackedObject":
341 """Get a raw unresolved object.
343 Args:
344 sha1: SHA-1 hash of the object
345 include_comp: Whether to include compressed data
347 Returns:
348 UnpackedObject instance
349 """
350 raise NotImplementedError(self.get_unpacked_object)
352 def iterobjects_subset(
353 self, shas: Iterable["ObjectID"], *, allow_missing: bool = False
354 ) -> Iterator[ShaFile]:
355 """Iterate over a subset of objects.
357 Args:
358 shas: Iterable of object SHAs to retrieve
359 allow_missing: If True, skip missing objects
361 Returns:
362 Iterator of ShaFile objects
363 """
364 raise NotImplementedError(self.iterobjects_subset)
366 def iter_unpacked_subset(
367 self,
368 shas: Iterable["ObjectID | RawObjectID"],
369 *,
370 include_comp: bool = False,
371 allow_missing: bool = False,
372 convert_ofs_delta: bool = True,
373 ) -> Iterator["UnpackedObject"]:
374 """Iterate over unpacked objects from a subset of SHAs.
376 Args:
377 shas: Set of object SHAs to retrieve
378 include_comp: Include compressed data if True
379 allow_missing: If True, skip missing objects
380 convert_ofs_delta: If True, convert offset deltas to ref deltas
382 Returns:
383 Iterator of UnpackedObject instances
384 """
385 raise NotImplementedError(self.iter_unpacked_subset)
388class UnpackedObjectStream:
389 """Abstract base class for a stream of unpacked objects."""
391 def __iter__(self) -> Iterator["UnpackedObject"]:
392 """Iterate over unpacked objects."""
393 raise NotImplementedError(self.__iter__)
395 def __len__(self) -> int:
396 """Return the number of objects in the stream."""
397 raise NotImplementedError(self.__len__)
400def take_msb_bytes(
401 read: Callable[[int], bytes], crc32: int | None = None
402) -> tuple[list[int], int | None]:
403 """Read bytes marked with most significant bit.
405 Args:
406 read: Read function
407 crc32: Optional CRC32 checksum to update
409 Returns:
410 Tuple of (list of bytes read, updated CRC32 or None)
411 """
412 ret: list[int] = []
413 while len(ret) == 0 or ret[-1] & 0x80:
414 b = read(1)
415 if crc32 is not None:
416 crc32 = binascii.crc32(b, crc32)
417 ret.append(ord(b[:1]))
418 return ret, crc32
421def take_msb_bytes_at(
422 contents: "bytes | mmap.mmap", offset: int, crc32: int | None = None
423) -> tuple[list[int], int, int | None]:
424 """Read bytes marked with most significant bit from a buffer at an offset.
426 Args:
427 contents: Buffer to read from
428 offset: Offset in contents to start reading at
429 crc32: Optional CRC32 checksum to update
431 Returns:
432 Tuple of (list of bytes read, offset just past them, updated CRC32 or None)
433 """
434 ret: list[int] = []
435 pos = offset
436 while len(ret) == 0 or ret[-1] & 0x80:
437 b = contents[pos : pos + 1]
438 if not b:
439 raise AssertionError(f"unexpected end of pack data at {pos}")
440 pos += 1
441 if crc32 is not None:
442 crc32 = binascii.crc32(b, crc32)
443 ret.append(ord(b))
444 return ret, pos, crc32
447class PackFileDisappeared(Exception):
448 """Raised when a pack file unexpectedly disappears.
450 This typically happens when a concurrent operation (e.g. ``git repack``
451 or ``git gc --auto``) removes a pack file between the moment dulwich
452 snapshots the pack directory and the moment it actually opens the
453 pack's ``.idx`` or ``.pack`` file.
455 The ``obj`` attribute holds the :class:`Pack` (or :class:`FilePackIndex`)
456 whose backing file vanished, so the caller can evict the stale object
457 from its cache and rescan the pack directory.
458 """
460 obj: "Pack | FilePackIndex"
462 def __init__(self, obj: "Pack | FilePackIndex") -> None:
463 """Initialize PackFileDisappeared exception.
465 Args:
466 obj: The pack or pack index that disappeared.
467 """
468 self.obj = obj
471class UnpackedObject:
472 """Class encapsulating an object unpacked from a pack file.
474 These objects should only be created from within unpack_object. Most
475 members start out as empty and are filled in at various points by
476 read_zlib_chunks, unpack_object, DeltaChainIterator, etc.
478 End users of this object should take care that the function they're getting
479 this object from is guaranteed to set the members they need.
480 """
482 __slots__ = [
483 "_sha", # Cached binary SHA.
484 "comp_chunks", # Compressed object chunks.
485 "crc32", # CRC32.
486 "decomp_chunks", # Decompressed object chunks.
487 "decomp_len", # Decompressed length of this object.
488 "delta_base", # Delta base offset or SHA.
489 "hash_func", # Hash function to use for computing object IDs.
490 "obj_chunks", # Decompressed and delta-resolved chunks.
491 "obj_type_num", # Type of this object.
492 "offset", # Offset in its pack.
493 "pack_type_num", # Type of this object in the pack (may be a delta).
494 ]
496 obj_type_num: int | None
497 obj_chunks: list[bytes] | None
498 delta_base: bytes | int | None
499 decomp_chunks: list[bytes]
500 comp_chunks: list[bytes] | None
501 decomp_len: int | None
502 crc32: int | None
503 offset: int | None
504 pack_type_num: int
505 _sha: bytes | None
506 hash_func: Callable[[], "HashObject"]
508 # TODO(dborowitz): read_zlib_chunks and unpack_object could very well be
509 # methods of this object.
510 def __init__(
511 self,
512 pack_type_num: int,
513 *,
514 delta_base: bytes | int | None = None,
515 decomp_len: int | None = None,
516 crc32: int | None = None,
517 sha: bytes | None = None,
518 decomp_chunks: list[bytes] | None = None,
519 offset: int | None = None,
520 hash_func: Callable[[], "HashObject"] = sha1,
521 ) -> None:
522 """Initialize an UnpackedObject.
524 Args:
525 pack_type_num: Type number of this object in the pack
526 delta_base: Delta base (offset or SHA) if this is a delta object
527 decomp_len: Decompressed length of this object
528 crc32: CRC32 checksum
529 sha: SHA hash of the object
530 decomp_chunks: Decompressed chunks
531 offset: Offset in the pack file
532 hash_func: Hash function to use (defaults to sha1)
533 """
534 self.offset = offset
535 self._sha = sha
536 self.pack_type_num = pack_type_num
537 self.delta_base = delta_base
538 self.comp_chunks = None
539 self.decomp_chunks: list[bytes] = decomp_chunks or []
540 if decomp_chunks is not None and decomp_len is None:
541 self.decomp_len = sum(map(len, decomp_chunks))
542 else:
543 self.decomp_len = decomp_len
544 self.crc32 = crc32
545 self.hash_func = hash_func
547 if pack_type_num in DELTA_TYPES:
548 self.obj_type_num = None
549 self.obj_chunks = None
550 else:
551 self.obj_type_num = pack_type_num
552 self.obj_chunks = self.decomp_chunks
553 self.delta_base = delta_base
555 def sha(self) -> RawObjectID:
556 """Return the binary SHA of this object."""
557 if self._sha is None:
558 assert self.obj_type_num is not None and self.obj_chunks is not None
559 self._sha = obj_sha(self.obj_type_num, self.obj_chunks, self.hash_func)
560 return RawObjectID(self._sha)
562 def sha_file(self) -> ShaFile:
563 """Return a ShaFile from this object."""
564 assert self.obj_type_num is not None and self.obj_chunks is not None
565 return ShaFile.from_raw_chunks(self.obj_type_num, self.obj_chunks)
567 # Only provided for backwards compatibility with code that expects either
568 # chunks or a delta tuple.
569 def _obj(self) -> OldUnpackedObject:
570 """Return the decompressed chunks, or (delta base, delta chunks)."""
571 if self.pack_type_num in DELTA_TYPES:
572 assert isinstance(self.delta_base, bytes | int)
573 return (self.delta_base, self.decomp_chunks)
574 else:
575 return self.decomp_chunks
577 def __eq__(self, other: object) -> bool:
578 """Check equality with another UnpackedObject."""
579 if not isinstance(other, UnpackedObject):
580 return False
581 for slot in self.__slots__:
582 if getattr(self, slot) != getattr(other, slot):
583 return False
584 return True
586 def __ne__(self, other: object) -> bool:
587 """Check inequality with another UnpackedObject."""
588 return not (self == other)
590 def __repr__(self) -> str:
591 """Return string representation of this UnpackedObject."""
592 data = [f"{s}={getattr(self, s)!r}" for s in self.__slots__]
593 return "{}({})".format(self.__class__.__name__, ", ".join(data))
596_ZLIB_BUFSIZE = 65536 # 64KB buffer for better I/O performance
598# Default maximum memory for caching delta base objects (matches Git's default
599# for core.deltaBaseCacheLimit).
600DEFAULT_DELTA_BASE_CACHE_LIMIT = 96 * 1024 * 1024 # 96 MiB
603def read_zlib_chunks(
604 read_some: Callable[[int], bytes],
605 unpacked: UnpackedObject,
606 include_comp: bool = False,
607 buffer_size: int = _ZLIB_BUFSIZE,
608) -> bytes:
609 """Read zlib data from a buffer.
611 This function requires that the buffer have additional data following the
612 compressed data, which is guaranteed to be the case for git pack files.
614 Args:
615 read_some: Read function that returns at least one byte, but may
616 return less than the requested size.
617 unpacked: An UnpackedObject to write result data to. If its crc32
618 attr is not None, the CRC32 of the compressed bytes will be computed
619 using this starting CRC32.
620 After this function, will have the following attrs set:
621 * comp_chunks (if include_comp is True)
622 * decomp_chunks
623 * decomp_len
624 * crc32
625 include_comp: If True, include compressed data in the result.
626 buffer_size: Size of the read buffer.
627 Returns: Leftover unused data from the decompression.
629 Raises:
630 zlib.error: if a decompression error occurred.
631 """
632 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
633 raise ValueError("non-negative zlib data stream size expected")
634 decomp_obj = zlib.decompressobj()
636 comp_chunks = []
637 decomp_chunks = unpacked.decomp_chunks
638 decomp_len = 0
639 crc32 = unpacked.crc32
640 max_decomp = unpacked.decomp_len
642 while True:
643 add = read_some(buffer_size)
644 if not add:
645 raise zlib.error("EOF before end of zlib stream")
646 comp_chunks.append(add)
647 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
648 remaining = max_decomp - decomp_len + 1
649 decomp = decomp_obj.decompress(add, remaining)
650 if decomp_obj.unconsumed_tail:
651 raise zlib.error("decompressed data exceeds expected size")
652 decomp_len += len(decomp)
653 decomp_chunks.append(decomp)
654 unused = decomp_obj.unused_data
655 if unused:
656 left = len(unused)
657 if crc32 is not None:
658 crc32 = binascii.crc32(add[:-left], crc32)
659 if include_comp:
660 comp_chunks[-1] = add[:-left]
661 break
662 elif crc32 is not None:
663 crc32 = binascii.crc32(add, crc32)
664 if crc32 is not None:
665 crc32 &= 0xFFFFFFFF
667 if decomp_len != unpacked.decomp_len:
668 raise zlib.error("decompressed data does not match expected size")
670 unpacked.crc32 = crc32
671 if include_comp:
672 unpacked.comp_chunks = comp_chunks
673 return unused
676def read_zlib_chunks_at(
677 contents: "bytes | mmap.mmap",
678 offset: int,
679 unpacked: UnpackedObject,
680 include_comp: bool = False,
681 buffer_size: int = _ZLIB_BUFSIZE,
682) -> int:
683 """Read zlib data from a buffer at a given offset.
685 Like :func:`read_zlib_chunks`, but indexes the buffer directly instead of
686 consuming a read callable, so concurrent readers do not share a position.
688 The buffer is fed to zlib in ``buffer_size`` slices rather than in one
689 piece. That bounds ``unused_data``, which zlib materialises as a copy of
690 everything it was handed past the end of the stream: passing the whole
691 mapping would copy the entire remainder of the pack for every object read.
693 Slices are taken as memoryviews, so the compressed data is decompressed
694 straight out of the mapping. With ``include_comp`` the chunks are kept on
695 ``unpacked`` and outlive the mapping, so those are copied.
697 Args:
698 contents: Buffer holding the compressed data.
699 offset: Offset in contents at which the zlib stream starts.
700 unpacked: An UnpackedObject to write result data to; see
701 :func:`read_zlib_chunks` for the attributes set on it.
702 include_comp: If True, include compressed data in the result.
703 buffer_size: Number of bytes to feed to zlib at a time.
704 Returns: Offset in contents just past the end of the zlib stream.
706 Raises:
707 zlib.error: if a decompression error occurred.
708 """
709 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
710 raise ValueError("non-negative zlib data stream size expected")
711 decomp_obj = zlib.decompressobj()
713 comp_chunks = []
714 decomp_chunks = unpacked.decomp_chunks
715 decomp_len = 0
716 crc32 = unpacked.crc32
717 max_decomp = unpacked.decomp_len
718 pos = offset
720 with memoryview(contents) as view:
721 while True:
722 add = view[pos : pos + buffer_size]
723 if not add:
724 raise zlib.error("EOF before end of zlib stream")
725 pos += len(add)
726 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
727 remaining = max_decomp - decomp_len + 1
728 decomp = decomp_obj.decompress(add, remaining)
729 if decomp_obj.unconsumed_tail:
730 raise zlib.error("decompressed data exceeds expected size")
731 decomp_len += len(decomp)
732 decomp_chunks.append(decomp)
733 unused = decomp_obj.unused_data
734 if unused:
735 left = len(unused)
736 pos -= left
737 add = add[:-left]
738 if crc32 is not None:
739 crc32 = binascii.crc32(add, crc32)
740 if include_comp:
741 comp_chunks.append(bytes(add))
742 if unused:
743 break
744 if crc32 is not None:
745 crc32 &= 0xFFFFFFFF
747 if decomp_len != unpacked.decomp_len:
748 raise zlib.error("decompressed data does not match expected size")
750 unpacked.crc32 = crc32
751 if include_comp:
752 unpacked.comp_chunks = comp_chunks
753 return pos
756def iter_sha1(iter: Iterable[bytes]) -> bytes:
757 """Return the hexdigest of the SHA1 over a set of names.
759 Args:
760 iter: Iterator over string objects
761 Returns: 40-byte hex sha1 digest
762 """
763 sha = sha1()
764 for name in iter:
765 sha.update(name)
766 return sha.hexdigest().encode("ascii")
769def load_pack_index(
770 path: str | os.PathLike[str], object_format: ObjectFormat
771) -> "PackIndex":
772 """Load an index file by path.
774 Args:
775 path: Path to the index file
776 object_format: Hash algorithm used by the repository
777 Returns: A PackIndex loaded from the given path
778 """
779 # Ownership of the file is transferred to the returned index, which mmaps
780 # it and closes it in PackIndex.close(). It must not be closed here: on
781 # Windows an mmap keeps the file locked, so closing the handle out from
782 # under a live mapping leaves the .idx undeletable until the index is GCed.
783 f = GitFile(path, "rb")
784 try:
785 return load_pack_index_file(path, f, object_format)
786 except BaseException:
787 f.close()
788 raise
791def _load_file_contents(
792 f: IO[bytes] | _GitFile, size: int | None = None
793) -> tuple[bytes | Any, int]:
794 """Load contents from a file, preferring mmap when possible.
796 Args:
797 f: File-like object to load
798 size: Expected size, or None to determine from file
799 Returns: Tuple of (contents, size)
800 """
801 # Avoid rolling a SpooledTemporaryFile to disk just to get a descriptor.
802 if getattr(f, "_rolled", True) is False:
803 fd = None
804 else:
805 try:
806 fd = f.fileno()
807 except (UnsupportedOperation, AttributeError):
808 fd = None
809 # Attempt to use mmap if possible
810 if fd is not None:
811 if size is None:
812 size = os.fstat(fd).st_size
813 if has_mmap:
814 try:
815 contents = mmap.mmap(fd, size, access=mmap.ACCESS_READ)
816 except (OSError, ValueError):
817 # Can't mmap - perhaps a socket or invalid file descriptor
818 pass
819 else:
820 return contents, size
821 contents_bytes = f.read()
822 size = len(contents_bytes)
823 return contents_bytes, size
826def _close_file_contents(contents: "bytes | mmap.mmap | None") -> None:
827 """Close contents returned by _load_file_contents, if closeable.
829 Callers must close the mapping before the file it maps: on Windows the
830 mapping holds a lock on the file, so the handle cannot be released while
831 it is alive.
832 """
833 close_fn = getattr(contents, "close", None)
834 if close_fn is not None:
835 close_fn()
838def load_pack_index_file(
839 path: str | os.PathLike[str],
840 f: IO[bytes] | _GitFile,
841 object_format: ObjectFormat,
842) -> "PackIndex":
843 """Load an index file from a file-like object.
845 Args:
846 path: Path for the index file
847 f: File-like object
848 object_format: Hash algorithm used by the repository
849 Returns: A PackIndex loaded from the given file
850 """
851 contents, size = _load_file_contents(f)
852 if contents[:4] == b"\377tOc":
853 version = struct.unpack(b">L", contents[4:8])[0]
854 if version == 2:
855 return PackIndex2(
856 path,
857 object_format,
858 file=f,
859 contents=contents,
860 size=size,
861 )
862 elif version == 3:
863 return PackIndex3(path, object_format, file=f, contents=contents, size=size)
864 else:
865 raise KeyError(f"Unknown pack index format {version}")
866 else:
867 return PackIndex1(path, object_format, file=f, contents=contents, size=size)
870def bisect_find_sha(
871 start: int, end: int, sha: bytes, unpack_name: Callable[[int], bytes]
872) -> int | None:
873 """Find a SHA in a data blob with sorted SHAs.
875 Args:
876 start: Start index of range to search
877 end: End index of range to search
878 sha: Sha to find
879 unpack_name: Callback to retrieve SHA by index
880 Returns: Index of the SHA, or None if it wasn't found
881 """
882 assert start <= end
883 while start <= end:
884 i = (start + end) // 2
885 file_sha = unpack_name(i)
886 if file_sha < sha:
887 start = i + 1
888 elif file_sha > sha:
889 end = i - 1
890 else:
891 return i
892 return None
895PackIndexEntry = tuple[RawObjectID, int, int | None]
898class PackIndex:
899 """An index in to a packfile.
901 Given a sha id of an object a pack index can tell you the location in the
902 packfile of that object if it has it.
903 """
905 object_format: "ObjectFormat"
907 def __eq__(self, other: object) -> bool:
908 """Check equality with another PackIndex."""
909 if not isinstance(other, PackIndex):
910 return False
912 for (name1, _, _), (name2, _, _) in zip(
913 self.iterentries(), other.iterentries()
914 ):
915 if name1 != name2:
916 return False
917 return True
919 def __ne__(self, other: object) -> bool:
920 """Check if this pack index is not equal to another."""
921 return not self.__eq__(other)
923 def __len__(self) -> int:
924 """Return the number of entries in this pack index."""
925 raise NotImplementedError(self.__len__)
927 def __iter__(self) -> Iterator[ObjectID]:
928 """Iterate over the SHAs in this pack."""
929 return map(lambda sha: sha_to_hex(RawObjectID(sha)), self._itersha())
931 def iterentries(self) -> Iterator[PackIndexEntry]:
932 """Iterate over the entries in this pack index.
934 Returns: iterator over tuples with object name, offset in packfile and
935 crc32 checksum.
936 """
937 raise NotImplementedError(self.iterentries)
939 def get_pack_checksum(self) -> bytes | None:
940 """Return the SHA1 checksum stored for the corresponding packfile.
942 Returns: 20-byte binary digest, or None if not available
943 """
944 raise NotImplementedError(self.get_pack_checksum)
946 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
947 """Return the offset in to the corresponding packfile for the object.
949 Given the name of an object it will return the offset that object
950 lives at within the corresponding pack file. If the pack file doesn't
951 have the object then None will be returned.
952 """
953 raise NotImplementedError(self.object_offset)
955 def object_sha1(self, index: int) -> bytes:
956 """Return the SHA1 corresponding to the index in the pack file."""
957 for name, offset, _crc32 in self.iterentries():
958 if offset == index:
959 return name
960 else:
961 raise KeyError(index)
963 def object_sha_at_position(self, pos: int) -> RawObjectID:
964 """Return the name of the object at the given position in the index.
966 Positions count objects in the sorted order of the index, starting at
967 zero; this is how pack bitmaps refer to objects.
969 Args:
970 pos: Position of the object in the sorted index
971 Returns: Binary object name
972 Raises:
973 IndexError: If pos is out of range
974 """
975 raise NotImplementedError(self.object_sha_at_position)
977 def _object_offset(self, sha: bytes) -> int:
978 """See object_offset.
980 Args:
981 sha: A *binary* SHA string. (20 characters long)_
982 """
983 raise NotImplementedError(self._object_offset)
985 def objects_sha1(self) -> bytes:
986 """Return the hex SHA1 over all the shas of all objects in this pack.
988 Note: This is used for the filename of the pack.
989 """
990 return iter_sha1(self._itersha())
992 def _itersha(self) -> Iterator[bytes]:
993 """Yield all the SHA1's of the objects in the index, sorted."""
994 raise NotImplementedError(self._itersha)
996 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
997 """Iterate over all SHA1s with the given prefix.
999 Args:
1000 prefix: Binary prefix to match
1001 Returns: Iterator of matching SHA1s
1002 """
1003 # Default implementation for PackIndex classes that don't override
1004 for sha, _, _ in self.iterentries():
1005 if sha.startswith(prefix):
1006 yield RawObjectID(sha)
1008 def close(self) -> None:
1009 """Close any open files."""
1011 def check(self) -> None:
1012 """Check the consistency of this pack index."""
1015class MemoryPackIndex(PackIndex):
1016 """Pack index that is stored entirely in memory."""
1018 def __init__(
1019 self,
1020 entries: list[PackIndexEntry],
1021 object_format: ObjectFormat,
1022 pack_checksum: bytes | None = None,
1023 ) -> None:
1024 """Create a new MemoryPackIndex.
1026 Args:
1027 entries: Sequence of name, idx, crc32 (sorted)
1028 object_format: Object format used by this index
1029 pack_checksum: Optional pack checksum
1030 """
1031 self._by_sha = {}
1032 self._by_offset = {}
1033 for name, offset, _crc32 in entries:
1034 self._by_sha[name] = offset
1035 self._by_offset[offset] = name
1036 self._entries = entries
1037 self._pack_checksum = pack_checksum
1038 self.object_format = object_format
1040 def get_pack_checksum(self) -> bytes | None:
1041 """Return the SHA checksum stored for the corresponding packfile."""
1042 return self._pack_checksum
1044 def __len__(self) -> int:
1045 """Return the number of entries in this pack index."""
1046 return len(self._entries)
1048 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1049 """Return the offset for the given SHA.
1051 Args:
1052 sha: SHA to look up (binary or hex)
1053 Returns: Offset in the pack file
1054 """
1055 lookup_sha: RawObjectID
1056 if len(sha) == self.object_format.hex_length:
1057 lookup_sha = hex_to_sha(ObjectID(sha))
1058 else:
1059 lookup_sha = RawObjectID(sha)
1060 return self._by_sha[lookup_sha]
1062 def object_sha1(self, index: int) -> bytes:
1063 """Return the SHA1 for the object at the given offset."""
1064 return self._by_offset[index]
1066 def object_sha_at_position(self, pos: int) -> RawObjectID:
1067 """Return the name of the object at the given position in the index."""
1068 if not 0 <= pos < len(self._entries):
1069 raise IndexError(pos)
1070 return RawObjectID(self._entries[pos][0])
1072 def _itersha(self) -> Iterator[bytes]:
1073 """Iterate over all SHA1s in the index."""
1074 return iter(self._by_sha)
1076 def iterentries(self) -> Iterator[PackIndexEntry]:
1077 """Iterate over all index entries."""
1078 return iter(self._entries)
1080 @classmethod
1081 def for_pack(cls, pack_data: "PackData") -> "MemoryPackIndex":
1082 """Create a MemoryPackIndex from a PackData object."""
1083 return MemoryPackIndex(
1084 list(pack_data.sorted_entries()),
1085 pack_checksum=pack_data.get_stored_checksum(),
1086 object_format=pack_data.object_format,
1087 )
1089 @classmethod
1090 def clone(cls, other_index: "PackIndex") -> "MemoryPackIndex":
1091 """Create a copy of another PackIndex in memory."""
1092 return cls(
1093 list(other_index.iterentries()),
1094 other_index.object_format,
1095 other_index.get_pack_checksum(),
1096 )
1099class FilePackIndex(PackIndex):
1100 """Pack index that is based on a file.
1102 To do the loop it opens the file, and indexes first 256 4 byte groups
1103 with the first byte of the sha id. The value in the four byte group indexed
1104 is the end of the group that shares the same starting byte. Subtract one
1105 from the starting byte and index again to find the start of the group.
1106 The values are sorted by sha id within the group, so do the math to find
1107 the start and end offset and then bisect in to find if the value is
1108 present.
1109 """
1111 _fan_out_table: list[int]
1112 _file: IO[bytes] | _GitFile
1114 def __init__(
1115 self,
1116 filename: str | os.PathLike[str],
1117 file: IO[bytes] | _GitFile | None = None,
1118 contents: "bytes | mmap.mmap | None" = None,
1119 size: int | None = None,
1120 ) -> None:
1121 """Create a pack index object.
1123 Provide it with the name of the index file to consider, and it will map
1124 it whenever required.
1125 """
1126 self._filename = filename
1127 # Take the size now, so it can be checked each time we map the file to
1128 # ensure that it hasn't changed.
1129 if file is None:
1130 self._file = GitFile(filename, "rb")
1131 else:
1132 self._file = file
1133 if contents is None:
1134 self._contents, self._size = _load_file_contents(self._file, size)
1135 else:
1136 self._contents = contents
1137 self._size = size if size is not None else len(contents)
1139 @property
1140 def path(self) -> str:
1141 """Return the path to this index file."""
1142 return os.fspath(self._filename)
1144 def __eq__(self, other: object) -> bool:
1145 """Check equality with another FilePackIndex."""
1146 # Quick optimization:
1147 if (
1148 isinstance(other, FilePackIndex)
1149 and self._fan_out_table != other._fan_out_table
1150 ):
1151 return False
1153 return super().__eq__(other)
1155 def close(self) -> None:
1156 """Close the underlying file and any mmap."""
1157 _close_file_contents(self._contents)
1158 self._file.close()
1160 def __del__(self) -> None:
1161 """Ensure the file and mmap are closed when GCed."""
1162 if not getattr(self._file, "closed", True):
1163 import warnings
1165 warnings.warn(
1166 f"unclosed pack index {self!r}",
1167 ResourceWarning,
1168 stacklevel=2,
1169 source=self,
1170 )
1171 try:
1172 self.close()
1173 except Exception:
1174 # Ignore errors during cleanup
1175 pass
1177 def __enter__(self) -> Self:
1178 """Enter context manager."""
1179 return self
1181 def __exit__(
1182 self,
1183 type: type | None,
1184 value: BaseException | None,
1185 traceback: TracebackType | None,
1186 ) -> None:
1187 """Exit context manager."""
1188 self.close()
1190 def __len__(self) -> int:
1191 """Return the number of entries in this pack index."""
1192 return self._fan_out_table[-1]
1194 def _unpack_entry(self, i: int) -> PackIndexEntry:
1195 """Unpack the i-th entry in the index file.
1197 Returns: Tuple with object name (SHA), offset in pack file and CRC32
1198 checksum (if known).
1199 """
1200 raise NotImplementedError(self._unpack_entry)
1202 def _unpack_name(self, i: int) -> bytes:
1203 """Unpack the i-th name from the index file."""
1204 raise NotImplementedError(self._unpack_name)
1206 def _unpack_offset(self, i: int) -> int:
1207 """Unpack the i-th object offset from the index file."""
1208 raise NotImplementedError(self._unpack_offset)
1210 def _unpack_crc32_checksum(self, i: int) -> int | None:
1211 """Unpack the crc32 checksum for the i-th object from the index file."""
1212 raise NotImplementedError(self._unpack_crc32_checksum)
1214 def _itersha(self) -> Iterator[bytes]:
1215 """Iterate over all SHA1s in the index."""
1216 for i in range(len(self)):
1217 yield self._unpack_name(i)
1219 def iterentries(self) -> Iterator[PackIndexEntry]:
1220 """Iterate over the entries in this pack index.
1222 Returns: iterator over tuples with object name, offset in packfile and
1223 crc32 checksum.
1224 """
1225 for i in range(len(self)):
1226 yield self._unpack_entry(i)
1228 def object_sha_at_position(self, pos: int) -> RawObjectID:
1229 """Return the name of the object at the given position in the index."""
1230 if not 0 <= pos < len(self):
1231 raise IndexError(pos)
1232 return RawObjectID(self._unpack_name(pos))
1234 def _read_fan_out_table(self, start_offset: int) -> list[int]:
1235 """Read the fan-out table from the index.
1237 The fan-out table contains 256 entries mapping first byte values
1238 to the number of objects with SHA1s less than or equal to that byte.
1240 Args:
1241 start_offset: Offset in the file where the fan-out table starts
1242 Returns: List of 256 integers
1243 """
1244 ret = []
1245 for i in range(0x100):
1246 fanout_entry = self._contents[
1247 start_offset + i * 4 : start_offset + (i + 1) * 4
1248 ]
1249 ret.append(struct.unpack(">L", fanout_entry)[0])
1250 return ret
1252 def check(self) -> None:
1253 """Check that the stored checksum matches the actual checksum."""
1254 actual = self.calculate_checksum()
1255 stored = self.get_stored_checksum()
1256 if actual != stored:
1257 raise ChecksumMismatch(stored, actual)
1259 def calculate_checksum(self) -> bytes:
1260 """Calculate the SHA1 checksum over this pack index.
1262 Returns: This is a 20-byte binary digest
1263 """
1264 return sha1(self._contents[:-20]).digest()
1266 def get_pack_checksum(self) -> bytes:
1267 """Return the SHA1 checksum stored for the corresponding packfile.
1269 Returns: 20-byte binary digest
1270 """
1271 return bytes(self._contents[-40:-20])
1273 def get_stored_checksum(self) -> bytes:
1274 """Return the SHA1 checksum stored for this index.
1276 Returns: 20-byte binary digest
1277 """
1278 return bytes(self._contents[-20:])
1280 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1281 """Return the offset in to the corresponding packfile for the object.
1283 Given the name of an object it will return the offset that object
1284 lives at within the corresponding pack file. If the pack file doesn't
1285 have the object then None will be returned.
1286 """
1287 lookup_sha: RawObjectID
1288 if len(sha) == self.object_format.hex_length: # hex string
1289 lookup_sha = hex_to_sha(ObjectID(sha))
1290 else:
1291 lookup_sha = RawObjectID(sha)
1292 try:
1293 return self._object_offset(lookup_sha)
1294 except ValueError as exc:
1295 closed = getattr(self._contents, "closed", None)
1296 if closed in (None, True):
1297 raise PackFileDisappeared(self) from exc
1298 raise
1300 def _object_offset(self, sha: bytes) -> int:
1301 """See object_offset.
1303 Args:
1304 sha: A *binary* SHA string. (20 characters long)_
1305 """
1306 hash_size = getattr(self, "hash_size", 20) # Default to SHA1 for v1
1307 assert len(sha) == hash_size
1308 idx = ord(sha[:1])
1309 if idx == 0:
1310 start = 0
1311 else:
1312 start = self._fan_out_table[idx - 1]
1313 end = self._fan_out_table[idx]
1314 i = bisect_find_sha(start, end, sha, self._unpack_name)
1315 if i is None:
1316 raise KeyError(sha)
1317 return self._unpack_offset(i)
1319 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
1320 """Iterate over all SHA1s with the given prefix."""
1321 start = ord(prefix[:1])
1322 if start == 0:
1323 start = 0
1324 else:
1325 start = self._fan_out_table[start - 1]
1326 end = ord(prefix[:1]) + 1
1327 if end == 0x100:
1328 end = len(self)
1329 else:
1330 end = self._fan_out_table[end]
1331 assert start <= end
1332 started = False
1333 for i in range(start, end):
1334 name: bytes = self._unpack_name(i)
1335 if name.startswith(prefix):
1336 yield RawObjectID(name)
1337 started = True
1338 elif started:
1339 break
1342class PackIndex1(FilePackIndex):
1343 """Version 1 Pack Index file."""
1345 object_format = SHA1
1347 def __init__(
1348 self,
1349 filename: str | os.PathLike[str],
1350 object_format: ObjectFormat,
1351 file: IO[bytes] | _GitFile | None = None,
1352 contents: bytes | None = None,
1353 size: int | None = None,
1354 ) -> None:
1355 """Initialize a version 1 pack index.
1357 Args:
1358 filename: Path to the index file
1359 object_format: Object format used by the repository
1360 file: Optional file object
1361 contents: Optional mmap'd contents
1362 size: Optional size of the index
1363 """
1364 super().__init__(filename, file, contents, size)
1366 # PackIndex1 only supports SHA1
1367 if object_format != SHA1:
1368 raise AssertionError(
1369 f"PackIndex1 only supports SHA1, not {object_format.name}"
1370 )
1372 self.object_format = object_format
1373 self.version = 1
1374 self._fan_out_table = self._read_fan_out_table(0)
1375 self.hash_size = self.object_format.oid_length
1376 self._entry_size = 4 + self.hash_size
1378 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, None]:
1379 base_offset = (0x100 * 4) + (i * self._entry_size)
1380 offset = unpack_from(">L", self._contents, base_offset)[0]
1381 name = self._contents[base_offset + 4 : base_offset + 4 + self.hash_size]
1382 return (RawObjectID(name), offset, None)
1384 def _unpack_name(self, i: int) -> bytes:
1385 offset = (0x100 * 4) + (i * self._entry_size) + 4
1386 return self._contents[offset : offset + self.hash_size]
1388 def _unpack_offset(self, i: int) -> int:
1389 offset = (0x100 * 4) + (i * self._entry_size)
1390 return int(unpack_from(">L", self._contents, offset)[0])
1392 def _unpack_crc32_checksum(self, i: int) -> None:
1393 # Not stored in v1 index files
1394 return None
1397class PackIndex2(FilePackIndex):
1398 """Version 2 Pack Index file."""
1400 object_format = SHA1
1402 def __init__(
1403 self,
1404 filename: str | os.PathLike[str],
1405 object_format: ObjectFormat,
1406 file: IO[bytes] | _GitFile | None = None,
1407 contents: bytes | None = None,
1408 size: int | None = None,
1409 ) -> None:
1410 """Initialize a version 2 pack index.
1412 Args:
1413 filename: Path to the index file
1414 object_format: Object format used by the repository
1415 file: Optional file object
1416 contents: Optional mmap'd contents
1417 size: Optional size of the index
1418 """
1419 super().__init__(filename, file, contents, size)
1420 self.object_format = object_format
1421 if self._contents[:4] != b"\377tOc":
1422 raise AssertionError("Not a v2 pack index file")
1423 (self.version,) = unpack_from(b">L", self._contents, 4)
1424 if self.version != 2:
1425 raise AssertionError(f"Version was {self.version}")
1426 self._fan_out_table = self._read_fan_out_table(8)
1427 self.hash_size = self.object_format.oid_length
1428 self._name_table_offset = 8 + 0x100 * 4
1429 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1430 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1431 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1432 self
1433 )
1435 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1436 return (
1437 RawObjectID(self._unpack_name(i)),
1438 self._unpack_offset(i),
1439 self._unpack_crc32_checksum(i),
1440 )
1442 def _unpack_name(self, i: int) -> bytes:
1443 offset = self._name_table_offset + i * self.hash_size
1444 return self._contents[offset : offset + self.hash_size]
1446 def _unpack_offset(self, i: int) -> int:
1447 offset = self._pack_offset_table_offset + i * 4
1448 offset_val = int(unpack_from(">L", self._contents, offset)[0])
1449 if offset_val & (2**31):
1450 offset = (
1451 self._pack_offset_largetable_offset + (offset_val & (2**31 - 1)) * 8
1452 )
1453 offset_val = int(unpack_from(">Q", self._contents, offset)[0])
1454 return offset_val
1456 def _unpack_crc32_checksum(self, i: int) -> int:
1457 return int(
1458 unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1459 )
1461 def get_pack_checksum(self) -> bytes:
1462 """Return the checksum stored for the corresponding packfile.
1464 Returns: binary digest (size depends on hash algorithm)
1465 """
1466 # Index ends with: pack_checksum + index_checksum
1467 # Each checksum is hash_size bytes
1468 checksum_size = self.hash_size
1469 return bytes(self._contents[-2 * checksum_size : -checksum_size])
1471 def get_stored_checksum(self) -> bytes:
1472 """Return the checksum stored for this index.
1474 Returns: binary digest (size depends on hash algorithm)
1475 """
1476 checksum_size = self.hash_size
1477 return bytes(self._contents[-checksum_size:])
1479 def calculate_checksum(self) -> bytes:
1480 """Calculate the checksum over this pack index.
1482 Returns: binary digest (size depends on hash algorithm)
1483 """
1484 # Determine hash function based on hash_size
1485 if self.hash_size == 20:
1486 hash_func = sha1
1487 elif self.hash_size == 32:
1488 hash_func = sha256
1489 else:
1490 raise ValueError(f"Unsupported hash size: {self.hash_size}")
1492 return hash_func(self._contents[: -self.hash_size]).digest()
1495class PackIndex3(FilePackIndex):
1496 """Version 3 Pack Index file.
1498 Supports variable hash sizes for SHA-1 (20 bytes) and SHA-256 (32 bytes).
1499 """
1501 def __init__(
1502 self,
1503 filename: str | os.PathLike[str],
1504 object_format: ObjectFormat,
1505 file: IO[bytes] | _GitFile | None = None,
1506 contents: bytes | None = None,
1507 size: int | None = None,
1508 ) -> None:
1509 """Initialize a version 3 pack index.
1511 Args:
1512 filename: Path to the index file
1513 object_format: Object format used by the repository
1514 file: Optional file object
1515 contents: Optional mmap'd contents
1516 size: Optional size of the index
1517 """
1518 super().__init__(filename, file, contents, size)
1519 if self._contents[:4] != b"\377tOc":
1520 raise AssertionError("Not a v3 pack index file")
1521 (self.version,) = unpack_from(b">L", self._contents, 4)
1522 if self.version != 3:
1523 raise AssertionError(f"Version was {self.version}")
1525 # Read hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
1526 (self.hash_format,) = unpack_from(b">L", self._contents, 8)
1527 file_object_format = OBJECT_FORMAT_TYPE_NUMS[self.hash_format]
1529 # Verify provided object_format matches what's in the file
1530 if object_format != file_object_format:
1531 raise AssertionError(
1532 f"Object format mismatch: provided {object_format.name}, "
1533 f"but file contains {file_object_format.name}"
1534 )
1536 self.object_format = object_format
1537 self.hash_size = self.object_format.oid_length
1539 # Read length of shortened object names
1540 (self.shortened_oid_len,) = unpack_from(b">L", self._contents, 12)
1542 # Calculate offsets based on variable hash size
1543 self._fan_out_table = self._read_fan_out_table(
1544 16
1545 ) # After header (4 + 4 + 4 + 4)
1546 self._name_table_offset = 16 + 0x100 * 4
1547 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1548 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1549 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1550 self
1551 )
1553 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1554 return (
1555 RawObjectID(self._unpack_name(i)),
1556 self._unpack_offset(i),
1557 self._unpack_crc32_checksum(i),
1558 )
1560 def _unpack_name(self, i: int) -> bytes:
1561 offset = self._name_table_offset + i * self.hash_size
1562 return self._contents[offset : offset + self.hash_size]
1564 def _unpack_offset(self, i: int) -> int:
1565 offset_pos = self._pack_offset_table_offset + i * 4
1566 offset = unpack_from(">L", self._contents, offset_pos)[0]
1567 assert isinstance(offset, int)
1568 if offset & (2**31):
1569 large_offset_pos = (
1570 self._pack_offset_largetable_offset + (offset & (2**31 - 1)) * 8
1571 )
1572 offset = unpack_from(">Q", self._contents, large_offset_pos)[0]
1573 assert isinstance(offset, int)
1574 return offset
1576 def _unpack_crc32_checksum(self, i: int) -> int:
1577 result = unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1578 assert isinstance(result, int)
1579 return result
1582def read_pack_header_at(
1583 contents: "bytes | mmap.mmap", offset: int = 0
1584) -> tuple[int, int]:
1585 """Read the header of a pack file from a buffer.
1587 Args:
1588 contents: Buffer holding the pack
1589 offset: Offset in contents at which the header starts
1590 Returns: Tuple of (pack version, number of objects).
1591 """
1592 header = contents[offset : offset + 12]
1593 if not header:
1594 raise AssertionError("file too short to contain pack")
1595 if header[:4] != b"PACK":
1596 raise AssertionError(f"Invalid pack header {bytes(header)!r}")
1597 (version,) = unpack_from(b">L", header, 4)
1598 if version not in (2, 3):
1599 raise AssertionError(f"Version was {version}")
1600 (num_objects,) = unpack_from(b">L", header, 8)
1601 return (version, num_objects)
1604def read_pack_header(read: Callable[[int], bytes]) -> tuple[int, int]:
1605 """Read the header of a pack file.
1607 Args:
1608 read: Read function
1609 Returns: Tuple of (pack version, number of objects).
1610 """
1611 return read_pack_header_at(read(12))
1614def chunks_length(chunks: bytes | Iterable[bytes]) -> int:
1615 """Get the total length of a sequence of chunks.
1617 Args:
1618 chunks: Either a single bytes object or an iterable of bytes
1619 Returns: Total length in bytes
1620 """
1621 if isinstance(chunks, bytes):
1622 return len(chunks)
1623 else:
1624 return sum(map(len, chunks))
1627def _decode_object_header(raw: list[int]) -> tuple[int, int]:
1628 """Decode an object type and size from a pack object header."""
1629 type_num = (raw[0] >> 4) & 0x07
1630 size = raw[0] & 0x0F
1631 for i, byte in enumerate(raw[1:]):
1632 size += (byte & 0x7F) << ((i * 7) + 4)
1633 return type_num, size
1636def _decode_delta_base_offset(raw: list[int]) -> int:
1637 """Decode an OFS_DELTA base offset from its variable-length encoding."""
1638 if raw[-1] & 0x80:
1639 raise AssertionError
1640 delta_base_offset = raw[0] & 0x7F
1641 for byte in raw[1:]:
1642 delta_base_offset += 1
1643 delta_base_offset <<= 7
1644 delta_base_offset += byte & 0x7F
1645 if delta_base_offset == 0:
1646 # A zero offset makes the delta reference itself, which would
1647 # loop forever in resolve_object. git's C client rejects this
1648 # with "delta offset == 0 is invalid".
1649 raise ApplyDeltaError("OFS_DELTA has delta_base_offset of 0")
1650 return delta_base_offset
1653def unpack_object(
1654 read_all: Callable[[int], bytes],
1655 hash_func: Callable[[], "HashObject"],
1656 read_some: Callable[[int], bytes] | None = None,
1657 compute_crc32: bool = False,
1658 include_comp: bool = False,
1659 zlib_bufsize: int = _ZLIB_BUFSIZE,
1660) -> tuple[UnpackedObject, bytes]:
1661 """Unpack a Git object.
1663 Args:
1664 read_all: Read function that blocks until the number of requested
1665 bytes are read.
1666 hash_func: Hash function to use for computing object IDs.
1667 read_some: Read function that returns at least one byte, but may not
1668 return the number of bytes requested.
1669 compute_crc32: If True, compute the CRC32 of the compressed data. If
1670 False, the returned CRC32 will be None.
1671 include_comp: If True, include compressed data in the result.
1672 zlib_bufsize: An optional buffer size for zlib operations.
1673 Returns: A tuple of (unpacked, unused), where unused is the unused data
1674 leftover from decompression, and unpacked in an UnpackedObject with
1675 the following attrs set:
1677 * obj_chunks (for non-delta types)
1678 * pack_type_num
1679 * delta_base (for delta types)
1680 * comp_chunks (if include_comp is True)
1681 * decomp_chunks
1682 * decomp_len
1683 * crc32 (if compute_crc32 is True)
1684 """
1685 if read_some is None:
1686 read_some = read_all
1687 if compute_crc32:
1688 crc32 = 0
1689 else:
1690 crc32 = None
1692 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1693 type_num, size = _decode_object_header(raw)
1695 delta_base: int | bytes | None
1696 raw_base = len(raw)
1697 if type_num == OFS_DELTA:
1698 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1699 raw_base += len(raw)
1700 delta_base = _decode_delta_base_offset(raw)
1701 elif type_num == REF_DELTA:
1702 # Determine hash size from hash_func
1703 hash_size = len(hash_func().digest())
1704 delta_base_obj = read_all(hash_size)
1705 if crc32 is not None:
1706 crc32 = binascii.crc32(delta_base_obj, crc32)
1707 delta_base = delta_base_obj
1708 raw_base += hash_size
1709 else:
1710 delta_base = None
1712 unpacked = UnpackedObject(
1713 type_num,
1714 delta_base=delta_base,
1715 decomp_len=size,
1716 crc32=crc32,
1717 hash_func=hash_func,
1718 )
1719 unused = read_zlib_chunks(
1720 read_some,
1721 unpacked,
1722 buffer_size=zlib_bufsize,
1723 include_comp=include_comp,
1724 )
1725 return unpacked, unused
1728def unpack_object_at(
1729 contents: "bytes | mmap.mmap",
1730 offset: int,
1731 hash_func: Callable[[], "HashObject"],
1732 compute_crc32: bool = False,
1733 include_comp: bool = False,
1734 zlib_bufsize: int = _ZLIB_BUFSIZE,
1735) -> tuple[UnpackedObject, int]:
1736 """Unpack a Git object from a buffer at a given offset.
1738 Like :func:`unpack_object`, but indexes the buffer directly rather than
1739 consuming a read callable, so any number of readers can work on the same
1740 buffer concurrently.
1742 Args:
1743 contents: Buffer holding the pack.
1744 offset: Offset in contents at which the object starts.
1745 hash_func: Hash function to use for computing object IDs.
1746 compute_crc32: If True, compute the CRC32 of the compressed data.
1747 include_comp: If True, include compressed data in the result.
1748 zlib_bufsize: An optional buffer size for zlib operations.
1749 Returns: A tuple of (unpacked, end), where end is the offset just past
1750 the object and unpacked is an UnpackedObject with its ``offset`` set;
1751 see :func:`unpack_object` for the other attributes.
1752 """
1753 crc32: int | None = 0 if compute_crc32 else None
1755 raw, pos, crc32 = take_msb_bytes_at(contents, offset, crc32=crc32)
1756 type_num, size = _decode_object_header(raw)
1758 delta_base: int | bytes | None
1759 if type_num == OFS_DELTA:
1760 raw, pos, crc32 = take_msb_bytes_at(contents, pos, crc32=crc32)
1761 delta_base = _decode_delta_base_offset(raw)
1762 elif type_num == REF_DELTA:
1763 hash_size = len(hash_func().digest())
1764 delta_base_obj = bytes(contents[pos : pos + hash_size])
1765 if len(delta_base_obj) != hash_size:
1766 raise AssertionError(f"unexpected end of pack data at {pos}")
1767 pos += hash_size
1768 if crc32 is not None:
1769 crc32 = binascii.crc32(delta_base_obj, crc32)
1770 delta_base = delta_base_obj
1771 else:
1772 delta_base = None
1774 unpacked = UnpackedObject(
1775 type_num,
1776 delta_base=delta_base,
1777 decomp_len=size,
1778 crc32=crc32,
1779 hash_func=hash_func,
1780 )
1781 unpacked.offset = offset
1782 end = read_zlib_chunks_at(
1783 contents,
1784 pos,
1785 unpacked,
1786 buffer_size=zlib_bufsize,
1787 include_comp=include_comp,
1788 )
1789 return unpacked, end
1792def _compute_object_size(value: tuple[int, Any]) -> int:
1793 """Compute the size of a unresolved object for use with LRUSizeCache."""
1794 (num, obj) = value
1795 if num in DELTA_TYPES:
1796 return chunks_length(obj[1])
1797 return chunks_length(obj)
1800class PackStreamReader:
1801 """Class to read a pack stream.
1803 The pack is read from a ReceivableProtocol using read() or recv() as
1804 appropriate.
1805 """
1807 def __init__(
1808 self,
1809 hash_func: Callable[[], "HashObject"],
1810 read_all: Callable[[int], bytes],
1811 read_some: Callable[[int], bytes] | None = None,
1812 zlib_bufsize: int = _ZLIB_BUFSIZE,
1813 ) -> None:
1814 """Initialize pack stream reader.
1816 Args:
1817 hash_func: Hash function to use for computing object IDs
1818 read_all: Function to read all requested bytes
1819 read_some: Function to read some bytes (optional)
1820 zlib_bufsize: Buffer size for zlib decompression
1821 """
1822 self.read_all = read_all
1823 if read_some is None:
1824 self.read_some = read_all
1825 else:
1826 self.read_some = read_some
1827 self.hash_func = hash_func
1828 self.sha = hash_func()
1829 self._hash_size = len(hash_func().digest())
1830 self._offset = 0
1831 self._rbuf = BytesIO()
1832 # trailer is a deque to avoid memory allocation on small reads
1833 self._trailer: deque[int] = deque()
1834 self._zlib_bufsize = zlib_bufsize
1836 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1837 """Read up to size bytes using the given callback.
1839 As a side effect, update the verifier's hash (excluding the last
1840 hash_size bytes read, which is the pack checksum).
1842 Args:
1843 read: The read callback to read from.
1844 size: The maximum number of bytes to read; the particular
1845 behavior is callback-specific.
1846 Returns: Bytes read
1847 """
1848 data = read(size)
1850 # maintain a trailer of the last hash_size bytes we've read
1851 n = len(data)
1852 self._offset += n
1853 tn = len(self._trailer)
1854 if n >= self._hash_size:
1855 to_pop = tn
1856 to_add = self._hash_size
1857 else:
1858 to_pop = max(n + tn - self._hash_size, 0)
1859 to_add = n
1860 self.sha.update(
1861 bytes(bytearray([self._trailer.popleft() for _ in range(to_pop)]))
1862 )
1863 self._trailer.extend(data[-to_add:])
1865 # hash everything but the trailer
1866 self.sha.update(data[:-to_add])
1867 return data
1869 def _buf_len(self) -> int:
1870 buf = self._rbuf
1871 start = buf.tell()
1872 buf.seek(0, SEEK_END)
1873 end = buf.tell()
1874 buf.seek(start)
1875 return end - start
1877 @property
1878 def offset(self) -> int:
1879 """Return current offset in the stream."""
1880 return self._offset - self._buf_len()
1882 def read(self, size: int) -> bytes:
1883 """Read, blocking until size bytes are read."""
1884 buf_len = self._buf_len()
1885 if buf_len >= size:
1886 return self._rbuf.read(size)
1887 buf_data = self._rbuf.read()
1888 self._rbuf = BytesIO()
1889 return buf_data + self._read(self.read_all, size - buf_len)
1891 def recv(self, size: int) -> bytes:
1892 """Read up to size bytes, blocking until one byte is read."""
1893 buf_len = self._buf_len()
1894 if buf_len:
1895 data = self._rbuf.read(size)
1896 if size >= buf_len:
1897 self._rbuf = BytesIO()
1898 return data
1899 return self._read(self.read_some, size)
1901 def __len__(self) -> int:
1902 """Return the number of objects in this pack."""
1903 return self._num_objects
1905 def read_objects(self, compute_crc32: bool = False) -> Iterator[UnpackedObject]:
1906 """Read the objects in this pack file.
1908 Args:
1909 compute_crc32: If True, compute the CRC32 of the compressed
1910 data. If False, the returned CRC32 will be None.
1911 Returns: Iterator over UnpackedObjects with the following members set:
1912 offset
1913 obj_type_num
1914 obj_chunks (for non-delta types)
1915 delta_base (for delta types)
1916 decomp_chunks
1917 decomp_len
1918 crc32 (if compute_crc32 is True)
1920 Raises:
1921 ChecksumMismatch: if the checksum of the pack contents does not
1922 match the checksum in the pack trailer.
1923 zlib.error: if an error occurred during zlib decompression.
1924 IOError: if an error occurred writing to the output file.
1925 """
1926 _pack_version, self._num_objects = read_pack_header(self.read)
1928 for _ in range(self._num_objects):
1929 offset = self.offset
1930 unpacked, unused = unpack_object(
1931 self.read,
1932 self.hash_func,
1933 read_some=self.recv,
1934 compute_crc32=compute_crc32,
1935 zlib_bufsize=self._zlib_bufsize,
1936 )
1937 unpacked.offset = offset
1939 # prepend any unused data to current read buffer
1940 buf = BytesIO()
1941 buf.write(unused)
1942 buf.write(self._rbuf.read())
1943 buf.seek(0)
1944 self._rbuf = buf
1946 yield unpacked
1948 if self._buf_len() < self._hash_size:
1949 # If the read buffer is full, then the last read() got the whole
1950 # trailer off the wire. If not, it means there is still some of the
1951 # trailer to read. We need to read() all hash_size bytes; N come from the
1952 # read buffer and (hash_size - N) come from the wire.
1953 self.read(self._hash_size)
1955 pack_sha = bytearray(self._trailer)
1956 if pack_sha != self.sha.digest():
1957 raise ChecksumMismatch(
1958 sha_to_hex(RawObjectID(bytes(pack_sha))), self.sha.hexdigest()
1959 )
1962class PackStreamCopier(PackStreamReader):
1963 """Class to verify a pack stream as it is being read.
1965 The pack is read from a ReceivableProtocol using read() or recv() as
1966 appropriate and written out to the given file-like object.
1967 """
1969 def __init__(
1970 self,
1971 hash_func: Callable[[], "HashObject"],
1972 read_all: Callable[[int], bytes],
1973 read_some: Callable[[int], bytes] | None,
1974 outfile: IO[bytes],
1975 delta_iter: "DeltaChainIterator[UnpackedObject] | None" = None,
1976 ) -> None:
1977 """Initialize the copier.
1979 Args:
1980 hash_func: Hash function to use for computing object IDs
1981 read_all: Read function that blocks until the number of
1982 requested bytes are read.
1983 read_some: Read function that returns at least one byte, but may
1984 not return the number of bytes requested.
1985 outfile: File-like object to write output through.
1986 delta_iter: Optional DeltaChainIterator to record deltas as we
1987 read them.
1988 """
1989 super().__init__(hash_func, read_all, read_some=read_some)
1990 self.outfile = outfile
1991 self._delta_iter = delta_iter
1993 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1994 """Read data from the read callback and write it to the file."""
1995 data = super()._read(read, size)
1996 self.outfile.write(data)
1997 return data
1999 def verify(self, progress: Callable[..., None] | None = None) -> None:
2000 """Verify a pack stream and write it to the output file.
2002 See PackStreamReader.iterobjects for a list of exceptions this may
2003 throw.
2004 """
2005 i = 0 # default count of entries if read_objects() is empty
2006 for i, unpacked in enumerate(self.read_objects()):
2007 if self._delta_iter:
2008 self._delta_iter.record(unpacked)
2009 if progress is not None:
2010 progress(f"copying pack entries: {i}/{len(self)}\r".encode("ascii"))
2011 if progress is not None:
2012 progress(f"copied {i} pack entries\n".encode("ascii"))
2015def obj_sha(
2016 type: int,
2017 chunks: bytes | Iterable[bytes],
2018 hash_func: Callable[[], "HashObject"] = sha1,
2019) -> bytes:
2020 """Compute the SHA for a numeric type and object chunks.
2022 Args:
2023 type: Object type number
2024 chunks: Object data chunks
2025 hash_func: Hash function to use (defaults to sha1)
2027 Returns:
2028 Binary hash digest
2029 """
2030 sha = hash_func()
2031 sha.update(object_header(type, chunks_length(chunks)))
2032 if isinstance(chunks, bytes):
2033 sha.update(chunks)
2034 else:
2035 for chunk in chunks:
2036 sha.update(chunk)
2037 return sha.digest()
2040def compute_file_sha(
2041 f: IO[bytes],
2042 hash_func: Callable[[], "HashObject"],
2043 start_ofs: int = 0,
2044 end_ofs: int = 0,
2045 buffer_size: int = 1 << 16,
2046) -> "HashObject":
2047 """Hash a portion of a file into a new SHA.
2049 Args:
2050 f: A file-like object to read from that supports seek().
2051 hash_func: A callable that returns a new HashObject.
2052 start_ofs: The offset in the file to start reading at.
2053 end_ofs: The offset in the file to end reading at, relative to the
2054 end of the file.
2055 buffer_size: A buffer size for reading.
2056 Returns: A new SHA object updated with data read from the file.
2057 """
2058 sha = hash_func()
2059 f.seek(0, SEEK_END)
2060 length = f.tell()
2061 if start_ofs < 0:
2062 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
2063 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
2064 raise AssertionError(
2065 f"Attempt to read beyond file length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, file length: {length}"
2066 )
2067 todo = length + end_ofs - start_ofs
2068 f.seek(start_ofs)
2069 while todo:
2070 data = f.read(min(todo, buffer_size))
2071 sha.update(data)
2072 todo -= len(data)
2073 return sha
2076def compute_buffer_sha(
2077 contents: "bytes | mmap.mmap",
2078 hash_func: Callable[[], "HashObject"],
2079 start_ofs: int = 0,
2080 end_ofs: int = 0,
2081) -> "HashObject":
2082 """Hash a portion of a buffer into a new SHA.
2084 The region is hashed in one pass through a memoryview, so a mapped pack
2085 is never copied. The view is released before returning rather than left
2086 to the garbage collector, since ``mmap.close()`` raises BufferError while
2087 an export is alive.
2089 Args:
2090 contents: Buffer to hash.
2091 hash_func: A callable that returns a new HashObject.
2092 start_ofs: The offset in the buffer to start hashing at.
2093 end_ofs: The offset to end hashing at, relative to the end of the
2094 buffer.
2095 Returns: A new SHA object updated with data read from the buffer.
2096 """
2097 sha = hash_func()
2098 length = len(contents)
2099 if start_ofs < 0:
2100 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
2101 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
2102 raise AssertionError(
2103 f"Attempt to read beyond buffer length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, buffer length: {length}"
2104 )
2105 with memoryview(contents) as view:
2106 sha.update(view[start_ofs : length + end_ofs])
2107 return sha
2110class PackData:
2111 """The data contained in a packfile.
2113 Pack files can be accessed both sequentially for exploding a pack, and
2114 directly with the help of an index to retrieve a specific object.
2116 The objects within are either complete or a delta against another.
2118 The header is variable length. If the MSB of each byte is set then it
2119 indicates that the subsequent byte is still part of the header.
2120 For the first byte the next MS bits are the type, which tells you the type
2121 of object, and whether it is a delta. The LS byte is the lowest bits of the
2122 size. For each subsequent byte the LS 7 bits are the next MS bits of the
2123 size, i.e. the last byte of the header contains the MS bits of the size.
2125 For the complete objects the data is stored as zlib deflated data.
2126 The size in the header is the uncompressed object size, so to uncompress
2127 you need to just keep feeding data to zlib until you get an object back,
2128 or it errors on bad data. This is done here by reading from the mapped
2129 pack contents starting at the deflated object.
2131 Currently there are no integrity checks done. Also no attempt is made to
2132 try and detect the delta case, or a request for an object at the wrong
2133 position. It will all just throw a zlib or KeyError.
2134 """
2136 def __init__(
2137 self,
2138 filename: str | os.PathLike[str],
2139 object_format: ObjectFormat,
2140 file: IO[bytes] | None = None,
2141 size: int | None = None,
2142 *,
2143 delta_window_size: int | None = None,
2144 window_memory: int | None = None,
2145 delta_cache_size: int | None = None,
2146 depth: int | None = None,
2147 threads: int | None = None,
2148 big_file_threshold: int | None = None,
2149 delta_base_cache_limit: int | None = None,
2150 ) -> None:
2151 """Create a PackData object representing the pack in the given filename.
2153 The file must exist and stay readable until the object is disposed of.
2154 It must also stay the same size. It is mapped into memory on open, so
2155 reads index the mapping directly rather than sharing a file position.
2157 The size argument is not trusted for checksum offsets, which are
2158 derived from the mapped length instead. When given it is checked
2159 against that length, so a caller passing a stale size gets an error
2160 rather than silently wrong offsets.
2161 """
2162 self._filename = filename
2163 self.object_format = object_format
2164 self._header_size = 12
2165 self.delta_window_size = delta_window_size
2166 self.window_memory = window_memory
2167 self.delta_cache_size = delta_cache_size
2168 self.depth = depth
2169 self.threads = threads
2170 self.big_file_threshold = big_file_threshold
2171 self.delta_base_cache_limit = delta_base_cache_limit
2172 self._file: IO[bytes]
2173 self._contents: bytes | mmap.mmap | None = None
2175 if file is None:
2176 self._file = GitFile(self._filename, "rb")
2177 self._close_file = True
2178 else:
2179 # A caller-supplied file stays the caller's to close; it may well
2180 # keep writing to it after we are done reading.
2181 self._file = file
2182 self._close_file = False
2183 try:
2184 # Map the pack once; every read indexes this buffer at an explicit
2185 # offset, so concurrent reads never contend on a file position.
2186 self._contents, self._size = _load_file_contents(self._file)
2187 if size is not None and size != self._size:
2188 raise AssertionError(
2189 f"{self._filename} is {self._size} bytes, but caller said {size}"
2190 )
2191 minimum_size = self._header_size + self.object_format.oid_length
2192 if self._size < minimum_size:
2193 raise AssertionError(
2194 f"{self._filename} is too small for a packfile ({self._size} < {minimum_size})"
2195 )
2196 (_version, self._num_objects) = read_pack_header_at(self._contents)
2198 # Use delta_base_cache_limit, then delta_cache_size, then default
2199 cache_size = (
2200 delta_base_cache_limit
2201 or delta_cache_size
2202 or DEFAULT_DELTA_BASE_CACHE_LIMIT
2203 )
2204 self._init_offset_cache(cache_size)
2205 except BaseException:
2206 self.close()
2207 raise
2209 def _init_offset_cache(self, max_size: int) -> None:
2210 """Initialize the resolved object cache."""
2211 self._offset_cache = LRUSizeCache[int, tuple[int, OldUnpackedObject]](
2212 max_size, compute_size=_compute_object_size
2213 )
2214 # Cache hits update the LRU linked list, so reads need locking too.
2215 self._offset_cache_lock = threading.Lock()
2217 @property
2218 def filename(self) -> str:
2219 """Get the filename of the pack file.
2221 Returns:
2222 Base filename without directory path
2223 """
2224 return os.path.basename(self._filename)
2226 @property
2227 def path(self) -> str | os.PathLike[str]:
2228 """Get the full path of the pack file.
2230 Returns:
2231 Full path to the pack file
2232 """
2233 return self._filename
2235 @classmethod
2236 def from_file(
2237 cls,
2238 file: IO[bytes],
2239 object_format: ObjectFormat,
2240 size: int | None = None,
2241 ) -> "PackData":
2242 """Create a PackData object from an open file.
2244 Args:
2245 file: Open file object
2246 object_format: Object format
2247 size: Optional expected file size, checked against the mapped length
2249 Returns:
2250 PackData instance
2251 """
2252 return cls(str(file), object_format, file=file, size=size)
2254 @classmethod
2255 def from_path(
2256 cls,
2257 path: str | os.PathLike[str],
2258 object_format: ObjectFormat,
2259 ) -> "PackData":
2260 """Create a PackData object from a file path.
2262 Args:
2263 path: Path to the pack file
2264 object_format: Object format
2266 Returns:
2267 PackData instance
2268 """
2269 return cls(filename=path, object_format=object_format)
2271 def _buffer(self) -> "bytes | mmap.mmap":
2272 """Return the mapped pack contents."""
2273 contents = self._contents
2274 if contents is None:
2275 raise ValueError(f"read from closed PackData: {self._filename}")
2276 return contents
2278 def close(self) -> None:
2279 """Release the mapping, and the pack file if we opened it.
2281 Callers must drop the mapping before writing to or renaming the pack:
2282 on Windows a live mapping locks the file.
2283 """
2284 contents = self._contents
2285 self._contents = None
2286 _close_file_contents(contents)
2287 if self._file is not None:
2288 if self._close_file:
2289 self._file.close()
2290 self._file = None # type: ignore
2292 def __del__(self) -> None:
2293 """Ensure pack file is closed when PackData is garbage collected."""
2294 if getattr(self, "_file", None) is not None:
2295 import warnings
2297 warnings.warn(
2298 f"unclosed PackData {self!r}",
2299 ResourceWarning,
2300 stacklevel=2,
2301 source=self,
2302 )
2303 try:
2304 self.close()
2305 except Exception:
2306 # Ignore errors during cleanup
2307 pass
2309 def __enter__(self) -> Self:
2310 """Enter context manager."""
2311 return self
2313 def __exit__(
2314 self,
2315 type: type | None,
2316 value: BaseException | None,
2317 traceback: TracebackType | None,
2318 ) -> None:
2319 """Exit context manager."""
2320 self.close()
2322 def __eq__(self, other: object) -> bool:
2323 """Check equality with another object."""
2324 if isinstance(other, PackData):
2325 return self.get_stored_checksum() == other.get_stored_checksum()
2326 return False
2328 def __len__(self) -> int:
2329 """Returns the number of objects in this pack."""
2330 return self._num_objects
2332 def calculate_checksum(self) -> bytes:
2333 """Calculate the checksum for this pack.
2335 Returns: Binary digest (size depends on hash algorithm)
2336 """
2337 return compute_buffer_sha(
2338 self._buffer(),
2339 hash_func=self.object_format.hash_func,
2340 end_ofs=-self.object_format.oid_length,
2341 ).digest()
2343 def iter_unpacked(self, *, include_comp: bool = False) -> Iterator[UnpackedObject]:
2344 """Iterate over unpacked objects in the pack."""
2345 if self._num_objects is None:
2346 return
2348 contents = self._buffer()
2349 offset = self._header_size
2350 for _ in range(self._num_objects):
2351 unpacked, offset = unpack_object_at(
2352 contents,
2353 offset,
2354 self.object_format.hash_func,
2355 compute_crc32=False,
2356 include_comp=include_comp,
2357 )
2358 yield unpacked
2360 def iterentries(
2361 self,
2362 progress: Callable[[int, int], None] | None = None,
2363 resolve_ext_ref: ResolveExtRefFn | None = None,
2364 ) -> Iterator[PackIndexEntry]:
2365 """Yield entries summarizing the contents of this pack.
2367 Args:
2368 progress: Progress function, called with current and total
2369 object count.
2370 resolve_ext_ref: Optional function to resolve external references
2371 Returns: iterator of tuples with (sha, offset, crc32)
2372 """
2373 num_objects = self._num_objects
2374 indexer = PackIndexer.for_pack_data(self, resolve_ext_ref=resolve_ext_ref)
2375 for i, result in enumerate(indexer):
2376 if progress is not None:
2377 progress(i, num_objects)
2378 yield result
2380 def sorted_entries(
2381 self,
2382 progress: Callable[[int, int], None] | None = None,
2383 resolve_ext_ref: ResolveExtRefFn | None = None,
2384 ) -> list[tuple[RawObjectID, int, int]]:
2385 """Return entries in this pack, sorted by SHA.
2387 Args:
2388 progress: Progress function, called with current and total
2389 object count
2390 resolve_ext_ref: Optional function to resolve external references
2391 Returns: Iterator of tuples with (sha, offset, crc32)
2392 """
2393 return sorted(
2394 self.iterentries(progress=progress, resolve_ext_ref=resolve_ext_ref) # type: ignore
2395 )
2397 def create_index_v1(
2398 self,
2399 filename: str,
2400 progress: Callable[..., None] | None = None,
2401 resolve_ext_ref: ResolveExtRefFn | None = None,
2402 ) -> bytes:
2403 """Create a version 1 file for this data file.
2405 Args:
2406 filename: Index filename.
2407 progress: Progress report function
2408 resolve_ext_ref: Optional function to resolve external references
2409 Returns: Checksum of index file
2410 """
2411 entries = self.sorted_entries(
2412 progress=progress, resolve_ext_ref=resolve_ext_ref
2413 )
2414 checksum = self.calculate_checksum()
2415 with GitFile(filename, "wb") as f:
2416 write_pack_index_v1(
2417 f,
2418 entries,
2419 checksum,
2420 )
2421 return checksum
2423 def create_index_v2(
2424 self,
2425 filename: str,
2426 progress: Callable[..., None] | None = None,
2427 resolve_ext_ref: ResolveExtRefFn | None = None,
2428 ) -> bytes:
2429 """Create a version 2 index file for this data file.
2431 Args:
2432 filename: Index filename.
2433 progress: Progress report function
2434 resolve_ext_ref: Optional function to resolve external references
2435 Returns: Checksum of index file
2436 """
2437 entries = self.sorted_entries(
2438 progress=progress, resolve_ext_ref=resolve_ext_ref
2439 )
2440 with GitFile(filename, "wb") as f:
2441 return write_pack_index_v2(f, entries, self.calculate_checksum())
2443 def create_index_v3(
2444 self,
2445 filename: str,
2446 progress: Callable[..., None] | None = None,
2447 resolve_ext_ref: ResolveExtRefFn | None = None,
2448 hash_format: int | None = None,
2449 ) -> bytes:
2450 """Create a version 3 index file for this data file.
2452 Args:
2453 filename: Index filename.
2454 progress: Progress report function
2455 resolve_ext_ref: Function to resolve external references
2456 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
2457 Returns: Checksum of index file
2458 """
2459 entries = self.sorted_entries(
2460 progress=progress, resolve_ext_ref=resolve_ext_ref
2461 )
2462 with GitFile(filename, "wb") as f:
2463 if hash_format is None:
2464 hash_format = 1 # Default to SHA-1
2465 return write_pack_index_v3(
2466 f, entries, self.calculate_checksum(), hash_format=hash_format
2467 )
2469 def create_index(
2470 self,
2471 filename: str,
2472 progress: Callable[..., None] | None = None,
2473 version: int = 2,
2474 resolve_ext_ref: ResolveExtRefFn | None = None,
2475 hash_format: int | None = None,
2476 ) -> bytes:
2477 """Create an index file for this data file.
2479 Args:
2480 filename: Index filename.
2481 progress: Progress report function
2482 version: Index version (1, 2, or 3)
2483 resolve_ext_ref: Function to resolve external references
2484 hash_format: Hash algorithm identifier for v3 (1 = SHA-1, 2 = SHA-256)
2485 Returns: Checksum of index file
2486 """
2487 match version:
2488 case 1:
2489 return self.create_index_v1(
2490 filename, progress, resolve_ext_ref=resolve_ext_ref
2491 )
2492 case 2:
2493 return self.create_index_v2(
2494 filename, progress, resolve_ext_ref=resolve_ext_ref
2495 )
2496 case 3:
2497 return self.create_index_v3(
2498 filename,
2499 progress,
2500 resolve_ext_ref=resolve_ext_ref,
2501 hash_format=hash_format,
2502 )
2503 case _:
2504 raise ValueError(f"unknown index format {version}")
2506 def get_stored_checksum(self) -> bytes:
2507 """Return the expected checksum stored in this pack."""
2508 checksum_size = self.object_format.oid_length
2509 return bytes(self._buffer()[self._size - checksum_size :])
2511 def check(self) -> None:
2512 """Check the consistency of this pack."""
2513 actual = self.calculate_checksum()
2514 stored = self.get_stored_checksum()
2515 if actual != stored:
2516 raise ChecksumMismatch(stored, actual)
2518 def get_unpacked_object_at(
2519 self, offset: int, *, include_comp: bool = False
2520 ) -> UnpackedObject:
2521 """Given offset in the packfile return a UnpackedObject."""
2522 assert offset >= self._header_size
2523 unpacked, _ = unpack_object_at(
2524 self._buffer(),
2525 offset,
2526 self.object_format.hash_func,
2527 include_comp=include_comp,
2528 )
2529 return unpacked
2531 def _get_cached_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2532 """Return the cached object at offset, or raise KeyError."""
2533 # Hot path: acquire/release directly rather than using the context
2534 # manager, which measurably speeds up cache hits on get_object_at.
2535 self._offset_cache_lock.acquire()
2536 try:
2537 return self._offset_cache[offset]
2538 finally:
2539 self._offset_cache_lock.release()
2541 def _cache_object_at(
2542 self, offset: int, type_num: int, chunks: OldUnpackedObject
2543 ) -> None:
2544 """Cache a resolved object at offset."""
2545 with self._offset_cache_lock:
2546 self._offset_cache[offset] = (type_num, chunks)
2548 def get_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2549 """Given an offset in to the packfile return the object that is there.
2551 Using the associated index the location of an object can be looked up,
2552 and then the packfile can be asked directly for that object using this
2553 function.
2554 """
2555 try:
2556 return self._get_cached_object_at(offset)
2557 except KeyError:
2558 pass
2559 unpacked = self.get_unpacked_object_at(offset, include_comp=False)
2560 return (unpacked.pack_type_num, unpacked._obj())
2563T = TypeVar("T")
2566class DeltaChainIterator(Generic[T]):
2567 """Abstract iterator over pack data based on delta chains.
2569 Each object in the pack is guaranteed to be inflated exactly once,
2570 regardless of how many objects reference it as a delta base. As a result,
2571 memory usage is proportional to the length of the longest delta chain.
2573 Subclasses can override _result to define the result type of the iterator.
2574 By default, results are UnpackedObjects with the following members set:
2576 * offset
2577 * obj_type_num
2578 * obj_chunks
2579 * pack_type_num
2580 * delta_base (for delta types)
2581 * comp_chunks (if _include_comp is True)
2582 * decomp_chunks
2583 * decomp_len
2584 * crc32 (if _compute_crc32 is True)
2585 """
2587 _compute_crc32 = False
2588 _include_comp = False
2590 def __init__(
2591 self,
2592 file_obj: IO[bytes] | None,
2593 hash_func: Callable[[], "HashObject"],
2594 *,
2595 resolve_ext_ref: ResolveExtRefFn | None = None,
2596 object_format: "ObjectFormat | None" = None,
2597 ) -> None:
2598 """Initialize DeltaChainIterator.
2600 Args:
2601 file_obj: File object to read pack data from
2602 hash_func: Hash function to use for computing object IDs
2603 resolve_ext_ref: Optional function to resolve external references
2604 object_format: Optional object format. Required by subclasses
2605 that materialise objects (e.g. PackInflater) when iterating
2606 packs in a non-default hash algorithm such as SHA-256.
2607 """
2608 self._file = file_obj
2609 self._contents: bytes | mmap.mmap | None = None
2610 self.hash_func = hash_func
2611 self._object_format = object_format
2612 self._resolve_ext_ref = resolve_ext_ref
2613 self._pending_ofs: dict[int, list[int]] = defaultdict(list)
2614 self._pending_ref: dict[bytes, list[int]] = defaultdict(list)
2615 self._full_ofs: list[tuple[int, int]] = []
2616 self._ext_refs: list[RawObjectID] = []
2618 @classmethod
2619 def for_pack_data(
2620 cls, pack_data: PackData, resolve_ext_ref: ResolveExtRefFn | None = None
2621 ) -> "DeltaChainIterator[T]":
2622 """Create a DeltaChainIterator from pack data.
2624 Args:
2625 pack_data: PackData object to iterate
2626 resolve_ext_ref: Optional function to resolve external refs
2628 Returns:
2629 DeltaChainIterator instance
2630 """
2631 walker = cls(
2632 None,
2633 pack_data.object_format.hash_func,
2634 resolve_ext_ref=resolve_ext_ref,
2635 object_format=pack_data.object_format,
2636 )
2637 walker.set_pack_data(pack_data)
2638 for unpacked in pack_data.iter_unpacked(include_comp=False):
2639 walker.record(unpacked)
2640 return walker
2642 @classmethod
2643 def for_pack_subset(
2644 cls,
2645 pack: "Pack",
2646 shas: Iterable[ObjectID | RawObjectID],
2647 *,
2648 allow_missing: bool = False,
2649 resolve_ext_ref: ResolveExtRefFn | None = None,
2650 ) -> "DeltaChainIterator[T]":
2651 """Create a DeltaChainIterator for a subset of objects.
2653 Args:
2654 pack: Pack object containing the data
2655 shas: Iterable of object SHAs to include
2656 allow_missing: If True, skip missing objects
2657 resolve_ext_ref: Optional function to resolve external refs
2659 Returns:
2660 DeltaChainIterator instance
2661 """
2662 walker = cls(
2663 None,
2664 pack.object_format.hash_func,
2665 resolve_ext_ref=resolve_ext_ref,
2666 object_format=pack.object_format,
2667 )
2668 walker.set_pack_data(pack.data)
2669 todo = set()
2670 for sha in shas:
2671 try:
2672 off = pack.index.object_offset(sha)
2673 except KeyError:
2674 if not allow_missing:
2675 raise
2676 else:
2677 todo.add(off)
2678 done = set()
2679 while todo:
2680 off = todo.pop()
2681 unpacked = pack.data.get_unpacked_object_at(off)
2682 walker.record(unpacked)
2683 done.add(off)
2684 base_ofs = None
2685 if unpacked.pack_type_num == OFS_DELTA:
2686 assert unpacked.offset is not None
2687 assert unpacked.delta_base is not None
2688 assert isinstance(unpacked.delta_base, int)
2689 base_ofs = unpacked.offset - unpacked.delta_base
2690 elif unpacked.pack_type_num == REF_DELTA:
2691 with suppress(KeyError):
2692 assert isinstance(unpacked.delta_base, bytes)
2693 base_ofs = pack.index.object_offset(
2694 RawObjectID(unpacked.delta_base)
2695 )
2696 if base_ofs is not None and base_ofs not in done:
2697 todo.add(base_ofs)
2698 return walker
2700 def record(self, unpacked: UnpackedObject) -> None:
2701 """Record an unpacked object for later processing.
2703 Args:
2704 unpacked: UnpackedObject to record
2705 """
2706 type_num = unpacked.pack_type_num
2707 offset = unpacked.offset
2708 assert offset is not None
2709 if type_num == OFS_DELTA:
2710 assert unpacked.delta_base is not None
2711 assert isinstance(unpacked.delta_base, int)
2712 base_offset = offset - unpacked.delta_base
2713 self._pending_ofs[base_offset].append(offset)
2714 elif type_num == REF_DELTA:
2715 assert isinstance(unpacked.delta_base, bytes)
2716 self._pending_ref[unpacked.delta_base].append(offset)
2717 else:
2718 self._full_ofs.append((offset, type_num))
2720 def set_pack_data(self, pack_data: PackData) -> None:
2721 """Set the pack data for iteration.
2723 Args:
2724 pack_data: PackData object to use
2725 """
2726 self._file = None
2727 self._contents = pack_data._buffer()
2729 def _walk_all_chains(self) -> Iterator[T]:
2730 for offset, type_num in self._full_ofs:
2731 yield from self._follow_chain(offset, type_num, None)
2732 yield from self._walk_ref_chains()
2733 assert not self._pending_ofs, repr(self._pending_ofs)
2735 def _ensure_no_pending(self) -> None:
2736 if self._pending_ref:
2737 raise UnresolvedDeltas(
2738 [sha_to_hex(RawObjectID(s)) for s in self._pending_ref]
2739 )
2741 def _walk_ref_chains(self) -> Iterator[T]:
2742 if not self._resolve_ext_ref:
2743 self._ensure_no_pending()
2744 return
2746 for base_sha, pending in sorted(self._pending_ref.items()):
2747 if base_sha not in self._pending_ref:
2748 continue
2749 try:
2750 type_num, chunks = self._resolve_ext_ref(RawObjectID(base_sha))
2751 except KeyError:
2752 # Not an external ref, but may depend on one. Either it will
2753 # get popped via a _follow_chain call, or we will raise an
2754 # error below.
2755 continue
2756 self._ext_refs.append(RawObjectID(base_sha))
2757 self._pending_ref.pop(base_sha)
2758 for new_offset in pending:
2759 yield from self._follow_chain(new_offset, type_num, chunks)
2761 self._ensure_no_pending()
2763 def _result(self, unpacked: UnpackedObject) -> T:
2764 raise NotImplementedError
2766 def _resolve_object(
2767 self,
2768 offset: int,
2769 obj_type_num: int,
2770 base_chunks: bytes | list[bytes] | None,
2771 ) -> UnpackedObject:
2772 if self._contents is not None:
2773 unpacked, _ = unpack_object_at(
2774 self._contents,
2775 offset,
2776 self.hash_func,
2777 compute_crc32=self._compute_crc32,
2778 include_comp=self._include_comp,
2779 )
2780 else:
2781 # No mapping was set, see set_pack_data. Read through the file position.
2782 assert self._file is not None
2783 self._file.seek(offset)
2784 unpacked, _ = unpack_object(
2785 self._file.read,
2786 self.hash_func,
2787 read_some=None,
2788 compute_crc32=self._compute_crc32,
2789 include_comp=self._include_comp,
2790 )
2791 unpacked.offset = offset
2792 if base_chunks is None:
2793 assert unpacked.pack_type_num == obj_type_num
2794 else:
2795 assert unpacked.pack_type_num in DELTA_TYPES
2796 unpacked.obj_type_num = obj_type_num
2797 unpacked.obj_chunks = apply_delta(base_chunks, unpacked.decomp_chunks)
2798 # A delta that resolves to a zero-byte payload for a
2799 # commit/tree/tag is malformed: ``_parse_message`` /
2800 # ``parse_tree`` accept the empty input silently, so without
2801 # this guard a too-short delta could materialise an
2802 # otherwise-valid SHA pointing at an empty commit object
2803 # (which ``git fsck`` rejects). Only blobs may legitimately
2804 # be empty, and an empty blob would never be stored as a
2805 # delta in practice.
2806 # Blob.type_num == 3 (avoid the import cycle).
2807 if obj_type_num != 3 and chunks_length(unpacked.obj_chunks) == 0:
2808 raise ApplyDeltaError(
2809 f"delta resolved to empty payload for type {obj_type_num}"
2810 )
2811 return unpacked
2813 def _follow_chain(
2814 self,
2815 offset: int,
2816 obj_type_num: int,
2817 base_chunks: bytes | list[bytes] | None,
2818 ) -> Iterator[T]:
2819 # Unlike PackData.get_object_at, there is no need to cache offsets as
2820 # this approach by design inflates each object exactly once.
2821 todo = [(offset, obj_type_num, base_chunks)]
2822 while todo:
2823 (offset, obj_type_num, base_chunks) = todo.pop()
2824 unpacked = self._resolve_object(offset, obj_type_num, base_chunks)
2825 yield self._result(unpacked)
2827 assert unpacked.offset is not None
2828 unblocked = chain(
2829 self._pending_ofs.pop(unpacked.offset, []),
2830 self._pending_ref.pop(unpacked.sha(), []),
2831 )
2832 todo.extend(
2833 (new_offset, unpacked.obj_type_num, unpacked.obj_chunks) # type: ignore
2834 for new_offset in unblocked
2835 )
2837 def __iter__(self) -> Iterator[T]:
2838 """Iterate over objects in the pack."""
2839 return self._walk_all_chains()
2841 def ext_refs(self) -> list[RawObjectID]:
2842 """Return external references."""
2843 return self._ext_refs
2846class UnpackedObjectIterator(DeltaChainIterator[UnpackedObject]):
2847 """Delta chain iterator that yield unpacked objects."""
2849 def _result(self, unpacked: UnpackedObject) -> UnpackedObject:
2850 """Return the unpacked object.
2852 Args:
2853 unpacked: The unpacked object
2855 Returns:
2856 The unpacked object unchanged
2857 """
2858 return unpacked
2861class PackIndexer(DeltaChainIterator[PackIndexEntry]):
2862 """Delta chain iterator that yields index entries."""
2864 _compute_crc32 = True
2866 def _result(self, unpacked: UnpackedObject) -> PackIndexEntry:
2867 """Convert unpacked object to pack index entry.
2869 Args:
2870 unpacked: The unpacked object
2872 Returns:
2873 Tuple of (sha, offset, crc32) for index entry
2874 """
2875 assert unpacked.offset is not None
2876 return unpacked.sha(), unpacked.offset, unpacked.crc32
2879class PackInflater(DeltaChainIterator[ShaFile]):
2880 """Delta chain iterator that yields ShaFile objects."""
2882 def _result(self, unpacked: UnpackedObject) -> ShaFile:
2883 """Convert unpacked object to ShaFile.
2885 Args:
2886 unpacked: The unpacked object
2888 Returns:
2889 ShaFile object from the unpacked data
2890 """
2891 assert unpacked.obj_type_num is not None and unpacked.obj_chunks is not None
2892 return ShaFile.from_raw_chunks(
2893 unpacked.obj_type_num,
2894 unpacked.obj_chunks,
2895 object_format=self._object_format,
2896 )
2899class SHA1Reader(BinaryIO):
2900 """Wrapper for file-like object that remembers the SHA1 of its data."""
2902 def __init__(self, f: IO[bytes]) -> None:
2903 """Initialize SHA1Reader.
2905 Args:
2906 f: File-like object to wrap
2907 """
2908 self.f = f
2909 self.sha1 = sha1(b"")
2911 def read(self, size: int = -1) -> bytes:
2912 """Read bytes and update SHA1.
2914 Args:
2915 size: Number of bytes to read, -1 for all
2917 Returns:
2918 Bytes read from file
2919 """
2920 data = self.f.read(size)
2921 self.sha1.update(data)
2922 return data
2924 def check_sha(self, allow_empty: bool = False) -> None:
2925 """Check if the SHA1 matches the expected value.
2927 Args:
2928 allow_empty: Allow empty SHA1 hash
2930 Raises:
2931 ChecksumMismatch: If SHA1 doesn't match
2932 """
2933 stored = self.f.read(20)
2934 # If git option index.skipHash is set the index will be empty
2935 if stored != self.sha1.digest() and (
2936 not allow_empty
2937 or (
2938 len(stored) == 20
2939 and sha_to_hex(RawObjectID(stored))
2940 != b"0000000000000000000000000000000000000000"
2941 )
2942 ):
2943 raise ChecksumMismatch(
2944 self.sha1.hexdigest(),
2945 sha_to_hex(RawObjectID(stored)) if stored else b"",
2946 )
2948 def close(self) -> None:
2949 """Close the underlying file."""
2950 return self.f.close()
2952 def tell(self) -> int:
2953 """Return current file position."""
2954 return self.f.tell()
2956 # BinaryIO abstract methods
2957 def readable(self) -> bool:
2958 """Check if file is readable."""
2959 return True
2961 def writable(self) -> bool:
2962 """Check if file is writable."""
2963 return False
2965 def seekable(self) -> bool:
2966 """Check if file is seekable."""
2967 return getattr(self.f, "seekable", lambda: False)()
2969 def seek(self, offset: int, whence: int = 0) -> int:
2970 """Seek to position in file.
2972 Args:
2973 offset: Position offset
2974 whence: Reference point (0=start, 1=current, 2=end)
2976 Returns:
2977 New file position
2978 """
2979 return self.f.seek(offset, whence)
2981 def flush(self) -> None:
2982 """Flush the file buffer."""
2983 if hasattr(self.f, "flush"):
2984 self.f.flush()
2986 def readline(self, size: int = -1) -> bytes:
2987 """Read a line from the file.
2989 Args:
2990 size: Maximum bytes to read
2992 Returns:
2993 Line read from file
2994 """
2995 return self.f.readline(size)
2997 def readlines(self, hint: int = -1) -> list[bytes]:
2998 """Read all lines from the file.
3000 Args:
3001 hint: Approximate number of bytes to read
3003 Returns:
3004 List of lines
3005 """
3006 return self.f.readlines(hint)
3008 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3009 """Write multiple lines to the file (not supported)."""
3010 raise UnsupportedOperation("writelines")
3012 def write(self, data: bytes, /) -> int: # type: ignore[override]
3013 """Write data to the file (not supported)."""
3014 raise UnsupportedOperation("write")
3016 def __enter__(self) -> Self:
3017 """Enter context manager."""
3018 return self
3020 def __exit__(
3021 self,
3022 type: type | None,
3023 value: BaseException | None,
3024 traceback: TracebackType | None,
3025 ) -> None:
3026 """Exit context manager and close file."""
3027 self.close()
3029 def __iter__(self) -> "SHA1Reader":
3030 """Return iterator for reading file lines."""
3031 return self
3033 def __next__(self) -> bytes:
3034 """Get next line from file.
3036 Returns:
3037 Next line
3039 Raises:
3040 StopIteration: When no more lines
3041 """
3042 line = self.readline()
3043 if not line:
3044 raise StopIteration
3045 return line
3047 def fileno(self) -> int:
3048 """Return file descriptor number."""
3049 return self.f.fileno()
3051 def isatty(self) -> bool:
3052 """Check if file is a terminal."""
3053 return getattr(self.f, "isatty", lambda: False)()
3055 def truncate(self, size: int | None = None) -> int:
3056 """Not supported for read-only file.
3058 Raises:
3059 UnsupportedOperation: Always raised
3060 """
3061 raise UnsupportedOperation("truncate")
3064class SHA1Writer(BinaryIO):
3065 """Wrapper for file-like object that remembers the SHA1 of its data."""
3067 def __init__(self, f: BinaryIO | IO[bytes]) -> None:
3068 """Initialize SHA1Writer.
3070 Args:
3071 f: File-like object to wrap
3072 """
3073 self.f = f
3074 self.length = 0
3075 self.sha1 = sha1(b"")
3076 self.digest: bytes | None = None
3078 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
3079 """Write data and update SHA1.
3081 Args:
3082 data: Data to write
3084 Returns:
3085 Number of bytes written
3086 """
3087 self.sha1.update(data)
3088 written = self.f.write(data)
3089 self.length += written
3090 return written
3092 def write_sha(self) -> bytes:
3093 """Write the SHA1 digest to the file.
3095 Returns:
3096 The SHA1 digest bytes
3097 """
3098 sha = self.sha1.digest()
3099 assert len(sha) == 20
3100 self.f.write(sha)
3101 self.length += len(sha)
3102 return sha
3104 def close(self) -> None:
3105 """Close the pack file and finalize the SHA."""
3106 self.digest = self.write_sha()
3107 self.f.close()
3109 def offset(self) -> int:
3110 """Get the total number of bytes written.
3112 Returns:
3113 Total bytes written
3114 """
3115 return self.length
3117 def tell(self) -> int:
3118 """Return current file position."""
3119 return self.f.tell()
3121 # BinaryIO abstract methods
3122 def readable(self) -> bool:
3123 """Check if file is readable."""
3124 return False
3126 def writable(self) -> bool:
3127 """Check if file is writable."""
3128 return True
3130 def seekable(self) -> bool:
3131 """Check if file is seekable."""
3132 return getattr(self.f, "seekable", lambda: False)()
3134 def seek(self, offset: int, whence: int = 0) -> int:
3135 """Seek to position in file.
3137 Args:
3138 offset: Position offset
3139 whence: Reference point (0=start, 1=current, 2=end)
3141 Returns:
3142 New file position
3143 """
3144 return self.f.seek(offset, whence)
3146 def flush(self) -> None:
3147 """Flush the file buffer."""
3148 if hasattr(self.f, "flush"):
3149 self.f.flush()
3151 def readline(self, size: int = -1) -> bytes:
3152 """Not supported for write-only file.
3154 Raises:
3155 UnsupportedOperation: Always raised
3156 """
3157 raise UnsupportedOperation("readline")
3159 def readlines(self, hint: int = -1) -> list[bytes]:
3160 """Not supported for write-only file.
3162 Raises:
3163 UnsupportedOperation: Always raised
3164 """
3165 raise UnsupportedOperation("readlines")
3167 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3168 """Write multiple lines to the file.
3170 Args:
3171 lines: Iterable of lines to write
3172 """
3173 for line in lines:
3174 self.write(line)
3176 def read(self, size: int = -1) -> bytes:
3177 """Not supported for write-only file.
3179 Raises:
3180 UnsupportedOperation: Always raised
3181 """
3182 raise UnsupportedOperation("read")
3184 def __enter__(self) -> Self:
3185 """Enter context manager."""
3186 return self
3188 def __exit__(
3189 self,
3190 type: type | None,
3191 value: BaseException | None,
3192 traceback: TracebackType | None,
3193 ) -> None:
3194 """Exit context manager and close file."""
3195 self.f.close()
3197 def __iter__(self) -> "SHA1Writer":
3198 """Return iterator."""
3199 return self
3201 def __next__(self) -> bytes:
3202 """Not supported for write-only file.
3204 Raises:
3205 UnsupportedOperation: Always raised
3206 """
3207 raise UnsupportedOperation("__next__")
3209 def fileno(self) -> int:
3210 """Return file descriptor number."""
3211 return self.f.fileno()
3213 def isatty(self) -> bool:
3214 """Check if file is a terminal."""
3215 return getattr(self.f, "isatty", lambda: False)()
3217 def truncate(self, size: int | None = None) -> int:
3218 """Not supported for write-only file.
3220 Raises:
3221 UnsupportedOperation: Always raised
3222 """
3223 raise UnsupportedOperation("truncate")
3226class HashWriter(BinaryIO):
3227 """Wrapper for file-like object that computes hash of its data.
3229 This is a generic version that works with any hash algorithm.
3230 """
3232 def __init__(
3233 self, f: BinaryIO | IO[bytes], hash_func: Callable[[], "HashObject"]
3234 ) -> None:
3235 """Initialize HashWriter.
3237 Args:
3238 f: File-like object to wrap
3239 hash_func: Hash function (e.g., sha1, sha256)
3240 """
3241 self.f = f
3242 self.length = 0
3243 self.hash_obj = hash_func()
3244 self.digest: bytes | None = None
3246 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
3247 """Write data and update hash.
3249 Args:
3250 data: Data to write
3252 Returns:
3253 Number of bytes written
3254 """
3255 self.hash_obj.update(data)
3256 written = self.f.write(data)
3257 self.length += written
3258 return written
3260 def write_hash(self) -> bytes:
3261 """Write the hash digest to the file.
3263 Returns:
3264 The hash digest bytes
3265 """
3266 digest = self.hash_obj.digest()
3267 self.f.write(digest)
3268 self.length += len(digest)
3269 return digest
3271 def close(self) -> None:
3272 """Close the pack file and finalize the hash."""
3273 self.digest = self.write_hash()
3274 self.f.close()
3276 def offset(self) -> int:
3277 """Get the total number of bytes written.
3279 Returns:
3280 Total bytes written
3281 """
3282 return self.length
3284 def tell(self) -> int:
3285 """Return current file position."""
3286 return self.f.tell()
3288 # BinaryIO abstract methods
3289 def readable(self) -> bool:
3290 """Check if file is readable."""
3291 return False
3293 def writable(self) -> bool:
3294 """Check if file is writable."""
3295 return True
3297 def seekable(self) -> bool:
3298 """Check if file is seekable."""
3299 return getattr(self.f, "seekable", lambda: False)()
3301 def seek(self, offset: int, whence: int = 0) -> int:
3302 """Seek to position in file.
3304 Args:
3305 offset: Position offset
3306 whence: Reference point (0=start, 1=current, 2=end)
3308 Returns:
3309 New file position
3310 """
3311 return self.f.seek(offset, whence)
3313 def flush(self) -> None:
3314 """Flush the file buffer."""
3315 if hasattr(self.f, "flush"):
3316 self.f.flush()
3318 def readline(self, size: int = -1) -> bytes:
3319 """Not supported for write-only file.
3321 Raises:
3322 UnsupportedOperation: Always raised
3323 """
3324 raise UnsupportedOperation("readline")
3326 def readlines(self, hint: int = -1) -> list[bytes]:
3327 """Not supported for write-only file.
3329 Raises:
3330 UnsupportedOperation: Always raised
3331 """
3332 raise UnsupportedOperation("readlines")
3334 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3335 """Write multiple lines to the file.
3337 Args:
3338 lines: Iterable of lines to write
3339 """
3340 for line in lines:
3341 self.write(line)
3343 def read(self, size: int = -1) -> bytes:
3344 """Not supported for write-only file.
3346 Raises:
3347 UnsupportedOperation: Always raised
3348 """
3349 raise UnsupportedOperation("read")
3351 def __enter__(self) -> Self:
3352 """Enter context manager."""
3353 return self
3355 def __exit__(
3356 self,
3357 type: type | None,
3358 value: BaseException | None,
3359 traceback: TracebackType | None,
3360 ) -> None:
3361 """Exit context manager and close file."""
3362 self.close()
3364 def __iter__(self) -> "HashWriter":
3365 """Return iterator."""
3366 return self
3368 def __next__(self) -> bytes:
3369 """Not supported for write-only file.
3371 Raises:
3372 UnsupportedOperation: Always raised
3373 """
3374 raise UnsupportedOperation("__next__")
3376 def fileno(self) -> int:
3377 """Return file descriptor number."""
3378 return self.f.fileno()
3380 def isatty(self) -> bool:
3381 """Check if file is a terminal."""
3382 return getattr(self.f, "isatty", lambda: False)()
3384 def truncate(self, size: int | None = None) -> int:
3385 """Not supported for write-only file.
3387 Raises:
3388 UnsupportedOperation: Always raised
3389 """
3390 raise UnsupportedOperation("truncate")
3393def pack_object_header(
3394 type_num: int,
3395 delta_base: bytes | int | None,
3396 size: int,
3397 object_format: "ObjectFormat",
3398) -> bytearray:
3399 """Create a pack object header for the given object info.
3401 Args:
3402 type_num: Numeric type of the object.
3403 delta_base: Delta base offset or ref, or None for whole objects.
3404 size: Uncompressed object size.
3405 object_format: Object format (hash algorithm) to use.
3406 Returns: A header for a packed object.
3407 """
3408 header = []
3409 c = (type_num << 4) | (size & 15)
3410 size >>= 4
3411 while size:
3412 header.append(c | 0x80)
3413 c = size & 0x7F
3414 size >>= 7
3415 header.append(c)
3416 if type_num == OFS_DELTA:
3417 assert isinstance(delta_base, int)
3418 ret = [delta_base & 0x7F]
3419 delta_base >>= 7
3420 while delta_base:
3421 delta_base -= 1
3422 ret.insert(0, 0x80 | (delta_base & 0x7F))
3423 delta_base >>= 7
3424 header.extend(ret)
3425 elif type_num == REF_DELTA:
3426 assert isinstance(delta_base, bytes)
3427 assert len(delta_base) == object_format.oid_length
3428 header += delta_base
3429 return bytearray(header)
3432def pack_object_chunks(
3433 type: int,
3434 object: list[bytes] | tuple[bytes | int, list[bytes]],
3435 object_format: "ObjectFormat",
3436 *,
3437 compression_level: int = -1,
3438) -> Iterator[bytes]:
3439 """Generate chunks for a pack object.
3441 Args:
3442 type: Numeric type of the object
3443 object: Object to write
3444 object_format: Object format (hash algorithm) to use
3445 compression_level: the zlib compression level
3446 Returns: Chunks
3447 """
3448 if type in DELTA_TYPES:
3449 if isinstance(object, tuple):
3450 delta_base, object = object
3451 else:
3452 raise TypeError("Delta types require a tuple of (delta_base, object)")
3453 else:
3454 delta_base = None
3456 # Convert object to list of bytes chunks
3457 match object:
3458 case bytes():
3459 chunks = [object]
3460 case list():
3461 chunks = object
3462 case ShaFile():
3463 chunks = object.as_raw_chunks()
3464 case _:
3465 # Shouldn't reach here with proper typing
3466 raise TypeError(f"Unexpected object type: {object.__class__.__name__}")
3468 yield bytes(
3469 pack_object_header(
3470 type, delta_base, sum(map(len, chunks)), object_format=object_format
3471 )
3472 )
3473 compressor = zlib.compressobj(level=compression_level)
3474 for data in chunks:
3475 yield compressor.compress(data)
3476 yield compressor.flush()
3479def write_pack_object(
3480 write: Callable[[bytes], int],
3481 type: int,
3482 object: list[bytes] | tuple[bytes | int, list[bytes]],
3483 object_format: "ObjectFormat",
3484 *,
3485 sha: "HashObject | None" = None,
3486 compression_level: int = -1,
3487) -> int:
3488 """Write pack object to a file.
3490 Args:
3491 write: Write function to use
3492 type: Numeric type of the object
3493 object: Object to write
3494 object_format: Object format (hash algorithm) to use
3495 sha: Optional SHA-1 hasher to update
3496 compression_level: the zlib compression level
3497 Returns: CRC32 checksum of the written object
3498 """
3499 crc32 = 0
3500 for chunk in pack_object_chunks(
3501 type, object, compression_level=compression_level, object_format=object_format
3502 ):
3503 write(chunk)
3504 if sha is not None:
3505 sha.update(chunk)
3506 crc32 = binascii.crc32(chunk, crc32)
3507 return crc32 & 0xFFFFFFFF
3510def write_pack(
3511 filename: str,
3512 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3513 object_format: "ObjectFormat",
3514 *,
3515 deltify: bool | None = None,
3516 delta_window_size: int | None = None,
3517 compression_level: int = -1,
3518) -> tuple[bytes, bytes]:
3519 """Write a new pack data file.
3521 Args:
3522 filename: Path to the new pack file (without .pack extension)
3523 objects: Objects to write to the pack
3524 object_format: Object format
3525 delta_window_size: Delta window size
3526 deltify: Whether to deltify pack objects
3527 compression_level: the zlib compression level
3528 Returns: Tuple with checksum of pack file and index file
3529 """
3530 with GitFile(filename + ".pack", "wb") as f:
3531 entries, data_sum = write_pack_objects(
3532 f,
3533 objects,
3534 delta_window_size=delta_window_size,
3535 deltify=deltify,
3536 compression_level=compression_level,
3537 object_format=object_format,
3538 )
3539 entries_list = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
3540 with GitFile(filename + ".idx", "wb") as f:
3541 idx_sha = write_pack_index(f, entries_list, data_sum)
3542 return data_sum, idx_sha
3545def pack_header_chunks(num_objects: int) -> Iterator[bytes]:
3546 """Yield chunks for a pack header."""
3547 yield b"PACK" # Pack header
3548 yield struct.pack(b">L", 2) # Pack version
3549 yield struct.pack(b">L", num_objects) # Number of objects in pack
3552def write_pack_header(
3553 write: Callable[[bytes], int] | IO[bytes], num_objects: int
3554) -> None:
3555 """Write a pack header for the given number of objects."""
3556 if not callable(write):
3557 write_fn: Callable[[bytes], int] = write.write
3558 warnings.warn(
3559 "write_pack_header() now takes a write rather than file argument",
3560 DeprecationWarning,
3561 stacklevel=2,
3562 )
3563 else:
3564 write_fn = write
3565 for chunk in pack_header_chunks(num_objects):
3566 write_fn(chunk)
3569def find_reusable_deltas(
3570 container: PackedObjectContainer,
3571 object_ids: Set[ObjectID],
3572 *,
3573 other_haves: Set[ObjectID] | None = None,
3574 progress: Callable[..., None] | None = None,
3575) -> Iterator[UnpackedObject]:
3576 """Find deltas in a pack that can be reused.
3578 Args:
3579 container: Pack container to search for deltas
3580 object_ids: Set of object IDs to find deltas for
3581 other_haves: Set of other object IDs we have
3582 progress: Optional progress reporting callback
3584 Returns:
3585 Iterator of UnpackedObject entries that can be reused
3586 """
3587 if other_haves is None:
3588 other_haves = set()
3589 reused = 0
3590 for i, unpacked in enumerate(
3591 container.iter_unpacked_subset(
3592 object_ids, allow_missing=True, convert_ofs_delta=True
3593 )
3594 ):
3595 if progress is not None and i % 1000 == 0:
3596 progress(f"checking for reusable deltas: {i}/{len(object_ids)}\r".encode())
3597 if unpacked.pack_type_num == REF_DELTA:
3598 hexsha = sha_to_hex(unpacked.delta_base) # type: ignore
3599 if hexsha in object_ids or hexsha in other_haves:
3600 yield unpacked
3601 reused += 1
3602 if progress is not None:
3603 progress((f"found {reused} deltas to reuse\n").encode())
3606def deltify_pack_objects(
3607 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, bytes | None]],
3608 *,
3609 window_size: int | None = None,
3610 progress: Callable[..., None] | None = None,
3611) -> Iterator[UnpackedObject]:
3612 """Generate deltas for pack objects.
3614 Args:
3615 objects: An iterable of (object, path) tuples to deltify.
3616 window_size: Window size; None for default
3617 progress: Optional progress reporting callback
3618 Returns: Iterator over type_num, object id, delta_base, content
3619 delta_base is None for full text entries
3620 """
3622 def objects_with_hints() -> Iterator[tuple[ShaFile, tuple[int, bytes | None]]]:
3623 for e in objects:
3624 if isinstance(e, ShaFile):
3625 yield (e, (e.type_num, None))
3626 else:
3627 yield (e[0], (e[0].type_num, e[1]))
3629 sorted_objs = sort_objects_for_delta(objects_with_hints())
3630 yield from deltas_from_sorted_objects(
3631 sorted_objs,
3632 window_size=window_size,
3633 progress=progress,
3634 )
3637def sort_objects_for_delta(
3638 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, PackHint | None]],
3639) -> Iterator[tuple[ShaFile, bytes | None]]:
3640 """Sort objects for optimal delta compression.
3642 Args:
3643 objects: Iterator of objects or (object, hint) tuples
3645 Returns:
3646 Iterator of sorted (ShaFile, path) tuples
3647 """
3648 magic = []
3649 for entry in objects:
3650 if isinstance(entry, tuple):
3651 obj, hint = entry
3652 if hint is None:
3653 type_num = None
3654 path = None
3655 else:
3656 (type_num, path) = hint
3657 else:
3658 obj = entry
3659 type_num = None
3660 path = None
3661 magic.append((type_num, path, -obj.raw_length(), obj))
3662 # Build a list of objects ordered by the magic Linus heuristic
3663 # This helps us find good objects to diff against us
3664 magic.sort()
3665 return ((x[3], x[1]) for x in magic)
3668def deltas_from_sorted_objects(
3669 objects: Iterator[tuple[ShaFile, bytes | None]],
3670 window_size: int | None = None,
3671 progress: Callable[..., None] | None = None,
3672) -> Iterator[UnpackedObject]:
3673 """Create deltas from sorted objects.
3675 Args:
3676 objects: Iterator of sorted objects to deltify
3677 window_size: Delta window size; None for default
3678 progress: Optional progress reporting callback
3680 Returns:
3681 Iterator of UnpackedObject entries
3682 """
3683 # TODO(jelmer): Use threads
3684 if window_size is None:
3685 window_size = DEFAULT_PACK_DELTA_WINDOW_SIZE
3687 possible_bases: deque[tuple[bytes, int, bytes]] = deque()
3688 for i, (o, path) in enumerate(objects):
3689 if progress is not None and i % 1000 == 0:
3690 progress((f"generating deltas: {i}\r").encode())
3691 raw = o.as_raw_chunks()
3692 raw_bytes = b"".join(raw) # Join once for efficiency
3693 winner = raw
3694 winner_len = sum(map(len, winner))
3695 winner_base = None
3696 for base_id, base_type_num, base_bytes in possible_bases:
3697 if base_type_num != o.type_num:
3698 continue
3699 delta_len = 0
3700 delta = []
3701 for chunk in create_delta(base_bytes, raw_bytes):
3702 delta_len += len(chunk)
3703 if delta_len >= winner_len:
3704 break
3705 delta.append(chunk)
3706 else:
3707 winner_base = base_id
3708 winner = delta
3709 winner_len = sum(map(len, winner))
3710 yield UnpackedObject(
3711 o.type_num,
3712 sha=o.sha().digest(),
3713 delta_base=winner_base,
3714 decomp_len=winner_len,
3715 decomp_chunks=winner,
3716 )
3717 possible_bases.appendleft((o.sha().digest(), o.type_num, raw_bytes))
3718 while len(possible_bases) > window_size:
3719 possible_bases.pop()
3722def pack_objects_to_data(
3723 objects: Sequence[ShaFile]
3724 | Sequence[tuple[ShaFile, bytes | None]]
3725 | Sequence[tuple[ShaFile, PackHint | None]],
3726 *,
3727 deltify: bool | None = None,
3728 delta_window_size: int | None = None,
3729 ofs_delta: bool = True,
3730 progress: Callable[..., None] | None = None,
3731) -> tuple[int, Iterator[UnpackedObject]]:
3732 """Create pack data from objects.
3734 Args:
3735 objects: Pack objects
3736 deltify: Whether to deltify pack objects
3737 delta_window_size: Delta window size
3738 ofs_delta: Whether to use offset deltas
3739 progress: Optional progress reporting callback
3740 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3741 """
3742 count = len(objects)
3743 if deltify is None:
3744 # PERFORMANCE/TODO(jelmer): This should be enabled but the python
3745 # implementation is *much* too slow at the moment.
3746 # Maybe consider enabling it just if the rust extension is available?
3747 deltify = False
3748 if deltify:
3749 return (
3750 count,
3751 deltify_pack_objects(
3752 iter(objects), # type: ignore
3753 window_size=delta_window_size,
3754 progress=progress,
3755 ),
3756 )
3757 else:
3759 def iter_without_path() -> Iterator[UnpackedObject]:
3760 for o in objects:
3761 if isinstance(o, tuple):
3762 yield full_unpacked_object(o[0])
3763 else:
3764 yield full_unpacked_object(o)
3766 return (count, iter_without_path())
3769def generate_unpacked_objects(
3770 container: PackedObjectContainer,
3771 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3772 delta_window_size: int | None = None,
3773 deltify: bool | None = None,
3774 reuse_deltas: bool = True,
3775 ofs_delta: bool = True,
3776 other_haves: set[ObjectID] | None = None,
3777 progress: Callable[..., None] | None = None,
3778) -> Iterator[UnpackedObject]:
3779 """Create pack data from objects.
3781 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3782 """
3783 todo = dict(object_ids)
3784 if reuse_deltas:
3785 for unpack in find_reusable_deltas(
3786 container, set(todo), other_haves=other_haves, progress=progress
3787 ):
3788 del todo[sha_to_hex(RawObjectID(unpack.sha()))]
3789 yield unpack
3790 if deltify is None:
3791 # PERFORMANCE/TODO(jelmer): This should be enabled but is *much* too
3792 # slow at the moment.
3793 deltify = False
3794 if deltify:
3795 objects_to_delta = container.iterobjects_subset(
3796 todo.keys(), allow_missing=False
3797 )
3798 sorted_objs = sort_objects_for_delta((o, todo[o.id]) for o in objects_to_delta)
3799 yield from deltas_from_sorted_objects(
3800 sorted_objs,
3801 window_size=delta_window_size,
3802 progress=progress,
3803 )
3804 else:
3805 for oid in todo:
3806 yield full_unpacked_object(container[oid])
3809def full_unpacked_object(o: ShaFile) -> UnpackedObject:
3810 """Create an UnpackedObject from a ShaFile.
3812 Args:
3813 o: ShaFile object to convert
3815 Returns:
3816 UnpackedObject with full object data
3817 """
3818 return UnpackedObject(
3819 o.type_num,
3820 delta_base=None,
3821 crc32=None,
3822 decomp_chunks=o.as_raw_chunks(),
3823 sha=o.sha().digest(),
3824 )
3827def write_pack_from_container(
3828 write: Callable[[bytes], None]
3829 | Callable[[bytes | bytearray | memoryview], int]
3830 | IO[bytes],
3831 container: PackedObjectContainer,
3832 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3833 object_format: "ObjectFormat",
3834 *,
3835 delta_window_size: int | None = None,
3836 deltify: bool | None = None,
3837 reuse_deltas: bool = True,
3838 compression_level: int = -1,
3839 other_haves: set[ObjectID] | None = None,
3840) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3841 """Write a new pack data file.
3843 Args:
3844 write: write function to use
3845 container: PackedObjectContainer
3846 object_ids: Sequence of (object_id, hint) tuples to write
3847 object_format: Object format (hash algorithm) to use
3848 delta_window_size: Sliding window size for searching for deltas;
3849 Set to None for default window size.
3850 deltify: Whether to deltify objects
3851 reuse_deltas: Whether to reuse existing deltas
3852 compression_level: the zlib compression level to use
3853 other_haves: Set of additional object IDs the receiver has
3854 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3855 """
3856 pack_contents_count = len(object_ids)
3857 pack_contents = generate_unpacked_objects(
3858 container,
3859 object_ids,
3860 delta_window_size=delta_window_size,
3861 deltify=deltify,
3862 reuse_deltas=reuse_deltas,
3863 other_haves=other_haves,
3864 )
3866 return write_pack_data(
3867 write,
3868 pack_contents,
3869 num_records=pack_contents_count,
3870 compression_level=compression_level,
3871 object_format=object_format,
3872 )
3875def write_pack_objects(
3876 write: Callable[[bytes], None] | IO[bytes],
3877 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3878 object_format: "ObjectFormat",
3879 *,
3880 delta_window_size: int | None = None,
3881 deltify: bool | None = None,
3882 compression_level: int = -1,
3883) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3884 """Write a new pack data file.
3886 Args:
3887 write: write function to use
3888 objects: Sequence of (object, path) tuples to write
3889 object_format: Object format (hash algorithm) to use
3890 delta_window_size: Sliding window size for searching for deltas;
3891 Set to None for default window size.
3892 deltify: Whether to deltify objects
3893 compression_level: the zlib compression level to use
3894 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3895 """
3896 pack_contents_count, pack_contents = pack_objects_to_data(objects, deltify=deltify)
3898 return write_pack_data(
3899 write,
3900 pack_contents,
3901 num_records=pack_contents_count,
3902 compression_level=compression_level,
3903 object_format=object_format,
3904 )
3907class PackChunkGenerator:
3908 """Generator for pack data chunks."""
3910 def __init__(
3911 self,
3912 object_format: "ObjectFormat",
3913 num_records: int | None = None,
3914 records: Iterator[UnpackedObject] | None = None,
3915 progress: Callable[..., None] | None = None,
3916 compression_level: int = -1,
3917 reuse_compressed: bool = True,
3918 ) -> None:
3919 """Initialize PackChunkGenerator.
3921 Args:
3922 num_records: Expected number of records
3923 records: Iterator of pack records
3924 progress: Optional progress callback
3925 compression_level: Compression level (-1 for default)
3926 reuse_compressed: Whether to reuse compressed chunks
3927 object_format: Object format (hash algorithm) to use
3928 """
3929 self.object_format = object_format
3930 self.cs = object_format.new_hash()
3931 self.entries: dict[bytes, tuple[int, int]] = {}
3932 if records is None:
3933 records = iter([]) # Empty iterator if None
3934 self._it = self._pack_data_chunks(
3935 records=records,
3936 num_records=num_records,
3937 progress=progress,
3938 compression_level=compression_level,
3939 reuse_compressed=reuse_compressed,
3940 )
3942 def sha1digest(self) -> bytes:
3943 """Return the SHA1 digest of the pack data."""
3944 return self.cs.digest()
3946 def __iter__(self) -> Iterator[bytes]:
3947 """Iterate over pack data chunks."""
3948 return self._it
3950 def _pack_data_chunks(
3951 self,
3952 records: Iterator[UnpackedObject],
3953 *,
3954 num_records: int | None = None,
3955 progress: Callable[..., None] | None = None,
3956 compression_level: int = -1,
3957 reuse_compressed: bool = True,
3958 ) -> Iterator[bytes]:
3959 """Iterate pack data file chunks.
3961 Args:
3962 records: Iterator over UnpackedObject
3963 num_records: Number of records (defaults to len(records) if not specified)
3964 progress: Function to report progress to
3965 compression_level: the zlib compression level
3966 reuse_compressed: Whether to reuse compressed chunks
3967 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3968 """
3969 # Write the pack
3970 if num_records is None:
3971 num_records = len(records) # type: ignore
3972 offset = 0
3973 for chunk in pack_header_chunks(num_records):
3974 yield chunk
3975 self.cs.update(chunk)
3976 offset += len(chunk)
3977 actual_num_records = 0
3978 for i, unpacked in enumerate(records):
3979 type_num = unpacked.pack_type_num
3980 if progress is not None and i % 1000 == 0:
3981 progress((f"writing pack data: {i}/{num_records}\r").encode("ascii"))
3982 raw: list[bytes] | tuple[int, list[bytes]] | tuple[bytes, list[bytes]]
3983 if unpacked.delta_base is not None:
3984 assert isinstance(unpacked.delta_base, bytes), (
3985 f"Expected bytes, got {type(unpacked.delta_base)}"
3986 )
3987 try:
3988 base_offset, _base_crc32 = self.entries[unpacked.delta_base]
3989 except KeyError:
3990 type_num = REF_DELTA
3991 assert isinstance(unpacked.delta_base, bytes)
3992 raw = (unpacked.delta_base, unpacked.decomp_chunks)
3993 else:
3994 type_num = OFS_DELTA
3995 raw = (offset - base_offset, unpacked.decomp_chunks)
3996 else:
3997 raw = unpacked.decomp_chunks
3998 chunks: list[bytes] | Iterator[bytes]
3999 if unpacked.comp_chunks is not None and reuse_compressed:
4000 chunks = unpacked.comp_chunks
4001 else:
4002 chunks = pack_object_chunks(
4003 type_num,
4004 raw,
4005 compression_level=compression_level,
4006 object_format=self.object_format,
4007 )
4008 crc32 = 0
4009 object_size = 0
4010 for chunk in chunks:
4011 yield chunk
4012 crc32 = binascii.crc32(chunk, crc32)
4013 self.cs.update(chunk)
4014 object_size += len(chunk)
4015 actual_num_records += 1
4016 self.entries[unpacked.sha()] = (offset, crc32)
4017 offset += object_size
4018 if actual_num_records != num_records:
4019 raise AssertionError(
4020 f"actual records written differs: {actual_num_records} != {num_records}"
4021 )
4023 yield self.cs.digest()
4026def write_pack_data(
4027 write: Callable[[bytes], None]
4028 | Callable[[bytes | bytearray | memoryview], int]
4029 | IO[bytes],
4030 records: Iterator[UnpackedObject],
4031 object_format: "ObjectFormat",
4032 *,
4033 num_records: int | None = None,
4034 progress: Callable[..., None] | None = None,
4035 compression_level: int = -1,
4036) -> tuple[dict[bytes, tuple[int, int]], bytes]:
4037 """Write a new pack data file.
4039 Args:
4040 write: Write function to use
4041 num_records: Number of records (defaults to len(records) if None)
4042 records: Iterator over type_num, object_id, delta_base, raw
4043 object_format: Object format (hash algorithm) to use
4044 progress: Function to report progress to
4045 compression_level: the zlib compression level
4046 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
4047 """
4048 chunk_generator = PackChunkGenerator(
4049 num_records=num_records,
4050 records=records,
4051 progress=progress,
4052 compression_level=compression_level,
4053 object_format=object_format,
4054 )
4055 for chunk in chunk_generator:
4056 if callable(write):
4057 write(chunk)
4058 else:
4059 write.write(chunk)
4060 return chunk_generator.entries, chunk_generator.sha1digest()
4063def write_pack_index_v1(
4064 f: IO[bytes],
4065 entries: Iterable[tuple[bytes, int, int | None]],
4066 pack_checksum: bytes,
4067) -> bytes:
4068 """Write a new pack index file.
4070 Args:
4071 f: A file-like object to write to
4072 entries: List of tuples with object name (sha), offset_in_pack,
4073 and crc32_checksum.
4074 pack_checksum: Checksum of the pack file.
4075 Returns: The SHA of the written index file
4076 """
4077 f = SHA1Writer(f)
4078 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4079 for name, _offset, _entry_checksum in entries:
4080 fan_out_table[ord(name[:1])] += 1
4081 # Fan-out table
4082 for i in range(0x100):
4083 f.write(struct.pack(">L", fan_out_table[i]))
4084 fan_out_table[i + 1] += fan_out_table[i]
4085 for name, offset, _entry_checksum in entries:
4086 if len(name) != 20:
4087 raise TypeError("pack index v1 only supports SHA-1 names")
4088 if not (offset <= 0xFFFFFFFF):
4089 raise TypeError("pack format 1 only supports offsets < 2Gb")
4090 f.write(struct.pack(">L20s", offset, name))
4091 assert len(pack_checksum) == 20
4092 f.write(pack_checksum)
4093 return f.write_sha()
4096def _delta_encode_size(size: int) -> bytes:
4097 ret = bytearray()
4098 c = size & 0x7F
4099 size >>= 7
4100 while size:
4101 ret.append(c | 0x80)
4102 c = size & 0x7F
4103 size >>= 7
4104 ret.append(c)
4105 return bytes(ret)
4108# The length of delta compression copy operations in version 2 packs is limited
4109# to 64K. To copy more, we use several copy operations. Version 3 packs allow
4110# 24-bit lengths in copy operations, but we always make version 2 packs.
4111_MAX_COPY_LEN = 0xFFFF
4114def _encode_copy_operation(start: int, length: int) -> bytes:
4115 scratch = bytearray([0x80])
4116 for i in range(4):
4117 if start & 0xFF << i * 8:
4118 scratch.append((start >> i * 8) & 0xFF)
4119 scratch[0] |= 1 << i
4120 for i in range(2):
4121 if length & 0xFF << i * 8:
4122 scratch.append((length >> i * 8) & 0xFF)
4123 scratch[0] |= 1 << (4 + i)
4124 return bytes(scratch)
4127def _create_delta_py(
4128 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
4129) -> Iterator[bytes]:
4130 """Use python difflib to work out how to transform base_buf to target_buf.
4132 Args:
4133 base_buf: Base buffer
4134 target_buf: Target buffer
4135 """
4136 if isinstance(base_buf, list):
4137 base_buf = b"".join(base_buf)
4138 if isinstance(target_buf, list):
4139 target_buf = b"".join(target_buf)
4140 # write delta header
4141 yield _delta_encode_size(len(base_buf))
4142 yield _delta_encode_size(len(target_buf))
4143 # write out delta opcodes
4144 seq = SequenceMatcher(isjunk=None, a=base_buf, b=target_buf)
4145 for opcode, i1, i2, j1, j2 in seq.get_opcodes():
4146 # Git patch opcodes don't care about deletes!
4147 # if opcode == 'replace' or opcode == 'delete':
4148 # pass
4149 if opcode == "equal":
4150 # If they are equal, unpacker will use data from base_buf
4151 # Write out an opcode that says what range to use
4152 copy_start = i1
4153 copy_len = i2 - i1
4154 while copy_len > 0:
4155 to_copy = min(copy_len, _MAX_COPY_LEN)
4156 yield _encode_copy_operation(copy_start, to_copy)
4157 copy_start += to_copy
4158 copy_len -= to_copy
4159 if opcode == "replace" or opcode == "insert":
4160 # If we are replacing a range or adding one, then we just
4161 # output it to the stream (prefixed by its size)
4162 s = j2 - j1
4163 o = j1
4164 while s > 127:
4165 yield bytes([127])
4166 yield bytes(memoryview(target_buf)[o : o + 127])
4167 s -= 127
4168 o += 127
4169 yield bytes([s])
4170 yield bytes(memoryview(target_buf)[o : o + s])
4173# Default to pure Python implementation
4174create_delta = _create_delta_py
4177def _apply_delta_py(
4178 src_buf: bytes | list[bytes], delta: bytes | list[bytes]
4179) -> list[bytes]:
4180 """Based on the similar function in git's patch-delta.c.
4182 Args:
4183 src_buf: Source buffer
4184 delta: Delta instructions
4185 """
4186 if not isinstance(src_buf, bytes):
4187 src_buf = b"".join(src_buf)
4188 if not isinstance(delta, bytes):
4189 delta = b"".join(delta)
4190 out = []
4191 index = 0
4192 delta_length = len(delta)
4194 def get_delta_header_size(delta: bytes, index: int) -> tuple[int, int]:
4195 size = 0
4196 i = 0
4197 while True:
4198 # Bound-check explicitly: ``delta[index:index+1]`` silently
4199 # returns b"" past the end, which would crash with TypeError
4200 # in ``ord`` and leave the caller unable to distinguish a
4201 # truncated delta from a programming bug.
4202 if index >= delta_length:
4203 raise ApplyDeltaError("delta truncated in size header")
4204 cmd = ord(delta[index : index + 1])
4205 index += 1
4206 size |= (cmd & ~0x80) << i
4207 i += 7
4208 if not cmd & 0x80:
4209 break
4210 return size, index
4212 def read_byte(delta: bytes) -> int:
4213 nonlocal index
4214 # Bound-check explicitly: ``delta[index:index+1]`` silently returns
4215 # b"" past the end, which would crash with TypeError in ``ord`` and
4216 # leave the caller unable to distinguish a truncated delta from a
4217 # programming bug.
4218 if index >= delta_length:
4219 raise ApplyDeltaError("delta truncated in copy op")
4220 index += 1
4221 return ord(delta[index - 1 : index])
4223 src_size, index = get_delta_header_size(delta, index)
4224 dest_size, index = get_delta_header_size(delta, index)
4225 if src_size != len(src_buf):
4226 raise ApplyDeltaError(
4227 f"Unexpected source buffer size: {src_size} vs {len(src_buf)}"
4228 )
4229 remaining = dest_size
4230 while index < delta_length:
4231 cmd = ord(delta[index : index + 1])
4232 index += 1
4233 if cmd & 0x80:
4234 cp_off = 0
4235 for i in range(4):
4236 if cmd & (1 << i):
4237 x = read_byte(delta)
4238 cp_off |= x << (i * 8)
4239 cp_size = 0
4240 # Version 3 packs can contain copy sizes larger than 64K.
4241 for i in range(3):
4242 if cmd & (1 << (4 + i)):
4243 x = read_byte(delta)
4244 cp_size |= x << (i * 8)
4245 if cp_size == 0:
4246 cp_size = 0x10000
4247 if (
4248 cp_off + cp_size < cp_size
4249 or cp_off + cp_size > src_size
4250 or cp_size > remaining
4251 ):
4252 break
4253 out.append(src_buf[cp_off : cp_off + cp_size])
4254 remaining -= cp_size
4255 elif cmd != 0:
4256 if cmd > remaining:
4257 raise ApplyDeltaError("Not enough space to copy")
4258 if index + cmd > delta_length:
4259 raise ApplyDeltaError("delta truncated in insert op")
4260 out.append(delta[index : index + cmd])
4261 index += cmd
4262 remaining -= cmd
4263 else:
4264 raise ApplyDeltaError("Invalid opcode 0")
4266 if index != delta_length:
4267 raise ApplyDeltaError(f"delta not empty: {delta[index:]!r}")
4269 if dest_size != chunks_length(out):
4270 raise ApplyDeltaError("dest size incorrect")
4272 return out
4275apply_delta = _apply_delta_py
4278def write_pack_index_v2(
4279 f: IO[bytes],
4280 entries: Iterable[tuple[bytes, int, int | None]],
4281 pack_checksum: bytes,
4282) -> bytes:
4283 """Write a new pack index file.
4285 Args:
4286 f: File-like object to write to
4287 entries: List of tuples with object name (sha), offset_in_pack, and
4288 crc32_checksum.
4289 pack_checksum: Checksum of the pack file.
4290 Returns: The checksum of the index file written
4291 """
4292 # Determine hash algorithm from pack_checksum length
4293 if len(pack_checksum) == 20:
4294 hash_func = sha1
4295 elif len(pack_checksum) == 32:
4296 hash_func = sha256
4297 else:
4298 raise ValueError(f"Unsupported pack checksum length: {len(pack_checksum)}")
4300 f_writer = HashWriter(f, hash_func)
4301 f_writer.write(b"\377tOc") # Magic!
4302 f_writer.write(struct.pack(">L", 2))
4304 # Convert to list to allow multiple iterations
4305 entries_list = list(entries)
4307 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4308 for name, offset, entry_checksum in entries_list:
4309 fan_out_table[ord(name[:1])] += 1
4311 if entries_list:
4312 hash_size = len(entries_list[0][0])
4313 else:
4314 hash_size = len(pack_checksum) # Use pack_checksum length as hash size
4316 # Fan-out table
4317 largetable: list[int] = []
4318 for i in range(0x100):
4319 f_writer.write(struct.pack(b">L", fan_out_table[i]))
4320 fan_out_table[i + 1] += fan_out_table[i]
4321 for name, offset, entry_checksum in entries_list:
4322 if len(name) != hash_size:
4323 raise TypeError(
4324 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4325 )
4326 f_writer.write(name)
4327 for name, offset, entry_checksum in entries_list:
4328 f_writer.write(struct.pack(b">L", entry_checksum))
4329 for name, offset, entry_checksum in entries_list:
4330 if offset < 2**31:
4331 f_writer.write(struct.pack(b">L", offset))
4332 else:
4333 f_writer.write(struct.pack(b">L", 2**31 + len(largetable)))
4334 largetable.append(offset)
4335 for offset in largetable:
4336 f_writer.write(struct.pack(b">Q", offset))
4337 f_writer.write(pack_checksum)
4338 return f_writer.write_hash()
4341def write_pack_index_v3(
4342 f: IO[bytes],
4343 entries: Iterable[tuple[bytes, int, int | None]],
4344 pack_checksum: bytes,
4345 hash_format: int = 1,
4346) -> bytes:
4347 """Write a new pack index file in v3 format.
4349 Args:
4350 f: File-like object to write to
4351 entries: List of tuples with object name (sha), offset_in_pack, and
4352 crc32_checksum.
4353 pack_checksum: Checksum of the pack file.
4354 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
4355 Returns: The SHA of the index file written
4356 """
4357 if hash_format == 1:
4358 hash_size = 20 # SHA-1
4359 writer_cls = SHA1Writer
4360 elif hash_format == 2:
4361 hash_size = 32 # SHA-256
4362 # TODO: Add SHA256Writer when SHA-256 support is implemented
4363 raise NotImplementedError("SHA-256 support not yet implemented")
4364 else:
4365 raise ValueError(f"Unknown hash algorithm {hash_format}")
4367 # Convert entries to list to allow multiple iterations
4368 entries_list = list(entries)
4370 # Calculate shortest unambiguous prefix length for object names
4371 # For now, use full hash size (this could be optimized)
4372 shortened_oid_len = hash_size
4374 f = writer_cls(f)
4375 f.write(b"\377tOc") # Magic!
4376 f.write(struct.pack(">L", 3)) # Version 3
4377 f.write(struct.pack(">L", hash_format)) # Hash algorithm
4378 f.write(struct.pack(">L", shortened_oid_len)) # Shortened OID length
4380 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4381 for name, offset, entry_checksum in entries_list:
4382 if len(name) != hash_size:
4383 raise ValueError(
4384 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4385 )
4386 fan_out_table[ord(name[:1])] += 1
4388 # Fan-out table
4389 largetable: list[int] = []
4390 for i in range(0x100):
4391 f.write(struct.pack(b">L", fan_out_table[i]))
4392 fan_out_table[i + 1] += fan_out_table[i]
4394 # Object names table
4395 for name, offset, entry_checksum in entries_list:
4396 f.write(name)
4398 # CRC32 checksums table
4399 for name, offset, entry_checksum in entries_list:
4400 f.write(struct.pack(b">L", entry_checksum))
4402 # Offset table
4403 for name, offset, entry_checksum in entries_list:
4404 if offset < 2**31:
4405 f.write(struct.pack(b">L", offset))
4406 else:
4407 f.write(struct.pack(b">L", 2**31 + len(largetable)))
4408 largetable.append(offset)
4410 # Large offset table
4411 for offset in largetable:
4412 f.write(struct.pack(b">Q", offset))
4414 assert len(pack_checksum) == hash_size, (
4415 f"Pack checksum has wrong length: expected {hash_size}, got {len(pack_checksum)}"
4416 )
4417 f.write(pack_checksum)
4418 return f.write_sha()
4421def write_pack_index(
4422 f: IO[bytes],
4423 entries: Iterable[tuple[bytes, int, int | None]],
4424 pack_checksum: bytes,
4425 progress: Callable[..., None] | None = None,
4426 version: int | None = None,
4427) -> bytes:
4428 """Write a pack index file.
4430 Args:
4431 f: File-like object to write to.
4432 entries: List of (checksum, offset, crc32) tuples
4433 pack_checksum: Checksum of the pack file.
4434 progress: Progress function (not currently used)
4435 version: Pack index version to use (1, 2, or 3). If None, defaults to DEFAULT_PACK_INDEX_VERSION.
4437 Returns:
4438 SHA of the written index file
4440 Raises:
4441 ValueError: If an unsupported version is specified
4442 """
4443 if version is None:
4444 version = DEFAULT_PACK_INDEX_VERSION
4446 match version:
4447 case 1:
4448 return write_pack_index_v1(f, entries, pack_checksum)
4449 case 2:
4450 return write_pack_index_v2(f, entries, pack_checksum)
4451 case 3:
4452 return write_pack_index_v3(f, entries, pack_checksum)
4453 case _:
4454 raise ValueError(f"Unsupported pack index version: {version}")
4457class Pack:
4458 """A Git pack object."""
4460 _data_load: Callable[[], PackData] | None
4461 _idx_load: Callable[[], PackIndex] | None
4463 _data: PackData | None
4464 _idx: PackIndex | None
4465 _bitmap: "PackBitmap | None"
4467 def __init__(
4468 self,
4469 basename: str,
4470 *,
4471 object_format: ObjectFormat,
4472 resolve_ext_ref: ResolveExtRefFn | None = None,
4473 delta_window_size: int | None = None,
4474 window_memory: int | None = None,
4475 delta_cache_size: int | None = None,
4476 depth: int | None = None,
4477 threads: int | None = None,
4478 big_file_threshold: int | None = None,
4479 delta_base_cache_limit: int | None = None,
4480 ) -> None:
4481 """Initialize a Pack object.
4483 Args:
4484 basename: Base path for pack files (without .pack/.idx extension)
4485 object_format: Hash algorithm used by the repository
4486 resolve_ext_ref: Optional function to resolve external references
4487 delta_window_size: Size of the delta compression window
4488 window_memory: Memory limit for delta compression window
4489 delta_cache_size: Size of the delta cache
4490 depth: Maximum depth for delta chains
4491 threads: Number of threads to use for operations
4492 big_file_threshold: Size threshold for big file handling
4493 delta_base_cache_limit: Maximum bytes for delta base object cache
4494 """
4495 self._basename = basename
4496 self.object_format = object_format
4497 self._data = None
4498 self._idx = None
4499 self._bitmap = None
4500 self._idx_path = self._basename + ".idx"
4501 self._data_path = self._basename + ".pack"
4502 self._bitmap_path = self._basename + ".bitmap"
4503 self.delta_window_size = delta_window_size
4504 self.window_memory = window_memory
4505 self.delta_cache_size = delta_cache_size
4506 self.depth = depth
4507 self.threads = threads
4508 self.big_file_threshold = big_file_threshold
4509 self.delta_base_cache_limit = delta_base_cache_limit
4510 self._idx_load = lambda: load_pack_index(self._idx_path, object_format)
4511 self._data_load = lambda: PackData(
4512 self._data_path,
4513 delta_window_size=delta_window_size,
4514 window_memory=window_memory,
4515 delta_cache_size=delta_cache_size,
4516 depth=depth,
4517 threads=threads,
4518 big_file_threshold=big_file_threshold,
4519 delta_base_cache_limit=delta_base_cache_limit,
4520 object_format=object_format,
4521 )
4522 self.resolve_ext_ref = resolve_ext_ref
4524 @classmethod
4525 def from_lazy_objects(
4526 cls,
4527 data_fn: Callable[[], PackData],
4528 idx_fn: Callable[[], PackIndex],
4529 ) -> "Pack":
4530 """Create a new pack object from callables to load pack data and index objects."""
4531 # Load index to get object format
4532 idx = idx_fn()
4533 ret = cls("", object_format=idx.object_format)
4534 ret._data_load = data_fn
4535 ret._idx = idx
4536 ret._idx_load = None
4537 return ret
4539 @classmethod
4540 def from_objects(cls, data: PackData, idx: PackIndex) -> "Pack":
4541 """Create a new pack object from pack data and index objects."""
4542 ret = cls("", object_format=idx.object_format)
4543 ret._data = data
4544 ret._data_load = None
4545 ret._idx = idx
4546 ret._idx_load = None
4547 ret.check_length_and_checksum()
4548 return ret
4550 def name(self) -> bytes:
4551 """The SHA over the SHAs of the objects in this pack."""
4552 return self.index.objects_sha1()
4554 @property
4555 def data(self) -> PackData:
4556 """The pack data object being used."""
4557 if self._data is None:
4558 assert self._data_load
4559 try:
4560 self._data = self._data_load()
4561 except FileNotFoundError as exc:
4562 raise PackFileDisappeared(self) from exc
4563 self.check_length_and_checksum()
4564 return self._data
4566 @property
4567 def index(self) -> PackIndex:
4568 """The index being used.
4570 Note: This may be an in-memory index
4571 """
4572 if self._idx is None:
4573 assert self._idx_load
4574 try:
4575 self._idx = self._idx_load()
4576 except FileNotFoundError as exc:
4577 raise PackFileDisappeared(self) from exc
4578 return self._idx
4580 @property
4581 def bitmap(self) -> "PackBitmap | None":
4582 """The bitmap being used, if available.
4584 Returns:
4585 PackBitmap instance, or None if no bitmap exists or the bitmap
4586 was built for a different pack
4588 Raises:
4589 ValueError: If bitmap file is invalid or corrupt
4590 """
4591 if self._bitmap is None:
4592 from .bitmap import read_bitmap
4594 try:
4595 self._bitmap = read_bitmap(
4596 self._bitmap_path,
4597 pack_index=self.index,
4598 pack_checksum=self.get_stored_checksum(),
4599 )
4600 except ChecksumMismatch:
4601 # The bitmap records the checksum of the pack it was built for.
4602 # A mismatch means it is stale or was swapped in from another
4603 # pack, so its positions no longer describe this pack's objects.
4604 # Ignore it and let callers fall back to graph traversal, the
4605 # same as git.
4606 logger.warning(
4607 "Ignoring bitmap %s: checksum does not match pack",
4608 self._bitmap_path,
4609 )
4610 return None
4611 return self._bitmap
4613 def ensure_bitmap(
4614 self,
4615 object_store: "BaseObjectStore",
4616 refs: dict["Ref", "ObjectID"],
4617 commit_interval: int | None = None,
4618 progress: Callable[[str], None] | None = None,
4619 ) -> "PackBitmap":
4620 """Ensure a bitmap exists for this pack, generating one if needed.
4622 Args:
4623 object_store: Object store to read objects from
4624 refs: Dictionary of ref names to commit SHAs
4625 commit_interval: Include every Nth commit in bitmap index
4626 progress: Optional progress reporting callback
4628 Returns:
4629 PackBitmap instance (either existing or newly generated)
4630 """
4631 from .bitmap import generate_bitmap, write_bitmap
4633 # Check if bitmap already exists
4634 try:
4635 existing = self.bitmap
4636 if existing is not None:
4637 return existing
4638 except FileNotFoundError:
4639 pass # No bitmap, we'll generate one
4641 # Generate new bitmap
4642 if progress:
4643 progress(f"Generating bitmap for {self.name().decode('utf-8')}...\n")
4645 pack_bitmap = generate_bitmap(
4646 self.index,
4647 object_store,
4648 refs,
4649 self.get_stored_checksum(),
4650 commit_interval=commit_interval,
4651 progress=progress,
4652 )
4654 # Write bitmap file
4655 write_bitmap(self._bitmap_path, pack_bitmap)
4657 if progress:
4658 progress(f"Wrote {self._bitmap_path}\n")
4660 # Update cached bitmap
4661 self._bitmap = pack_bitmap
4663 return pack_bitmap
4665 @property
4666 def mmap_size(self) -> int:
4667 """Return the total mmapped memory usage of this pack.
4669 This includes the pack data file and index file sizes,
4670 but only for components that have been loaded (and thus mmapped).
4671 """
4672 total = 0
4673 if self._data is not None:
4674 total += self._data._size
4675 if self._idx is not None and isinstance(self._idx, FilePackIndex):
4676 total += self._idx._size
4677 return total
4679 def close(self) -> None:
4680 """Close the pack file and index."""
4681 if self._data is not None:
4682 self._data.close()
4683 self._data = None
4684 if self._idx is not None:
4685 self._idx.close()
4686 self._idx = None
4688 def __del__(self) -> None:
4689 """Ensure pack file is closed when Pack is garbage collected."""
4690 if self._data is not None or self._idx is not None:
4691 import warnings
4693 warnings.warn(
4694 f"unclosed Pack {self!r}", ResourceWarning, stacklevel=2, source=self
4695 )
4696 try:
4697 self.close()
4698 except Exception:
4699 # Ignore errors during cleanup
4700 pass
4702 def __enter__(self) -> Self:
4703 """Enter context manager."""
4704 return self
4706 def __exit__(
4707 self,
4708 type: type | None,
4709 value: BaseException | None,
4710 traceback: TracebackType | None,
4711 ) -> None:
4712 """Exit context manager."""
4713 self.close()
4715 def __eq__(self, other: object) -> bool:
4716 """Check equality with another pack."""
4717 if not isinstance(other, Pack):
4718 return False
4719 return self.index == other.index
4721 def __len__(self) -> int:
4722 """Number of entries in this pack."""
4723 return len(self.index)
4725 def __repr__(self) -> str:
4726 """Return string representation of this pack."""
4727 return f"{self.__class__.__name__}({self._basename!r})"
4729 def __iter__(self) -> Iterator[ObjectID]:
4730 """Iterate over all the sha1s of the objects in this pack."""
4731 return iter(self.index)
4733 def check_length_and_checksum(self) -> None:
4734 """Sanity check the length and checksum of the pack index and data."""
4735 assert len(self.index) == len(self.data), (
4736 f"Length mismatch: {len(self.index)} (index) != {len(self.data)} (data)"
4737 )
4738 idx_stored_checksum = self.index.get_pack_checksum()
4739 data_stored_checksum = self.data.get_stored_checksum()
4740 if (
4741 idx_stored_checksum is not None
4742 and idx_stored_checksum != data_stored_checksum
4743 ):
4744 raise ChecksumMismatch(
4745 sha_to_hex(RawObjectID(idx_stored_checksum)),
4746 sha_to_hex(RawObjectID(data_stored_checksum)),
4747 )
4749 def check(self) -> None:
4750 """Check the integrity of this pack.
4752 Raises:
4753 ChecksumMismatch: if a checksum for the index or data is wrong
4754 """
4755 self.index.check()
4756 self.data.check()
4757 for obj in self.iterobjects():
4758 obj.check()
4759 # TODO: object connectivity checks
4761 def get_stored_checksum(self) -> bytes:
4762 """Return the stored checksum of the pack data."""
4763 return self.data.get_stored_checksum()
4765 def pack_tuples(self) -> list[tuple[ShaFile, None]]:
4766 """Return pack tuples for all objects in pack."""
4767 return [(o, None) for o in self.iterobjects()]
4769 def __contains__(self, sha1: ObjectID | RawObjectID) -> bool:
4770 """Check whether this pack contains a particular SHA1."""
4771 try:
4772 self.index.object_offset(sha1)
4773 return True
4774 except KeyError:
4775 return False
4777 def get_raw(self, sha1: RawObjectID | ObjectID) -> tuple[int, bytes]:
4778 """Get raw object data by SHA1."""
4779 offset = self.index.object_offset(sha1)
4780 obj_type, obj = self.data.get_object_at(offset)
4781 type_num, chunks = self.resolve_object(offset, obj_type, obj)
4782 return type_num, b"".join(chunks) # type: ignore[arg-type]
4784 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
4785 """Retrieve the specified SHA1."""
4786 type, uncomp = self.get_raw(sha1)
4787 return ShaFile.from_raw_string(type, uncomp, sha=sha1)
4789 def iterobjects(self) -> Iterator[ShaFile]:
4790 """Iterate over the objects in this pack."""
4791 return iter(
4792 PackInflater.for_pack_data(self.data, resolve_ext_ref=self.resolve_ext_ref)
4793 )
4795 def iterobjects_subset(
4796 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
4797 ) -> Iterator[ShaFile]:
4798 """Iterate over a subset of objects in this pack."""
4799 return (
4800 uo
4801 for uo in PackInflater.for_pack_subset(
4802 self,
4803 shas,
4804 allow_missing=allow_missing,
4805 resolve_ext_ref=self.resolve_ext_ref,
4806 )
4807 if uo.id in shas
4808 )
4810 def iter_unpacked_subset(
4811 self,
4812 shas: Iterable[ObjectID | RawObjectID],
4813 *,
4814 include_comp: bool = False,
4815 allow_missing: bool = False,
4816 convert_ofs_delta: bool = False,
4817 ) -> Iterator[UnpackedObject]:
4818 """Iterate over unpacked objects in subset."""
4819 ofs_pending: dict[int, list[UnpackedObject]] = defaultdict(list)
4820 ofs: dict[int, bytes] = {}
4821 todo: set[ObjectID | RawObjectID] = set(shas)
4822 for unpacked in self.iter_unpacked(include_comp=include_comp):
4823 sha = unpacked.sha()
4824 if unpacked.offset is not None:
4825 ofs[unpacked.offset] = sha
4826 hexsha = sha_to_hex(RawObjectID(sha))
4827 if hexsha in todo:
4828 if unpacked.pack_type_num == OFS_DELTA:
4829 assert isinstance(unpacked.delta_base, int)
4830 assert unpacked.offset is not None
4831 base_offset = unpacked.offset - unpacked.delta_base
4832 try:
4833 unpacked.delta_base = ofs[base_offset]
4834 except KeyError:
4835 ofs_pending[base_offset].append(unpacked)
4836 continue
4837 else:
4838 unpacked.pack_type_num = REF_DELTA
4839 yield unpacked
4840 todo.remove(hexsha)
4841 if unpacked.offset is not None:
4842 for child in ofs_pending.pop(unpacked.offset, []):
4843 child.pack_type_num = REF_DELTA
4844 child.delta_base = sha
4845 yield child
4846 assert not ofs_pending
4847 if not allow_missing and todo:
4848 raise UnresolvedDeltas(list(todo))
4850 def iter_unpacked(self, include_comp: bool = False) -> Iterator[UnpackedObject]:
4851 """Iterate over all unpacked objects in this pack."""
4852 ofs_to_entries = {
4853 ofs: (sha, crc32) for (sha, ofs, crc32) in self.index.iterentries()
4854 }
4855 for unpacked in self.data.iter_unpacked(include_comp=include_comp):
4856 assert unpacked.offset is not None
4857 (sha, crc32) = ofs_to_entries[unpacked.offset]
4858 unpacked._sha = sha
4859 unpacked.crc32 = crc32
4860 yield unpacked
4862 def keep(self, msg: bytes | None = None) -> str:
4863 """Add a .keep file for the pack, preventing git from garbage collecting it.
4865 Args:
4866 msg: A message written inside the .keep file; can be used later
4867 to determine whether or not a .keep file is obsolete.
4868 Returns: The path of the .keep file, as a string.
4869 """
4870 keepfile_name = f"{self._basename}.keep"
4871 with GitFile(keepfile_name, "wb") as keepfile:
4872 if msg:
4873 keepfile.write(msg)
4874 keepfile.write(b"\n")
4875 return keepfile_name
4877 def unkeep(self) -> bool:
4878 """Remove the .keep file for the pack, allowing git to garbage collect it.
4880 This is the counterpart of :meth:`keep`. It is not an error to call
4881 this on a pack that has no .keep file.
4883 Returns: True if a .keep file was removed, False if there was none.
4884 """
4885 try:
4886 os.unlink(f"{self._basename}.keep")
4887 except FileNotFoundError:
4888 return False
4889 return True
4891 def get_ref(
4892 self, sha: RawObjectID | ObjectID
4893 ) -> tuple[int | None, int, OldUnpackedObject]:
4894 """Get the object for a ref SHA, only looking in this pack."""
4895 # TODO: cache these results
4896 try:
4897 offset = self.index.object_offset(sha)
4898 except KeyError:
4899 offset = None
4900 if offset:
4901 type, obj = self.data.get_object_at(offset)
4902 elif self.resolve_ext_ref:
4903 type, obj = self.resolve_ext_ref(sha)
4904 else:
4905 raise KeyError(sha)
4906 return offset, type, obj
4908 def resolve_object(
4909 self,
4910 offset: int,
4911 type: int,
4912 obj: OldUnpackedObject,
4913 get_ref: Callable[
4914 [RawObjectID | ObjectID], tuple[int | None, int, OldUnpackedObject]
4915 ]
4916 | None = None,
4917 ) -> tuple[int, OldUnpackedObject]:
4918 """Resolve an object, possibly resolving deltas when necessary.
4920 Returns: Tuple with object type and contents.
4921 """
4922 # Walk down the delta chain, building a stack of deltas to reach
4923 # the requested object.
4924 base_offset: int | None = offset
4925 base_type = type
4926 base_obj = obj
4927 delta_stack = []
4928 seen_ref_offsets: set[int] = {offset}
4929 while base_type in DELTA_TYPES:
4930 prev_offset = base_offset
4931 if get_ref is None:
4932 get_ref = self.get_ref
4933 assert isinstance(base_obj, tuple), (
4934 f"Expected delta tuple, got {base_obj.__class__.__name__}"
4935 )
4936 if base_type == OFS_DELTA:
4937 (delta_offset, delta) = base_obj
4938 # TODO: clean up asserts and replace with nicer error messages
4939 assert isinstance(delta_offset, int), (
4940 f"Expected int, got {delta_offset.__class__}"
4941 )
4942 assert base_offset is not None
4943 base_offset = base_offset - delta_offset
4944 base_type, base_obj = self.data.get_object_at(base_offset)
4945 assert isinstance(base_type, int)
4946 elif base_type == REF_DELTA:
4947 (basename, delta) = base_obj
4948 assert (
4949 isinstance(basename, bytes)
4950 and len(basename) == self.object_format.oid_length
4951 )
4952 base_offset_temp, base_type, base_obj = get_ref(RawObjectID(basename))
4953 assert isinstance(base_type, int)
4954 # base_offset_temp can be None for thin packs (external references)
4955 base_offset = base_offset_temp
4956 if base_offset == prev_offset: # object is based on itself
4957 raise UnresolvedDeltas([basename])
4958 # A repeated REF_DELTA base offset means the chain cycles;
4959 # without this check the loop would never terminate.
4960 if base_offset is not None:
4961 if base_offset in seen_ref_offsets:
4962 raise DeltaCycle([basename])
4963 seen_ref_offsets.add(base_offset)
4964 else:
4965 raise AssertionError(f"Unexpected delta type: {base_type}")
4966 delta_stack.append((prev_offset, base_type, delta))
4968 # Now grab the base object (mustn't be a delta) and apply the
4969 # deltas all the way up the stack.
4970 chunks = base_obj
4971 for prev_offset, _delta_type, delta in reversed(delta_stack):
4972 # Convert chunks to bytes for apply_delta if needed
4973 if isinstance(chunks, list):
4974 chunks_bytes = b"".join(chunks)
4975 elif isinstance(chunks, tuple):
4976 # For tuple type, second element is the actual data
4977 _, chunk_data = chunks
4978 if isinstance(chunk_data, list):
4979 chunks_bytes = b"".join(chunk_data)
4980 else:
4981 chunks_bytes = chunk_data
4982 else:
4983 chunks_bytes = chunks
4985 # Apply delta and get result as list
4986 chunks = apply_delta(chunks_bytes, delta)
4988 if prev_offset is not None:
4989 self.data._cache_object_at(prev_offset, base_type, chunks)
4990 return base_type, chunks
4992 def entries(
4993 self, progress: Callable[[int, int], None] | None = None
4994 ) -> Iterator[PackIndexEntry]:
4995 """Yield entries summarizing the contents of this pack.
4997 Args:
4998 progress: Progress function, called with current and total
4999 object count.
5000 Returns: iterator of tuples with (sha, offset, crc32)
5001 """
5002 return self.data.iterentries(
5003 progress=progress, resolve_ext_ref=self.resolve_ext_ref
5004 )
5006 def sorted_entries(
5007 self, progress: Callable[[int, int], None] | None = None
5008 ) -> Iterator[PackIndexEntry]:
5009 """Return entries in this pack, sorted by SHA.
5011 Args:
5012 progress: Progress function, called with current and total
5013 object count
5014 Returns: Iterator of tuples with (sha, offset, crc32)
5015 """
5016 return iter(
5017 self.data.sorted_entries(
5018 progress=progress, resolve_ext_ref=self.resolve_ext_ref
5019 )
5020 )
5022 def get_unpacked_object(
5023 self,
5024 sha: ObjectID | RawObjectID,
5025 *,
5026 include_comp: bool = False,
5027 convert_ofs_delta: bool = True,
5028 ) -> UnpackedObject:
5029 """Get the unpacked object for a sha.
5031 Args:
5032 sha: SHA of object to fetch
5033 include_comp: Whether to include compression data in UnpackedObject
5034 convert_ofs_delta: Whether to convert offset deltas to ref deltas
5035 """
5036 offset = self.index.object_offset(sha)
5037 unpacked = self.data.get_unpacked_object_at(offset, include_comp=include_comp)
5038 if unpacked.pack_type_num == OFS_DELTA and convert_ofs_delta:
5039 assert isinstance(unpacked.delta_base, int)
5040 unpacked.delta_base = self.index.object_sha1(offset - unpacked.delta_base)
5041 unpacked.pack_type_num = REF_DELTA
5042 return unpacked
5045def extend_pack(
5046 f: BinaryIO,
5047 object_ids: Set["RawObjectID"],
5048 get_raw: Callable[["RawObjectID | ObjectID"], tuple[int, bytes]],
5049 object_format: "ObjectFormat",
5050 *,
5051 compression_level: int = -1,
5052 progress: Callable[[bytes], None] | None = None,
5053) -> tuple[bytes, list[tuple[RawObjectID, int, int]]]:
5054 """Extend a pack file with more objects.
5056 The caller should make sure that object_ids does not contain any objects
5057 that are already in the pack
5058 """
5059 # Update the header with the new number of objects.
5060 f.seek(0)
5061 _version, num_objects = read_pack_header(f.read)
5063 if object_ids:
5064 f.seek(0)
5065 write_pack_header(f.write, num_objects + len(object_ids))
5067 # Must flush before reading (http://bugs.python.org/issue3207)
5068 f.flush()
5070 # Rescan the rest of the pack, computing the SHA with the new header.
5071 new_sha = compute_file_sha(
5072 f, hash_func=object_format.hash_func, end_ofs=-object_format.oid_length
5073 )
5075 # Must reposition before writing (http://bugs.python.org/issue3207)
5076 f.seek(0, os.SEEK_CUR)
5078 extra_entries = []
5080 # Complete the pack.
5081 for i, object_id in enumerate(object_ids):
5082 if progress is not None:
5083 progress(
5084 (f"writing extra base objects: {i}/{len(object_ids)}\r").encode("ascii")
5085 )
5086 assert len(object_id) == object_format.oid_length
5087 type_num, data = get_raw(object_id)
5088 offset = f.tell()
5089 crc32 = write_pack_object(
5090 f.write,
5091 type_num,
5092 [data], # Convert bytes to list[bytes]
5093 sha=new_sha,
5094 compression_level=compression_level,
5095 object_format=object_format,
5096 )
5097 extra_entries.append((object_id, offset, crc32))
5098 pack_sha = new_sha.digest()
5099 f.write(pack_sha)
5100 return pack_sha, extra_entries
5103try:
5104 from dulwich._pack import ( # type: ignore
5105 apply_delta,
5106 bisect_find_sha,
5107 )
5108except ImportError:
5109 pass
5111# Try to import the Rust version of create_delta
5112try:
5113 from dulwich._pack import create_delta as _create_delta_rs
5114except ImportError:
5115 pass
5116else:
5117 # Wrap the Rust version to match the Python API (returns bytes instead of Iterator)
5118 def _create_delta_rs_wrapper(
5119 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
5120 ) -> Iterator[bytes]:
5121 """Wrapper for Rust create_delta to match Python API."""
5122 if isinstance(base_buf, list):
5123 base_buf = b"".join(base_buf)
5124 if isinstance(target_buf, list):
5125 target_buf = b"".join(target_buf)
5126 yield _create_delta_rs(base_buf, target_buf)
5128 create_delta = _create_delta_rs_wrapper