Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/pack.py: 22%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# pack.py -- For dealing with packed git objects.
2# Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
3# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
23"""Classes for dealing with packed git objects.
25A pack is a compact representation of a bunch of objects, stored
26using deltas where possible.
28They have two parts, the pack file, which stores the data, and an index
29that tells you where the data is.
31To find an object you look in all of the index files 'til you find a
32match for the object name. You then use the pointer got from this as
33a pointer in to the corresponding packfile.
34"""
36__all__ = [
37 "DEFAULT_PACK_DELTA_WINDOW_SIZE",
38 "DEFAULT_PACK_INDEX_VERSION",
39 "DELTA_TYPES",
40 "OFS_DELTA",
41 "PACK_SPOOL_FILE_MAX_SIZE",
42 "REF_DELTA",
43 "DeltaChainIterator",
44 "FilePackIndex",
45 "MemoryPackIndex",
46 "ObjectContainer",
47 "Pack",
48 "PackChunkGenerator",
49 "PackData",
50 "PackFileDisappeared",
51 "PackHint",
52 "PackIndex",
53 "PackIndex1",
54 "PackIndex2",
55 "PackIndex3",
56 "PackIndexEntry",
57 "PackIndexer",
58 "PackInflater",
59 "PackStreamCopier",
60 "PackStreamReader",
61 "PackedObjectContainer",
62 "SHA1Reader",
63 "SHA1Writer",
64 "UnpackedObject",
65 "UnpackedObjectIterator",
66 "UnpackedObjectStream",
67 "UnresolvedDeltas",
68 "apply_delta",
69 "bisect_find_sha",
70 "chunks_length",
71 "compute_file_sha",
72 "deltas_from_sorted_objects",
73 "deltify_pack_objects",
74 "extend_pack",
75 "find_reusable_deltas",
76 "full_unpacked_object",
77 "generate_unpacked_objects",
78 "iter_sha1",
79 "load_pack_index",
80 "load_pack_index_file",
81 "obj_sha",
82 "pack_header_chunks",
83 "pack_object_chunks",
84 "pack_object_header",
85 "pack_objects_to_data",
86 "read_pack_header",
87 "read_zlib_chunks",
88 "sort_objects_for_delta",
89 "take_msb_bytes",
90 "unpack_object",
91 "verify_and_read",
92 "write_pack",
93 "write_pack_data",
94 "write_pack_from_container",
95 "write_pack_header",
96 "write_pack_index",
97 "write_pack_object",
98 "write_pack_objects",
99]
101import binascii
102from collections import defaultdict, deque
103from contextlib import suppress
104from io import BytesIO, UnsupportedOperation
106try:
107 from cdifflib import CSequenceMatcher as SequenceMatcher
108except ModuleNotFoundError:
109 from difflib import SequenceMatcher
111import logging
112import os
113import struct
114import sys
115import warnings
116import zlib
117from collections.abc import Callable, Iterable, Iterator, Sequence, Set
118from hashlib import sha1, sha256
119from itertools import chain
120from os import SEEK_CUR, SEEK_END
121from struct import unpack_from
122from types import TracebackType
123from typing import (
124 IO,
125 TYPE_CHECKING,
126 Any,
127 BinaryIO,
128 Generic,
129 Protocol,
130 TypeVar,
131)
133if sys.version_info >= (3, 11):
134 from typing import Self
135else:
136 from typing_extensions import Self
138import mmap
140from .errors import ApplyDeltaError, ChecksumMismatch
141from .file import GitFile, _GitFile
142from .lru_cache import LRUSizeCache
143from .object_format import OBJECT_FORMAT_TYPE_NUMS, SHA1, ObjectFormat
144from .objects import (
145 ObjectID,
146 RawObjectID,
147 ShaFile,
148 hex_to_sha,
149 object_header,
150 sha_to_hex,
151)
153if TYPE_CHECKING:
154 from _hashlib import HASH as HashObject
156 from .bitmap import PackBitmap
157 from .commit_graph import CommitGraph
158 from .object_store import BaseObjectStore
159 from .refs import Ref
161logger = logging.getLogger(__name__)
163# Some platforms (e.g. plan9) don't support mmap properly
164has_mmap = sys.platform != "Plan9"
166OFS_DELTA = 6
167REF_DELTA = 7
169DELTA_TYPES = (OFS_DELTA, REF_DELTA)
172DEFAULT_PACK_DELTA_WINDOW_SIZE = 10
174# Keep pack files under 16Mb in memory, otherwise write them out to disk
175PACK_SPOOL_FILE_MAX_SIZE = 16 * 1024 * 1024
177# Default pack index version to use when none is specified
178DEFAULT_PACK_INDEX_VERSION = 2
181OldUnpackedObject = tuple[bytes | int, list[bytes]] | list[bytes] | bytes
182ResolveExtRefFn = Callable[[RawObjectID | ObjectID], tuple[int, bytes | list[bytes]]]
183ProgressFn = Callable[[int, str], None]
184PackHint = tuple[int, bytes | None]
187def verify_and_read(
188 read_func: Callable[[int], bytes],
189 expected_hash: bytes,
190 hash_algo: str,
191 progress: Callable[[bytes], None] | None = None,
192) -> Iterator[bytes]:
193 """Read from stream, verify hash, then yield verified chunks.
195 This function downloads data to a temporary file (in-memory for small files,
196 on-disk for large ones) while computing its hash. Only after the hash is
197 verified to match expected_hash will it yield any data. This prevents
198 corrupted or malicious data from reaching the caller.
200 Args:
201 read_func: Function to read bytes (like file.read or HTTP response reader)
202 expected_hash: Expected hash as hex string bytes (e.g., b'a3b2c1...')
203 hash_algo: Hash algorithm name ('sha1' or 'sha256')
204 progress: Optional progress callback
206 Yields:
207 Chunks of verified data (only after hash verification succeeds)
209 Raises:
210 ValueError: If hash doesn't match or algorithm unsupported
211 """
212 from tempfile import SpooledTemporaryFile
214 from .object_format import OBJECT_FORMATS
216 # Get the hash function for this algorithm
217 obj_format = OBJECT_FORMATS.get(hash_algo)
218 if obj_format is None:
219 raise ValueError(f"Unsupported hash algorithm: {hash_algo}")
221 hasher = obj_format.new_hash()
223 # Download to temporary file (memory or disk) while computing hash
224 with SpooledTemporaryFile(
225 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="dulwich-verify-"
226 ) as temp_file:
227 # Read data, hash it, and write to temp file
228 while True:
229 chunk = read_func(65536) # Read in 64KB chunks
230 if not chunk:
231 break
232 hasher.update(chunk)
233 temp_file.write(chunk)
235 # Verify hash BEFORE yielding any data
236 computed_hash = hasher.hexdigest().encode("ascii")
237 if computed_hash != expected_hash:
238 raise ValueError(
239 f"hash mismatch: expected {expected_hash.decode('ascii')}, "
240 f"got {computed_hash.decode('ascii')}"
241 )
243 # Hash verified! Now read from temp file and yield chunks
244 if progress:
245 progress(b"Hash verified, processing data\n")
247 temp_file.seek(0)
248 while True:
249 chunk = temp_file.read(65536)
250 if not chunk:
251 break
252 yield chunk
255class UnresolvedDeltas(Exception):
256 """Delta objects could not be resolved."""
258 def __init__(self, shas: list[bytes]) -> None:
259 """Initialize UnresolvedDeltas exception.
261 Args:
262 shas: List of SHA hashes for unresolved delta objects
263 """
264 self.shas = shas
267class ObjectContainer(Protocol):
268 """Protocol for objects that can contain git objects."""
270 def add_object(self, obj: ShaFile) -> None:
271 """Add a single object to this object store."""
273 def add_objects(
274 self,
275 objects: Sequence[tuple[ShaFile, str | None]],
276 progress: Callable[..., None] | None = None,
277 ) -> "Pack | None":
278 """Add a set of objects to this object store.
280 Args:
281 objects: Iterable over a list of (object, path) tuples
282 progress: Progress callback for object insertion
283 Returns: Optional Pack object of the objects written.
284 """
286 def __contains__(self, sha1: "ObjectID") -> bool:
287 """Check if a hex sha is present."""
288 ...
290 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
291 """Retrieve an object."""
292 ...
294 def get_commit_graph(self) -> "CommitGraph | None":
295 """Get the commit graph for this object store.
297 Returns:
298 CommitGraph object if available, None otherwise
299 """
300 return None
303class PackedObjectContainer(ObjectContainer):
304 """Container for objects packed in a pack file."""
306 def get_unpacked_object(
307 self, sha1: "ObjectID | RawObjectID", *, include_comp: bool = False
308 ) -> "UnpackedObject":
309 """Get a raw unresolved object.
311 Args:
312 sha1: SHA-1 hash of the object
313 include_comp: Whether to include compressed data
315 Returns:
316 UnpackedObject instance
317 """
318 raise NotImplementedError(self.get_unpacked_object)
320 def iterobjects_subset(
321 self, shas: Iterable["ObjectID"], *, allow_missing: bool = False
322 ) -> Iterator[ShaFile]:
323 """Iterate over a subset of objects.
325 Args:
326 shas: Iterable of object SHAs to retrieve
327 allow_missing: If True, skip missing objects
329 Returns:
330 Iterator of ShaFile objects
331 """
332 raise NotImplementedError(self.iterobjects_subset)
334 def iter_unpacked_subset(
335 self,
336 shas: Iterable["ObjectID | RawObjectID"],
337 *,
338 include_comp: bool = False,
339 allow_missing: bool = False,
340 convert_ofs_delta: bool = True,
341 ) -> Iterator["UnpackedObject"]:
342 """Iterate over unpacked objects from a subset of SHAs.
344 Args:
345 shas: Set of object SHAs to retrieve
346 include_comp: Include compressed data if True
347 allow_missing: If True, skip missing objects
348 convert_ofs_delta: If True, convert offset deltas to ref deltas
350 Returns:
351 Iterator of UnpackedObject instances
352 """
353 raise NotImplementedError(self.iter_unpacked_subset)
356class UnpackedObjectStream:
357 """Abstract base class for a stream of unpacked objects."""
359 def __iter__(self) -> Iterator["UnpackedObject"]:
360 """Iterate over unpacked objects."""
361 raise NotImplementedError(self.__iter__)
363 def __len__(self) -> int:
364 """Return the number of objects in the stream."""
365 raise NotImplementedError(self.__len__)
368def take_msb_bytes(
369 read: Callable[[int], bytes], crc32: int | None = None
370) -> tuple[list[int], int | None]:
371 """Read bytes marked with most significant bit.
373 Args:
374 read: Read function
375 crc32: Optional CRC32 checksum to update
377 Returns:
378 Tuple of (list of bytes read, updated CRC32 or None)
379 """
380 ret: list[int] = []
381 while len(ret) == 0 or ret[-1] & 0x80:
382 b = read(1)
383 if crc32 is not None:
384 crc32 = binascii.crc32(b, crc32)
385 ret.append(ord(b[:1]))
386 return ret, crc32
389class PackFileDisappeared(Exception):
390 """Raised when a pack file unexpectedly disappears.
392 This typically happens when a concurrent operation (e.g. ``git repack``
393 or ``git gc --auto``) removes a pack file between the moment dulwich
394 snapshots the pack directory and the moment it actually opens the
395 pack's ``.idx`` or ``.pack`` file.
397 The ``obj`` attribute holds the :class:`Pack` (or :class:`FilePackIndex`)
398 whose backing file vanished, so the caller can evict the stale object
399 from its cache and rescan the pack directory.
400 """
402 obj: "Pack | FilePackIndex"
404 def __init__(self, obj: "Pack | FilePackIndex") -> None:
405 """Initialize PackFileDisappeared exception.
407 Args:
408 obj: The pack or pack index that disappeared.
409 """
410 self.obj = obj
413class UnpackedObject:
414 """Class encapsulating an object unpacked from a pack file.
416 These objects should only be created from within unpack_object. Most
417 members start out as empty and are filled in at various points by
418 read_zlib_chunks, unpack_object, DeltaChainIterator, etc.
420 End users of this object should take care that the function they're getting
421 this object from is guaranteed to set the members they need.
422 """
424 __slots__ = [
425 "_sha", # Cached binary SHA.
426 "comp_chunks", # Compressed object chunks.
427 "crc32", # CRC32.
428 "decomp_chunks", # Decompressed object chunks.
429 "decomp_len", # Decompressed length of this object.
430 "delta_base", # Delta base offset or SHA.
431 "hash_func", # Hash function to use for computing object IDs.
432 "obj_chunks", # Decompressed and delta-resolved chunks.
433 "obj_type_num", # Type of this object.
434 "offset", # Offset in its pack.
435 "pack_type_num", # Type of this object in the pack (may be a delta).
436 ]
438 obj_type_num: int | None
439 obj_chunks: list[bytes] | None
440 delta_base: None | bytes | int
441 decomp_chunks: list[bytes]
442 comp_chunks: list[bytes] | None
443 decomp_len: int | None
444 crc32: int | None
445 offset: int | None
446 pack_type_num: int
447 _sha: bytes | None
448 hash_func: Callable[[], "HashObject"]
450 # TODO(dborowitz): read_zlib_chunks and unpack_object could very well be
451 # methods of this object.
452 def __init__(
453 self,
454 pack_type_num: int,
455 *,
456 delta_base: None | bytes | int = None,
457 decomp_len: int | None = None,
458 crc32: int | None = None,
459 sha: bytes | None = None,
460 decomp_chunks: list[bytes] | None = None,
461 offset: int | None = None,
462 hash_func: Callable[[], "HashObject"] = sha1,
463 ) -> None:
464 """Initialize an UnpackedObject.
466 Args:
467 pack_type_num: Type number of this object in the pack
468 delta_base: Delta base (offset or SHA) if this is a delta object
469 decomp_len: Decompressed length of this object
470 crc32: CRC32 checksum
471 sha: SHA hash of the object
472 decomp_chunks: Decompressed chunks
473 offset: Offset in the pack file
474 hash_func: Hash function to use (defaults to sha1)
475 """
476 self.offset = offset
477 self._sha = sha
478 self.pack_type_num = pack_type_num
479 self.delta_base = delta_base
480 self.comp_chunks = None
481 self.decomp_chunks: list[bytes] = decomp_chunks or []
482 if decomp_chunks is not None and decomp_len is None:
483 self.decomp_len = sum(map(len, decomp_chunks))
484 else:
485 self.decomp_len = decomp_len
486 self.crc32 = crc32
487 self.hash_func = hash_func
489 if pack_type_num in DELTA_TYPES:
490 self.obj_type_num = None
491 self.obj_chunks = None
492 else:
493 self.obj_type_num = pack_type_num
494 self.obj_chunks = self.decomp_chunks
495 self.delta_base = delta_base
497 def sha(self) -> RawObjectID:
498 """Return the binary SHA of this object."""
499 if self._sha is None:
500 assert self.obj_type_num is not None and self.obj_chunks is not None
501 self._sha = obj_sha(self.obj_type_num, self.obj_chunks, self.hash_func)
502 return RawObjectID(self._sha)
504 def sha_file(self) -> ShaFile:
505 """Return a ShaFile from this object."""
506 assert self.obj_type_num is not None and self.obj_chunks is not None
507 return ShaFile.from_raw_chunks(self.obj_type_num, self.obj_chunks)
509 # Only provided for backwards compatibility with code that expects either
510 # chunks or a delta tuple.
511 def _obj(self) -> OldUnpackedObject:
512 """Return the decompressed chunks, or (delta base, delta chunks)."""
513 if self.pack_type_num in DELTA_TYPES:
514 assert isinstance(self.delta_base, bytes | int)
515 return (self.delta_base, self.decomp_chunks)
516 else:
517 return self.decomp_chunks
519 def __eq__(self, other: object) -> bool:
520 """Check equality with another UnpackedObject."""
521 if not isinstance(other, UnpackedObject):
522 return False
523 for slot in self.__slots__:
524 if getattr(self, slot) != getattr(other, slot):
525 return False
526 return True
528 def __ne__(self, other: object) -> bool:
529 """Check inequality with another UnpackedObject."""
530 return not (self == other)
532 def __repr__(self) -> str:
533 """Return string representation of this UnpackedObject."""
534 data = [f"{s}={getattr(self, s)!r}" for s in self.__slots__]
535 return "{}({})".format(self.__class__.__name__, ", ".join(data))
538_ZLIB_BUFSIZE = 65536 # 64KB buffer for better I/O performance
540# Default maximum memory for caching delta base objects (matches Git's default
541# for core.deltaBaseCacheLimit).
542DEFAULT_DELTA_BASE_CACHE_LIMIT = 96 * 1024 * 1024 # 96 MiB
545def read_zlib_chunks(
546 read_some: Callable[[int], bytes],
547 unpacked: UnpackedObject,
548 include_comp: bool = False,
549 buffer_size: int = _ZLIB_BUFSIZE,
550) -> bytes:
551 """Read zlib data from a buffer.
553 This function requires that the buffer have additional data following the
554 compressed data, which is guaranteed to be the case for git pack files.
556 Args:
557 read_some: Read function that returns at least one byte, but may
558 return less than the requested size.
559 unpacked: An UnpackedObject to write result data to. If its crc32
560 attr is not None, the CRC32 of the compressed bytes will be computed
561 using this starting CRC32.
562 After this function, will have the following attrs set:
563 * comp_chunks (if include_comp is True)
564 * decomp_chunks
565 * decomp_len
566 * crc32
567 include_comp: If True, include compressed data in the result.
568 buffer_size: Size of the read buffer.
569 Returns: Leftover unused data from the decompression.
571 Raises:
572 zlib.error: if a decompression error occurred.
573 """
574 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
575 raise ValueError("non-negative zlib data stream size expected")
576 decomp_obj = zlib.decompressobj()
578 comp_chunks = []
579 decomp_chunks = unpacked.decomp_chunks
580 decomp_len = 0
581 crc32 = unpacked.crc32
582 max_decomp = unpacked.decomp_len
584 while True:
585 add = read_some(buffer_size)
586 if not add:
587 raise zlib.error("EOF before end of zlib stream")
588 comp_chunks.append(add)
589 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
590 remaining = max_decomp - decomp_len + 1
591 decomp = decomp_obj.decompress(add, remaining)
592 if decomp_obj.unconsumed_tail:
593 raise zlib.error("decompressed data exceeds expected size")
594 decomp_len += len(decomp)
595 decomp_chunks.append(decomp)
596 unused = decomp_obj.unused_data
597 if unused:
598 left = len(unused)
599 if crc32 is not None:
600 crc32 = binascii.crc32(add[:-left], crc32)
601 if include_comp:
602 comp_chunks[-1] = add[:-left]
603 break
604 elif crc32 is not None:
605 crc32 = binascii.crc32(add, crc32)
606 if crc32 is not None:
607 crc32 &= 0xFFFFFFFF
609 if decomp_len != unpacked.decomp_len:
610 raise zlib.error("decompressed data does not match expected size")
612 unpacked.crc32 = crc32
613 if include_comp:
614 unpacked.comp_chunks = comp_chunks
615 return unused
618def iter_sha1(iter: Iterable[bytes]) -> bytes:
619 """Return the hexdigest of the SHA1 over a set of names.
621 Args:
622 iter: Iterator over string objects
623 Returns: 40-byte hex sha1 digest
624 """
625 sha = sha1()
626 for name in iter:
627 sha.update(name)
628 return sha.hexdigest().encode("ascii")
631def load_pack_index(
632 path: str | os.PathLike[str], object_format: ObjectFormat
633) -> "PackIndex":
634 """Load an index file by path.
636 Args:
637 path: Path to the index file
638 object_format: Hash algorithm used by the repository
639 Returns: A PackIndex loaded from the given path
640 """
641 # Ownership of the file is transferred to the returned index, which mmaps
642 # it and closes it in PackIndex.close(). It must not be closed here: on
643 # Windows an mmap keeps the file locked, so closing the handle out from
644 # under a live mapping leaves the .idx undeletable until the index is GCed.
645 f = GitFile(path, "rb")
646 try:
647 return load_pack_index_file(path, f, object_format)
648 except BaseException:
649 f.close()
650 raise
653def _load_file_contents(
654 f: IO[bytes] | _GitFile, size: int | None = None
655) -> tuple[bytes | Any, int]:
656 """Load contents from a file, preferring mmap when possible.
658 Args:
659 f: File-like object to load
660 size: Expected size, or None to determine from file
661 Returns: Tuple of (contents, size)
662 """
663 try:
664 fd = f.fileno()
665 except (UnsupportedOperation, AttributeError):
666 fd = None
667 # Attempt to use mmap if possible
668 if fd is not None:
669 if size is None:
670 size = os.fstat(fd).st_size
671 if has_mmap:
672 try:
673 contents = mmap.mmap(fd, size, access=mmap.ACCESS_READ)
674 except (OSError, ValueError):
675 # Can't mmap - perhaps a socket or invalid file descriptor
676 pass
677 else:
678 return contents, size
679 contents_bytes = f.read()
680 size = len(contents_bytes)
681 return contents_bytes, size
684def load_pack_index_file(
685 path: str | os.PathLike[str],
686 f: IO[bytes] | _GitFile,
687 object_format: ObjectFormat,
688) -> "PackIndex":
689 """Load an index file from a file-like object.
691 Args:
692 path: Path for the index file
693 f: File-like object
694 object_format: Hash algorithm used by the repository
695 Returns: A PackIndex loaded from the given file
696 """
697 contents, size = _load_file_contents(f)
698 if contents[:4] == b"\377tOc":
699 version = struct.unpack(b">L", contents[4:8])[0]
700 if version == 2:
701 return PackIndex2(
702 path,
703 object_format,
704 file=f,
705 contents=contents,
706 size=size,
707 )
708 elif version == 3:
709 return PackIndex3(path, object_format, file=f, contents=contents, size=size)
710 else:
711 raise KeyError(f"Unknown pack index format {version}")
712 else:
713 return PackIndex1(path, object_format, file=f, contents=contents, size=size)
716def bisect_find_sha(
717 start: int, end: int, sha: bytes, unpack_name: Callable[[int], bytes]
718) -> int | None:
719 """Find a SHA in a data blob with sorted SHAs.
721 Args:
722 start: Start index of range to search
723 end: End index of range to search
724 sha: Sha to find
725 unpack_name: Callback to retrieve SHA by index
726 Returns: Index of the SHA, or None if it wasn't found
727 """
728 assert start <= end
729 while start <= end:
730 i = (start + end) // 2
731 file_sha = unpack_name(i)
732 if file_sha < sha:
733 start = i + 1
734 elif file_sha > sha:
735 end = i - 1
736 else:
737 return i
738 return None
741PackIndexEntry = tuple[RawObjectID, int, int | None]
744class PackIndex:
745 """An index in to a packfile.
747 Given a sha id of an object a pack index can tell you the location in the
748 packfile of that object if it has it.
749 """
751 object_format: "ObjectFormat"
753 def __eq__(self, other: object) -> bool:
754 """Check equality with another PackIndex."""
755 if not isinstance(other, PackIndex):
756 return False
758 for (name1, _, _), (name2, _, _) in zip(
759 self.iterentries(), other.iterentries()
760 ):
761 if name1 != name2:
762 return False
763 return True
765 def __ne__(self, other: object) -> bool:
766 """Check if this pack index is not equal to another."""
767 return not self.__eq__(other)
769 def __len__(self) -> int:
770 """Return the number of entries in this pack index."""
771 raise NotImplementedError(self.__len__)
773 def __iter__(self) -> Iterator[ObjectID]:
774 """Iterate over the SHAs in this pack."""
775 return map(lambda sha: sha_to_hex(RawObjectID(sha)), self._itersha())
777 def iterentries(self) -> Iterator[PackIndexEntry]:
778 """Iterate over the entries in this pack index.
780 Returns: iterator over tuples with object name, offset in packfile and
781 crc32 checksum.
782 """
783 raise NotImplementedError(self.iterentries)
785 def get_pack_checksum(self) -> bytes | None:
786 """Return the SHA1 checksum stored for the corresponding packfile.
788 Returns: 20-byte binary digest, or None if not available
789 """
790 raise NotImplementedError(self.get_pack_checksum)
792 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
793 """Return the offset in to the corresponding packfile for the object.
795 Given the name of an object it will return the offset that object
796 lives at within the corresponding pack file. If the pack file doesn't
797 have the object then None will be returned.
798 """
799 raise NotImplementedError(self.object_offset)
801 def object_sha1(self, index: int) -> bytes:
802 """Return the SHA1 corresponding to the index in the pack file."""
803 for name, offset, _crc32 in self.iterentries():
804 if offset == index:
805 return name
806 else:
807 raise KeyError(index)
809 def _object_offset(self, sha: bytes) -> int:
810 """See object_offset.
812 Args:
813 sha: A *binary* SHA string. (20 characters long)_
814 """
815 raise NotImplementedError(self._object_offset)
817 def objects_sha1(self) -> bytes:
818 """Return the hex SHA1 over all the shas of all objects in this pack.
820 Note: This is used for the filename of the pack.
821 """
822 return iter_sha1(self._itersha())
824 def _itersha(self) -> Iterator[bytes]:
825 """Yield all the SHA1's of the objects in the index, sorted."""
826 raise NotImplementedError(self._itersha)
828 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
829 """Iterate over all SHA1s with the given prefix.
831 Args:
832 prefix: Binary prefix to match
833 Returns: Iterator of matching SHA1s
834 """
835 # Default implementation for PackIndex classes that don't override
836 for sha, _, _ in self.iterentries():
837 if sha.startswith(prefix):
838 yield RawObjectID(sha)
840 def close(self) -> None:
841 """Close any open files."""
843 def check(self) -> None:
844 """Check the consistency of this pack index."""
847class MemoryPackIndex(PackIndex):
848 """Pack index that is stored entirely in memory."""
850 def __init__(
851 self,
852 entries: list[PackIndexEntry],
853 object_format: ObjectFormat,
854 pack_checksum: bytes | None = None,
855 ) -> None:
856 """Create a new MemoryPackIndex.
858 Args:
859 entries: Sequence of name, idx, crc32 (sorted)
860 object_format: Object format used by this index
861 pack_checksum: Optional pack checksum
862 """
863 self._by_sha = {}
864 self._by_offset = {}
865 for name, offset, _crc32 in entries:
866 self._by_sha[name] = offset
867 self._by_offset[offset] = name
868 self._entries = entries
869 self._pack_checksum = pack_checksum
870 self.object_format = object_format
872 def get_pack_checksum(self) -> bytes | None:
873 """Return the SHA checksum stored for the corresponding packfile."""
874 return self._pack_checksum
876 def __len__(self) -> int:
877 """Return the number of entries in this pack index."""
878 return len(self._entries)
880 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
881 """Return the offset for the given SHA.
883 Args:
884 sha: SHA to look up (binary or hex)
885 Returns: Offset in the pack file
886 """
887 lookup_sha: RawObjectID
888 if len(sha) == self.object_format.hex_length:
889 lookup_sha = hex_to_sha(ObjectID(sha))
890 else:
891 lookup_sha = RawObjectID(sha)
892 return self._by_sha[lookup_sha]
894 def object_sha1(self, index: int) -> bytes:
895 """Return the SHA1 for the object at the given offset."""
896 return self._by_offset[index]
898 def _itersha(self) -> Iterator[bytes]:
899 """Iterate over all SHA1s in the index."""
900 return iter(self._by_sha)
902 def iterentries(self) -> Iterator[PackIndexEntry]:
903 """Iterate over all index entries."""
904 return iter(self._entries)
906 @classmethod
907 def for_pack(cls, pack_data: "PackData") -> "MemoryPackIndex":
908 """Create a MemoryPackIndex from a PackData object."""
909 return MemoryPackIndex(
910 list(pack_data.sorted_entries()),
911 pack_checksum=pack_data.get_stored_checksum(),
912 object_format=pack_data.object_format,
913 )
915 @classmethod
916 def clone(cls, other_index: "PackIndex") -> "MemoryPackIndex":
917 """Create a copy of another PackIndex in memory."""
918 return cls(
919 list(other_index.iterentries()),
920 other_index.object_format,
921 other_index.get_pack_checksum(),
922 )
925class FilePackIndex(PackIndex):
926 """Pack index that is based on a file.
928 To do the loop it opens the file, and indexes first 256 4 byte groups
929 with the first byte of the sha id. The value in the four byte group indexed
930 is the end of the group that shares the same starting byte. Subtract one
931 from the starting byte and index again to find the start of the group.
932 The values are sorted by sha id within the group, so do the math to find
933 the start and end offset and then bisect in to find if the value is
934 present.
935 """
937 _fan_out_table: list[int]
938 _file: IO[bytes] | _GitFile
940 def __init__(
941 self,
942 filename: str | os.PathLike[str],
943 file: IO[bytes] | _GitFile | None = None,
944 contents: "bytes | mmap.mmap | None" = None,
945 size: int | None = None,
946 ) -> None:
947 """Create a pack index object.
949 Provide it with the name of the index file to consider, and it will map
950 it whenever required.
951 """
952 self._filename = filename
953 # Take the size now, so it can be checked each time we map the file to
954 # ensure that it hasn't changed.
955 if file is None:
956 self._file = GitFile(filename, "rb")
957 else:
958 self._file = file
959 if contents is None:
960 self._contents, self._size = _load_file_contents(self._file, size)
961 else:
962 self._contents = contents
963 self._size = size if size is not None else len(contents)
965 @property
966 def path(self) -> str:
967 """Return the path to this index file."""
968 return os.fspath(self._filename)
970 def __eq__(self, other: object) -> bool:
971 """Check equality with another FilePackIndex."""
972 # Quick optimization:
973 if (
974 isinstance(other, FilePackIndex)
975 and self._fan_out_table != other._fan_out_table
976 ):
977 return False
979 return super().__eq__(other)
981 def close(self) -> None:
982 """Close the underlying file and any mmap."""
983 # Close the mmap before the file: on Windows the mapping holds a lock
984 # on the file, so the handle cannot be released while it is alive.
985 close_fn = getattr(self._contents, "close", None)
986 if close_fn is not None:
987 close_fn()
988 self._file.close()
990 def __del__(self) -> None:
991 """Ensure the file and mmap are closed when GCed."""
992 if not getattr(self._file, "closed", True):
993 import warnings
995 warnings.warn(
996 f"unclosed pack index {self!r}",
997 ResourceWarning,
998 stacklevel=2,
999 source=self,
1000 )
1001 try:
1002 self.close()
1003 except Exception:
1004 # Ignore errors during cleanup
1005 pass
1007 def __enter__(self) -> Self:
1008 """Enter context manager."""
1009 return self
1011 def __exit__(
1012 self,
1013 type: type | None,
1014 value: BaseException | None,
1015 traceback: TracebackType | None,
1016 ) -> None:
1017 """Exit context manager."""
1018 self.close()
1020 def __len__(self) -> int:
1021 """Return the number of entries in this pack index."""
1022 return self._fan_out_table[-1]
1024 def _unpack_entry(self, i: int) -> PackIndexEntry:
1025 """Unpack the i-th entry in the index file.
1027 Returns: Tuple with object name (SHA), offset in pack file and CRC32
1028 checksum (if known).
1029 """
1030 raise NotImplementedError(self._unpack_entry)
1032 def _unpack_name(self, i: int) -> bytes:
1033 """Unpack the i-th name from the index file."""
1034 raise NotImplementedError(self._unpack_name)
1036 def _unpack_offset(self, i: int) -> int:
1037 """Unpack the i-th object offset from the index file."""
1038 raise NotImplementedError(self._unpack_offset)
1040 def _unpack_crc32_checksum(self, i: int) -> int | None:
1041 """Unpack the crc32 checksum for the i-th object from the index file."""
1042 raise NotImplementedError(self._unpack_crc32_checksum)
1044 def _itersha(self) -> Iterator[bytes]:
1045 """Iterate over all SHA1s in the index."""
1046 for i in range(len(self)):
1047 yield self._unpack_name(i)
1049 def iterentries(self) -> Iterator[PackIndexEntry]:
1050 """Iterate over the entries in this pack index.
1052 Returns: iterator over tuples with object name, offset in packfile and
1053 crc32 checksum.
1054 """
1055 for i in range(len(self)):
1056 yield self._unpack_entry(i)
1058 def _read_fan_out_table(self, start_offset: int) -> list[int]:
1059 """Read the fan-out table from the index.
1061 The fan-out table contains 256 entries mapping first byte values
1062 to the number of objects with SHA1s less than or equal to that byte.
1064 Args:
1065 start_offset: Offset in the file where the fan-out table starts
1066 Returns: List of 256 integers
1067 """
1068 ret = []
1069 for i in range(0x100):
1070 fanout_entry = self._contents[
1071 start_offset + i * 4 : start_offset + (i + 1) * 4
1072 ]
1073 ret.append(struct.unpack(">L", fanout_entry)[0])
1074 return ret
1076 def check(self) -> None:
1077 """Check that the stored checksum matches the actual checksum."""
1078 actual = self.calculate_checksum()
1079 stored = self.get_stored_checksum()
1080 if actual != stored:
1081 raise ChecksumMismatch(stored, actual)
1083 def calculate_checksum(self) -> bytes:
1084 """Calculate the SHA1 checksum over this pack index.
1086 Returns: This is a 20-byte binary digest
1087 """
1088 return sha1(self._contents[:-20]).digest()
1090 def get_pack_checksum(self) -> bytes:
1091 """Return the SHA1 checksum stored for the corresponding packfile.
1093 Returns: 20-byte binary digest
1094 """
1095 return bytes(self._contents[-40:-20])
1097 def get_stored_checksum(self) -> bytes:
1098 """Return the SHA1 checksum stored for this index.
1100 Returns: 20-byte binary digest
1101 """
1102 return bytes(self._contents[-20:])
1104 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1105 """Return the offset in to the corresponding packfile for the object.
1107 Given the name of an object it will return the offset that object
1108 lives at within the corresponding pack file. If the pack file doesn't
1109 have the object then None will be returned.
1110 """
1111 lookup_sha: RawObjectID
1112 if len(sha) == self.object_format.hex_length: # hex string
1113 lookup_sha = hex_to_sha(ObjectID(sha))
1114 else:
1115 lookup_sha = RawObjectID(sha)
1116 try:
1117 return self._object_offset(lookup_sha)
1118 except ValueError as exc:
1119 closed = getattr(self._contents, "closed", None)
1120 if closed in (None, True):
1121 raise PackFileDisappeared(self) from exc
1122 raise
1124 def _object_offset(self, sha: bytes) -> int:
1125 """See object_offset.
1127 Args:
1128 sha: A *binary* SHA string. (20 characters long)_
1129 """
1130 hash_size = getattr(self, "hash_size", 20) # Default to SHA1 for v1
1131 assert len(sha) == hash_size
1132 idx = ord(sha[:1])
1133 if idx == 0:
1134 start = 0
1135 else:
1136 start = self._fan_out_table[idx - 1]
1137 end = self._fan_out_table[idx]
1138 i = bisect_find_sha(start, end, sha, self._unpack_name)
1139 if i is None:
1140 raise KeyError(sha)
1141 return self._unpack_offset(i)
1143 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
1144 """Iterate over all SHA1s with the given prefix."""
1145 start = ord(prefix[:1])
1146 if start == 0:
1147 start = 0
1148 else:
1149 start = self._fan_out_table[start - 1]
1150 end = ord(prefix[:1]) + 1
1151 if end == 0x100:
1152 end = len(self)
1153 else:
1154 end = self._fan_out_table[end]
1155 assert start <= end
1156 started = False
1157 for i in range(start, end):
1158 name: bytes = self._unpack_name(i)
1159 if name.startswith(prefix):
1160 yield RawObjectID(name)
1161 started = True
1162 elif started:
1163 break
1166class PackIndex1(FilePackIndex):
1167 """Version 1 Pack Index file."""
1169 object_format = SHA1
1171 def __init__(
1172 self,
1173 filename: str | os.PathLike[str],
1174 object_format: ObjectFormat,
1175 file: IO[bytes] | _GitFile | None = None,
1176 contents: bytes | None = None,
1177 size: int | None = None,
1178 ) -> None:
1179 """Initialize a version 1 pack index.
1181 Args:
1182 filename: Path to the index file
1183 object_format: Object format used by the repository
1184 file: Optional file object
1185 contents: Optional mmap'd contents
1186 size: Optional size of the index
1187 """
1188 super().__init__(filename, file, contents, size)
1190 # PackIndex1 only supports SHA1
1191 if object_format != SHA1:
1192 raise AssertionError(
1193 f"PackIndex1 only supports SHA1, not {object_format.name}"
1194 )
1196 self.object_format = object_format
1197 self.version = 1
1198 self._fan_out_table = self._read_fan_out_table(0)
1199 self.hash_size = self.object_format.oid_length
1200 self._entry_size = 4 + self.hash_size
1202 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, None]:
1203 base_offset = (0x100 * 4) + (i * self._entry_size)
1204 offset = unpack_from(">L", self._contents, base_offset)[0]
1205 name = self._contents[base_offset + 4 : base_offset + 4 + self.hash_size]
1206 return (RawObjectID(name), offset, None)
1208 def _unpack_name(self, i: int) -> bytes:
1209 offset = (0x100 * 4) + (i * self._entry_size) + 4
1210 return self._contents[offset : offset + self.hash_size]
1212 def _unpack_offset(self, i: int) -> int:
1213 offset = (0x100 * 4) + (i * self._entry_size)
1214 return int(unpack_from(">L", self._contents, offset)[0])
1216 def _unpack_crc32_checksum(self, i: int) -> None:
1217 # Not stored in v1 index files
1218 return None
1221class PackIndex2(FilePackIndex):
1222 """Version 2 Pack Index file."""
1224 object_format = SHA1
1226 def __init__(
1227 self,
1228 filename: str | os.PathLike[str],
1229 object_format: ObjectFormat,
1230 file: IO[bytes] | _GitFile | None = None,
1231 contents: bytes | None = None,
1232 size: int | None = None,
1233 ) -> None:
1234 """Initialize a version 2 pack index.
1236 Args:
1237 filename: Path to the index file
1238 object_format: Object format used by the repository
1239 file: Optional file object
1240 contents: Optional mmap'd contents
1241 size: Optional size of the index
1242 """
1243 super().__init__(filename, file, contents, size)
1244 self.object_format = object_format
1245 if self._contents[:4] != b"\377tOc":
1246 raise AssertionError("Not a v2 pack index file")
1247 (self.version,) = unpack_from(b">L", self._contents, 4)
1248 if self.version != 2:
1249 raise AssertionError(f"Version was {self.version}")
1250 self._fan_out_table = self._read_fan_out_table(8)
1251 self.hash_size = self.object_format.oid_length
1252 self._name_table_offset = 8 + 0x100 * 4
1253 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1254 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1255 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1256 self
1257 )
1259 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1260 return (
1261 RawObjectID(self._unpack_name(i)),
1262 self._unpack_offset(i),
1263 self._unpack_crc32_checksum(i),
1264 )
1266 def _unpack_name(self, i: int) -> bytes:
1267 offset = self._name_table_offset + i * self.hash_size
1268 return self._contents[offset : offset + self.hash_size]
1270 def _unpack_offset(self, i: int) -> int:
1271 offset = self._pack_offset_table_offset + i * 4
1272 offset_val = int(unpack_from(">L", self._contents, offset)[0])
1273 if offset_val & (2**31):
1274 offset = (
1275 self._pack_offset_largetable_offset + (offset_val & (2**31 - 1)) * 8
1276 )
1277 offset_val = int(unpack_from(">Q", self._contents, offset)[0])
1278 return offset_val
1280 def _unpack_crc32_checksum(self, i: int) -> int:
1281 return int(
1282 unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1283 )
1285 def get_pack_checksum(self) -> bytes:
1286 """Return the checksum stored for the corresponding packfile.
1288 Returns: binary digest (size depends on hash algorithm)
1289 """
1290 # Index ends with: pack_checksum + index_checksum
1291 # Each checksum is hash_size bytes
1292 checksum_size = self.hash_size
1293 return bytes(self._contents[-2 * checksum_size : -checksum_size])
1295 def get_stored_checksum(self) -> bytes:
1296 """Return the checksum stored for this index.
1298 Returns: binary digest (size depends on hash algorithm)
1299 """
1300 checksum_size = self.hash_size
1301 return bytes(self._contents[-checksum_size:])
1303 def calculate_checksum(self) -> bytes:
1304 """Calculate the checksum over this pack index.
1306 Returns: binary digest (size depends on hash algorithm)
1307 """
1308 # Determine hash function based on hash_size
1309 if self.hash_size == 20:
1310 hash_func = sha1
1311 elif self.hash_size == 32:
1312 hash_func = sha256
1313 else:
1314 raise ValueError(f"Unsupported hash size: {self.hash_size}")
1316 return hash_func(self._contents[: -self.hash_size]).digest()
1319class PackIndex3(FilePackIndex):
1320 """Version 3 Pack Index file.
1322 Supports variable hash sizes for SHA-1 (20 bytes) and SHA-256 (32 bytes).
1323 """
1325 def __init__(
1326 self,
1327 filename: str | os.PathLike[str],
1328 object_format: ObjectFormat,
1329 file: IO[bytes] | _GitFile | None = None,
1330 contents: bytes | None = None,
1331 size: int | None = None,
1332 ) -> None:
1333 """Initialize a version 3 pack index.
1335 Args:
1336 filename: Path to the index file
1337 object_format: Object format used by the repository
1338 file: Optional file object
1339 contents: Optional mmap'd contents
1340 size: Optional size of the index
1341 """
1342 super().__init__(filename, file, contents, size)
1343 if self._contents[:4] != b"\377tOc":
1344 raise AssertionError("Not a v3 pack index file")
1345 (self.version,) = unpack_from(b">L", self._contents, 4)
1346 if self.version != 3:
1347 raise AssertionError(f"Version was {self.version}")
1349 # Read hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
1350 (self.hash_format,) = unpack_from(b">L", self._contents, 8)
1351 file_object_format = OBJECT_FORMAT_TYPE_NUMS[self.hash_format]
1353 # Verify provided object_format matches what's in the file
1354 if object_format != file_object_format:
1355 raise AssertionError(
1356 f"Object format mismatch: provided {object_format.name}, "
1357 f"but file contains {file_object_format.name}"
1358 )
1360 self.object_format = object_format
1361 self.hash_size = self.object_format.oid_length
1363 # Read length of shortened object names
1364 (self.shortened_oid_len,) = unpack_from(b">L", self._contents, 12)
1366 # Calculate offsets based on variable hash size
1367 self._fan_out_table = self._read_fan_out_table(
1368 16
1369 ) # After header (4 + 4 + 4 + 4)
1370 self._name_table_offset = 16 + 0x100 * 4
1371 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1372 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1373 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1374 self
1375 )
1377 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1378 return (
1379 RawObjectID(self._unpack_name(i)),
1380 self._unpack_offset(i),
1381 self._unpack_crc32_checksum(i),
1382 )
1384 def _unpack_name(self, i: int) -> bytes:
1385 offset = self._name_table_offset + i * self.hash_size
1386 return self._contents[offset : offset + self.hash_size]
1388 def _unpack_offset(self, i: int) -> int:
1389 offset_pos = self._pack_offset_table_offset + i * 4
1390 offset = unpack_from(">L", self._contents, offset_pos)[0]
1391 assert isinstance(offset, int)
1392 if offset & (2**31):
1393 large_offset_pos = (
1394 self._pack_offset_largetable_offset + (offset & (2**31 - 1)) * 8
1395 )
1396 offset = unpack_from(">Q", self._contents, large_offset_pos)[0]
1397 assert isinstance(offset, int)
1398 return offset
1400 def _unpack_crc32_checksum(self, i: int) -> int:
1401 result = unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1402 assert isinstance(result, int)
1403 return result
1406def read_pack_header(read: Callable[[int], bytes]) -> tuple[int, int]:
1407 """Read the header of a pack file.
1409 Args:
1410 read: Read function
1411 Returns: Tuple of (pack version, number of objects). If no data is
1412 available to read, returns (None, None).
1413 """
1414 header = read(12)
1415 if not header:
1416 raise AssertionError("file too short to contain pack")
1417 if header[:4] != b"PACK":
1418 raise AssertionError(f"Invalid pack header {header!r}")
1419 (version,) = unpack_from(b">L", header, 4)
1420 if version not in (2, 3):
1421 raise AssertionError(f"Version was {version}")
1422 (num_objects,) = unpack_from(b">L", header, 8)
1423 return (version, num_objects)
1426def chunks_length(chunks: bytes | Iterable[bytes]) -> int:
1427 """Get the total length of a sequence of chunks.
1429 Args:
1430 chunks: Either a single bytes object or an iterable of bytes
1431 Returns: Total length in bytes
1432 """
1433 if isinstance(chunks, bytes):
1434 return len(chunks)
1435 else:
1436 return sum(map(len, chunks))
1439def unpack_object(
1440 read_all: Callable[[int], bytes],
1441 hash_func: Callable[[], "HashObject"],
1442 read_some: Callable[[int], bytes] | None = None,
1443 compute_crc32: bool = False,
1444 include_comp: bool = False,
1445 zlib_bufsize: int = _ZLIB_BUFSIZE,
1446) -> tuple[UnpackedObject, bytes]:
1447 """Unpack a Git object.
1449 Args:
1450 read_all: Read function that blocks until the number of requested
1451 bytes are read.
1452 hash_func: Hash function to use for computing object IDs.
1453 read_some: Read function that returns at least one byte, but may not
1454 return the number of bytes requested.
1455 compute_crc32: If True, compute the CRC32 of the compressed data. If
1456 False, the returned CRC32 will be None.
1457 include_comp: If True, include compressed data in the result.
1458 zlib_bufsize: An optional buffer size for zlib operations.
1459 Returns: A tuple of (unpacked, unused), where unused is the unused data
1460 leftover from decompression, and unpacked in an UnpackedObject with
1461 the following attrs set:
1463 * obj_chunks (for non-delta types)
1464 * pack_type_num
1465 * delta_base (for delta types)
1466 * comp_chunks (if include_comp is True)
1467 * decomp_chunks
1468 * decomp_len
1469 * crc32 (if compute_crc32 is True)
1470 """
1471 if read_some is None:
1472 read_some = read_all
1473 if compute_crc32:
1474 crc32 = 0
1475 else:
1476 crc32 = None
1478 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1479 type_num = (raw[0] >> 4) & 0x07
1480 size = raw[0] & 0x0F
1481 for i, byte in enumerate(raw[1:]):
1482 size += (byte & 0x7F) << ((i * 7) + 4)
1484 delta_base: int | bytes | None
1485 raw_base = len(raw)
1486 if type_num == OFS_DELTA:
1487 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1488 raw_base += len(raw)
1489 if raw[-1] & 0x80:
1490 raise AssertionError
1491 delta_base_offset = raw[0] & 0x7F
1492 for byte in raw[1:]:
1493 delta_base_offset += 1
1494 delta_base_offset <<= 7
1495 delta_base_offset += byte & 0x7F
1496 if delta_base_offset == 0:
1497 # A zero offset makes the delta reference itself, which would
1498 # loop forever in resolve_object. git's C client rejects this
1499 # with "delta offset == 0 is invalid".
1500 raise ApplyDeltaError("OFS_DELTA has delta_base_offset of 0")
1501 delta_base = delta_base_offset
1502 elif type_num == REF_DELTA:
1503 # Determine hash size from hash_func
1504 hash_size = len(hash_func().digest())
1505 delta_base_obj = read_all(hash_size)
1506 if crc32 is not None:
1507 crc32 = binascii.crc32(delta_base_obj, crc32)
1508 delta_base = delta_base_obj
1509 raw_base += hash_size
1510 else:
1511 delta_base = None
1513 unpacked = UnpackedObject(
1514 type_num,
1515 delta_base=delta_base,
1516 decomp_len=size,
1517 crc32=crc32,
1518 hash_func=hash_func,
1519 )
1520 unused = read_zlib_chunks(
1521 read_some,
1522 unpacked,
1523 buffer_size=zlib_bufsize,
1524 include_comp=include_comp,
1525 )
1526 return unpacked, unused
1529def _compute_object_size(value: tuple[int, Any]) -> int:
1530 """Compute the size of a unresolved object for use with LRUSizeCache."""
1531 (num, obj) = value
1532 if num in DELTA_TYPES:
1533 return chunks_length(obj[1])
1534 return chunks_length(obj)
1537class PackStreamReader:
1538 """Class to read a pack stream.
1540 The pack is read from a ReceivableProtocol using read() or recv() as
1541 appropriate.
1542 """
1544 def __init__(
1545 self,
1546 hash_func: Callable[[], "HashObject"],
1547 read_all: Callable[[int], bytes],
1548 read_some: Callable[[int], bytes] | None = None,
1549 zlib_bufsize: int = _ZLIB_BUFSIZE,
1550 ) -> None:
1551 """Initialize pack stream reader.
1553 Args:
1554 hash_func: Hash function to use for computing object IDs
1555 read_all: Function to read all requested bytes
1556 read_some: Function to read some bytes (optional)
1557 zlib_bufsize: Buffer size for zlib decompression
1558 """
1559 self.read_all = read_all
1560 if read_some is None:
1561 self.read_some = read_all
1562 else:
1563 self.read_some = read_some
1564 self.hash_func = hash_func
1565 self.sha = hash_func()
1566 self._hash_size = len(hash_func().digest())
1567 self._offset = 0
1568 self._rbuf = BytesIO()
1569 # trailer is a deque to avoid memory allocation on small reads
1570 self._trailer: deque[int] = deque()
1571 self._zlib_bufsize = zlib_bufsize
1573 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1574 """Read up to size bytes using the given callback.
1576 As a side effect, update the verifier's hash (excluding the last
1577 hash_size bytes read, which is the pack checksum).
1579 Args:
1580 read: The read callback to read from.
1581 size: The maximum number of bytes to read; the particular
1582 behavior is callback-specific.
1583 Returns: Bytes read
1584 """
1585 data = read(size)
1587 # maintain a trailer of the last hash_size bytes we've read
1588 n = len(data)
1589 self._offset += n
1590 tn = len(self._trailer)
1591 if n >= self._hash_size:
1592 to_pop = tn
1593 to_add = self._hash_size
1594 else:
1595 to_pop = max(n + tn - self._hash_size, 0)
1596 to_add = n
1597 self.sha.update(
1598 bytes(bytearray([self._trailer.popleft() for _ in range(to_pop)]))
1599 )
1600 self._trailer.extend(data[-to_add:])
1602 # hash everything but the trailer
1603 self.sha.update(data[:-to_add])
1604 return data
1606 def _buf_len(self) -> int:
1607 buf = self._rbuf
1608 start = buf.tell()
1609 buf.seek(0, SEEK_END)
1610 end = buf.tell()
1611 buf.seek(start)
1612 return end - start
1614 @property
1615 def offset(self) -> int:
1616 """Return current offset in the stream."""
1617 return self._offset - self._buf_len()
1619 def read(self, size: int) -> bytes:
1620 """Read, blocking until size bytes are read."""
1621 buf_len = self._buf_len()
1622 if buf_len >= size:
1623 return self._rbuf.read(size)
1624 buf_data = self._rbuf.read()
1625 self._rbuf = BytesIO()
1626 return buf_data + self._read(self.read_all, size - buf_len)
1628 def recv(self, size: int) -> bytes:
1629 """Read up to size bytes, blocking until one byte is read."""
1630 buf_len = self._buf_len()
1631 if buf_len:
1632 data = self._rbuf.read(size)
1633 if size >= buf_len:
1634 self._rbuf = BytesIO()
1635 return data
1636 return self._read(self.read_some, size)
1638 def __len__(self) -> int:
1639 """Return the number of objects in this pack."""
1640 return self._num_objects
1642 def read_objects(self, compute_crc32: bool = False) -> Iterator[UnpackedObject]:
1643 """Read the objects in this pack file.
1645 Args:
1646 compute_crc32: If True, compute the CRC32 of the compressed
1647 data. If False, the returned CRC32 will be None.
1648 Returns: Iterator over UnpackedObjects with the following members set:
1649 offset
1650 obj_type_num
1651 obj_chunks (for non-delta types)
1652 delta_base (for delta types)
1653 decomp_chunks
1654 decomp_len
1655 crc32 (if compute_crc32 is True)
1657 Raises:
1658 ChecksumMismatch: if the checksum of the pack contents does not
1659 match the checksum in the pack trailer.
1660 zlib.error: if an error occurred during zlib decompression.
1661 IOError: if an error occurred writing to the output file.
1662 """
1663 _pack_version, self._num_objects = read_pack_header(self.read)
1665 for _ in range(self._num_objects):
1666 offset = self.offset
1667 unpacked, unused = unpack_object(
1668 self.read,
1669 self.hash_func,
1670 read_some=self.recv,
1671 compute_crc32=compute_crc32,
1672 zlib_bufsize=self._zlib_bufsize,
1673 )
1674 unpacked.offset = offset
1676 # prepend any unused data to current read buffer
1677 buf = BytesIO()
1678 buf.write(unused)
1679 buf.write(self._rbuf.read())
1680 buf.seek(0)
1681 self._rbuf = buf
1683 yield unpacked
1685 if self._buf_len() < self._hash_size:
1686 # If the read buffer is full, then the last read() got the whole
1687 # trailer off the wire. If not, it means there is still some of the
1688 # trailer to read. We need to read() all hash_size bytes; N come from the
1689 # read buffer and (hash_size - N) come from the wire.
1690 self.read(self._hash_size)
1692 pack_sha = bytearray(self._trailer)
1693 if pack_sha != self.sha.digest():
1694 raise ChecksumMismatch(
1695 sha_to_hex(RawObjectID(bytes(pack_sha))), self.sha.hexdigest()
1696 )
1699class PackStreamCopier(PackStreamReader):
1700 """Class to verify a pack stream as it is being read.
1702 The pack is read from a ReceivableProtocol using read() or recv() as
1703 appropriate and written out to the given file-like object.
1704 """
1706 def __init__(
1707 self,
1708 hash_func: Callable[[], "HashObject"],
1709 read_all: Callable[[int], bytes],
1710 read_some: Callable[[int], bytes] | None,
1711 outfile: IO[bytes],
1712 delta_iter: "DeltaChainIterator[UnpackedObject] | None" = None,
1713 ) -> None:
1714 """Initialize the copier.
1716 Args:
1717 hash_func: Hash function to use for computing object IDs
1718 read_all: Read function that blocks until the number of
1719 requested bytes are read.
1720 read_some: Read function that returns at least one byte, but may
1721 not return the number of bytes requested.
1722 outfile: File-like object to write output through.
1723 delta_iter: Optional DeltaChainIterator to record deltas as we
1724 read them.
1725 """
1726 super().__init__(hash_func, read_all, read_some=read_some)
1727 self.outfile = outfile
1728 self._delta_iter = delta_iter
1730 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1731 """Read data from the read callback and write it to the file."""
1732 data = super()._read(read, size)
1733 self.outfile.write(data)
1734 return data
1736 def verify(self, progress: Callable[..., None] | None = None) -> None:
1737 """Verify a pack stream and write it to the output file.
1739 See PackStreamReader.iterobjects for a list of exceptions this may
1740 throw.
1741 """
1742 i = 0 # default count of entries if read_objects() is empty
1743 for i, unpacked in enumerate(self.read_objects()):
1744 if self._delta_iter:
1745 self._delta_iter.record(unpacked)
1746 if progress is not None:
1747 progress(f"copying pack entries: {i}/{len(self)}\r".encode("ascii"))
1748 if progress is not None:
1749 progress(f"copied {i} pack entries\n".encode("ascii"))
1752def obj_sha(
1753 type: int,
1754 chunks: bytes | Iterable[bytes],
1755 hash_func: Callable[[], "HashObject"] = sha1,
1756) -> bytes:
1757 """Compute the SHA for a numeric type and object chunks.
1759 Args:
1760 type: Object type number
1761 chunks: Object data chunks
1762 hash_func: Hash function to use (defaults to sha1)
1764 Returns:
1765 Binary hash digest
1766 """
1767 sha = hash_func()
1768 sha.update(object_header(type, chunks_length(chunks)))
1769 if isinstance(chunks, bytes):
1770 sha.update(chunks)
1771 else:
1772 for chunk in chunks:
1773 sha.update(chunk)
1774 return sha.digest()
1777def compute_file_sha(
1778 f: IO[bytes],
1779 hash_func: Callable[[], "HashObject"],
1780 start_ofs: int = 0,
1781 end_ofs: int = 0,
1782 buffer_size: int = 1 << 16,
1783) -> "HashObject":
1784 """Hash a portion of a file into a new SHA.
1786 Args:
1787 f: A file-like object to read from that supports seek().
1788 hash_func: A callable that returns a new HashObject.
1789 start_ofs: The offset in the file to start reading at.
1790 end_ofs: The offset in the file to end reading at, relative to the
1791 end of the file.
1792 buffer_size: A buffer size for reading.
1793 Returns: A new SHA object updated with data read from the file.
1794 """
1795 sha = hash_func()
1796 f.seek(0, SEEK_END)
1797 length = f.tell()
1798 if start_ofs < 0:
1799 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
1800 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
1801 raise AssertionError(
1802 f"Attempt to read beyond file length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, file length: {length}"
1803 )
1804 todo = length + end_ofs - start_ofs
1805 f.seek(start_ofs)
1806 while todo:
1807 data = f.read(min(todo, buffer_size))
1808 sha.update(data)
1809 todo -= len(data)
1810 return sha
1813class PackData:
1814 """The data contained in a packfile.
1816 Pack files can be accessed both sequentially for exploding a pack, and
1817 directly with the help of an index to retrieve a specific object.
1819 The objects within are either complete or a delta against another.
1821 The header is variable length. If the MSB of each byte is set then it
1822 indicates that the subsequent byte is still part of the header.
1823 For the first byte the next MS bits are the type, which tells you the type
1824 of object, and whether it is a delta. The LS byte is the lowest bits of the
1825 size. For each subsequent byte the LS 7 bits are the next MS bits of the
1826 size, i.e. the last byte of the header contains the MS bits of the size.
1828 For the complete objects the data is stored as zlib deflated data.
1829 The size in the header is the uncompressed object size, so to uncompress
1830 you need to just keep feeding data to zlib until you get an object back,
1831 or it errors on bad data. This is done here by just giving the complete
1832 buffer from the start of the deflated object on. This is bad, but until I
1833 get mmap sorted out it will have to do.
1835 Currently there are no integrity checks done. Also no attempt is made to
1836 try and detect the delta case, or a request for an object at the wrong
1837 position. It will all just throw a zlib or KeyError.
1838 """
1840 def __init__(
1841 self,
1842 filename: str | os.PathLike[str],
1843 object_format: ObjectFormat,
1844 file: IO[bytes] | None = None,
1845 size: int | None = None,
1846 *,
1847 delta_window_size: int | None = None,
1848 window_memory: int | None = None,
1849 delta_cache_size: int | None = None,
1850 depth: int | None = None,
1851 threads: int | None = None,
1852 big_file_threshold: int | None = None,
1853 delta_base_cache_limit: int | None = None,
1854 ) -> None:
1855 """Create a PackData object representing the pack in the given filename.
1857 The file must exist and stay readable until the object is disposed of.
1858 It must also stay the same size. It will be mapped whenever needed.
1860 Currently there is a restriction on the size of the pack as the python
1861 mmap implementation is flawed.
1862 """
1863 self._filename = filename
1864 self.object_format = object_format
1865 self._size = size
1866 self._header_size = 12
1867 self.delta_window_size = delta_window_size
1868 self.window_memory = window_memory
1869 self.delta_cache_size = delta_cache_size
1870 self.depth = depth
1871 self.threads = threads
1872 self.big_file_threshold = big_file_threshold
1873 self.delta_base_cache_limit = delta_base_cache_limit
1874 self._file: IO[bytes]
1876 if file is None:
1877 self._file = GitFile(self._filename, "rb")
1878 else:
1879 self._file = file
1880 (_version, self._num_objects) = read_pack_header(self._file.read)
1882 # Use delta_base_cache_limit, then delta_cache_size, then default
1883 cache_size = (
1884 delta_base_cache_limit or delta_cache_size or DEFAULT_DELTA_BASE_CACHE_LIMIT
1885 )
1886 self._offset_cache = LRUSizeCache[int, tuple[int, OldUnpackedObject]](
1887 cache_size, compute_size=_compute_object_size
1888 )
1890 @property
1891 def filename(self) -> str:
1892 """Get the filename of the pack file.
1894 Returns:
1895 Base filename without directory path
1896 """
1897 return os.path.basename(self._filename)
1899 @property
1900 def path(self) -> str | os.PathLike[str]:
1901 """Get the full path of the pack file.
1903 Returns:
1904 Full path to the pack file
1905 """
1906 return self._filename
1908 @classmethod
1909 def from_file(
1910 cls,
1911 file: IO[bytes],
1912 object_format: ObjectFormat,
1913 size: int | None = None,
1914 ) -> "PackData":
1915 """Create a PackData object from an open file.
1917 Args:
1918 file: Open file object
1919 object_format: Object format
1920 size: Optional file size
1922 Returns:
1923 PackData instance
1924 """
1925 return cls(str(file), object_format, file=file, size=size)
1927 @classmethod
1928 def from_path(
1929 cls,
1930 path: str | os.PathLike[str],
1931 object_format: ObjectFormat,
1932 ) -> "PackData":
1933 """Create a PackData object from a file path.
1935 Args:
1936 path: Path to the pack file
1937 object_format: Object format
1939 Returns:
1940 PackData instance
1941 """
1942 return cls(filename=path, object_format=object_format)
1944 def close(self) -> None:
1945 """Close the underlying pack file."""
1946 if self._file is not None:
1947 self._file.close()
1948 self._file = None # type: ignore
1950 def __del__(self) -> None:
1951 """Ensure pack file is closed when PackData is garbage collected."""
1952 if getattr(self, "_file", None) is not None:
1953 import warnings
1955 warnings.warn(
1956 f"unclosed PackData {self!r}",
1957 ResourceWarning,
1958 stacklevel=2,
1959 source=self,
1960 )
1961 try:
1962 self.close()
1963 except Exception:
1964 # Ignore errors during cleanup
1965 pass
1967 def __enter__(self) -> Self:
1968 """Enter context manager."""
1969 return self
1971 def __exit__(
1972 self,
1973 type: type | None,
1974 value: BaseException | None,
1975 traceback: TracebackType | None,
1976 ) -> None:
1977 """Exit context manager."""
1978 self.close()
1980 def __eq__(self, other: object) -> bool:
1981 """Check equality with another object."""
1982 if isinstance(other, PackData):
1983 return self.get_stored_checksum() == other.get_stored_checksum()
1984 return False
1986 def _get_size(self) -> int:
1987 if self._size is not None:
1988 return self._size
1989 self._size = os.path.getsize(self._filename)
1990 if self._size < self._header_size:
1991 errmsg = f"{self._filename} is too small for a packfile ({self._size} < {self._header_size})"
1992 raise AssertionError(errmsg)
1993 return self._size
1995 def __len__(self) -> int:
1996 """Returns the number of objects in this pack."""
1997 return self._num_objects
1999 def calculate_checksum(self) -> bytes:
2000 """Calculate the checksum for this pack.
2002 Returns: Binary digest (size depends on hash algorithm)
2003 """
2004 return compute_file_sha(
2005 self._file,
2006 hash_func=self.object_format.hash_func,
2007 end_ofs=-self.object_format.oid_length,
2008 ).digest()
2010 def iter_unpacked(self, *, include_comp: bool = False) -> Iterator[UnpackedObject]:
2011 """Iterate over unpacked objects in the pack."""
2012 self._file.seek(self._header_size)
2014 if self._num_objects is None:
2015 return
2017 for _ in range(self._num_objects):
2018 offset = self._file.tell()
2019 unpacked, unused = unpack_object(
2020 self._file.read,
2021 self.object_format.hash_func,
2022 compute_crc32=False,
2023 include_comp=include_comp,
2024 )
2025 unpacked.offset = offset
2026 yield unpacked
2027 # Back up over unused data.
2028 self._file.seek(-len(unused), SEEK_CUR)
2030 def iterentries(
2031 self,
2032 progress: Callable[[int, int], None] | None = None,
2033 resolve_ext_ref: ResolveExtRefFn | None = None,
2034 ) -> Iterator[PackIndexEntry]:
2035 """Yield entries summarizing the contents of this pack.
2037 Args:
2038 progress: Progress function, called with current and total
2039 object count.
2040 resolve_ext_ref: Optional function to resolve external references
2041 Returns: iterator of tuples with (sha, offset, crc32)
2042 """
2043 num_objects = self._num_objects
2044 indexer = PackIndexer.for_pack_data(self, resolve_ext_ref=resolve_ext_ref)
2045 for i, result in enumerate(indexer):
2046 if progress is not None:
2047 progress(i, num_objects)
2048 yield result
2050 def sorted_entries(
2051 self,
2052 progress: Callable[[int, int], None] | None = None,
2053 resolve_ext_ref: ResolveExtRefFn | None = None,
2054 ) -> list[tuple[RawObjectID, int, int]]:
2055 """Return entries in this pack, sorted by SHA.
2057 Args:
2058 progress: Progress function, called with current and total
2059 object count
2060 resolve_ext_ref: Optional function to resolve external references
2061 Returns: Iterator of tuples with (sha, offset, crc32)
2062 """
2063 return sorted(
2064 self.iterentries(progress=progress, resolve_ext_ref=resolve_ext_ref) # type: ignore
2065 )
2067 def create_index_v1(
2068 self,
2069 filename: str,
2070 progress: Callable[..., None] | None = None,
2071 resolve_ext_ref: ResolveExtRefFn | None = None,
2072 ) -> bytes:
2073 """Create a version 1 file for this data file.
2075 Args:
2076 filename: Index filename.
2077 progress: Progress report function
2078 resolve_ext_ref: Optional function to resolve external references
2079 Returns: Checksum of index file
2080 """
2081 entries = self.sorted_entries(
2082 progress=progress, resolve_ext_ref=resolve_ext_ref
2083 )
2084 checksum = self.calculate_checksum()
2085 with GitFile(filename, "wb") as f:
2086 write_pack_index_v1(
2087 f,
2088 entries,
2089 checksum,
2090 )
2091 return checksum
2093 def create_index_v2(
2094 self,
2095 filename: str,
2096 progress: Callable[..., None] | None = None,
2097 resolve_ext_ref: ResolveExtRefFn | None = None,
2098 ) -> bytes:
2099 """Create a version 2 index file for this data file.
2101 Args:
2102 filename: Index filename.
2103 progress: Progress report function
2104 resolve_ext_ref: Optional function to resolve external references
2105 Returns: Checksum of index file
2106 """
2107 entries = self.sorted_entries(
2108 progress=progress, resolve_ext_ref=resolve_ext_ref
2109 )
2110 with GitFile(filename, "wb") as f:
2111 return write_pack_index_v2(f, entries, self.calculate_checksum())
2113 def create_index_v3(
2114 self,
2115 filename: str,
2116 progress: Callable[..., None] | None = None,
2117 resolve_ext_ref: ResolveExtRefFn | None = None,
2118 hash_format: int | None = None,
2119 ) -> bytes:
2120 """Create a version 3 index file for this data file.
2122 Args:
2123 filename: Index filename.
2124 progress: Progress report function
2125 resolve_ext_ref: Function to resolve external references
2126 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
2127 Returns: Checksum of index file
2128 """
2129 entries = self.sorted_entries(
2130 progress=progress, resolve_ext_ref=resolve_ext_ref
2131 )
2132 with GitFile(filename, "wb") as f:
2133 if hash_format is None:
2134 hash_format = 1 # Default to SHA-1
2135 return write_pack_index_v3(
2136 f, entries, self.calculate_checksum(), hash_format=hash_format
2137 )
2139 def create_index(
2140 self,
2141 filename: str,
2142 progress: Callable[..., None] | None = None,
2143 version: int = 2,
2144 resolve_ext_ref: ResolveExtRefFn | None = None,
2145 hash_format: int | None = None,
2146 ) -> bytes:
2147 """Create an index file for this data file.
2149 Args:
2150 filename: Index filename.
2151 progress: Progress report function
2152 version: Index version (1, 2, or 3)
2153 resolve_ext_ref: Function to resolve external references
2154 hash_format: Hash algorithm identifier for v3 (1 = SHA-1, 2 = SHA-256)
2155 Returns: Checksum of index file
2156 """
2157 if version == 1:
2158 return self.create_index_v1(
2159 filename, progress, resolve_ext_ref=resolve_ext_ref
2160 )
2161 elif version == 2:
2162 return self.create_index_v2(
2163 filename, progress, resolve_ext_ref=resolve_ext_ref
2164 )
2165 elif version == 3:
2166 return self.create_index_v3(
2167 filename,
2168 progress,
2169 resolve_ext_ref=resolve_ext_ref,
2170 hash_format=hash_format,
2171 )
2172 else:
2173 raise ValueError(f"unknown index format {version}")
2175 def get_stored_checksum(self) -> bytes:
2176 """Return the expected checksum stored in this pack."""
2177 checksum_size = self.object_format.oid_length
2178 self._file.seek(-checksum_size, SEEK_END)
2179 return self._file.read(checksum_size)
2181 def check(self) -> None:
2182 """Check the consistency of this pack."""
2183 actual = self.calculate_checksum()
2184 stored = self.get_stored_checksum()
2185 if actual != stored:
2186 raise ChecksumMismatch(stored, actual)
2188 def get_unpacked_object_at(
2189 self, offset: int, *, include_comp: bool = False
2190 ) -> UnpackedObject:
2191 """Given offset in the packfile return a UnpackedObject."""
2192 assert offset >= self._header_size
2193 self._file.seek(offset)
2194 unpacked, _ = unpack_object(
2195 self._file.read, self.object_format.hash_func, include_comp=include_comp
2196 )
2197 unpacked.offset = offset
2198 return unpacked
2200 def get_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2201 """Given an offset in to the packfile return the object that is there.
2203 Using the associated index the location of an object can be looked up,
2204 and then the packfile can be asked directly for that object using this
2205 function.
2206 """
2207 try:
2208 return self._offset_cache[offset]
2209 except KeyError:
2210 pass
2211 unpacked = self.get_unpacked_object_at(offset, include_comp=False)
2212 return (unpacked.pack_type_num, unpacked._obj())
2215T = TypeVar("T")
2218class DeltaChainIterator(Generic[T]):
2219 """Abstract iterator over pack data based on delta chains.
2221 Each object in the pack is guaranteed to be inflated exactly once,
2222 regardless of how many objects reference it as a delta base. As a result,
2223 memory usage is proportional to the length of the longest delta chain.
2225 Subclasses can override _result to define the result type of the iterator.
2226 By default, results are UnpackedObjects with the following members set:
2228 * offset
2229 * obj_type_num
2230 * obj_chunks
2231 * pack_type_num
2232 * delta_base (for delta types)
2233 * comp_chunks (if _include_comp is True)
2234 * decomp_chunks
2235 * decomp_len
2236 * crc32 (if _compute_crc32 is True)
2237 """
2239 _compute_crc32 = False
2240 _include_comp = False
2242 def __init__(
2243 self,
2244 file_obj: IO[bytes] | None,
2245 hash_func: Callable[[], "HashObject"],
2246 *,
2247 resolve_ext_ref: ResolveExtRefFn | None = None,
2248 object_format: "ObjectFormat | None" = None,
2249 ) -> None:
2250 """Initialize DeltaChainIterator.
2252 Args:
2253 file_obj: File object to read pack data from
2254 hash_func: Hash function to use for computing object IDs
2255 resolve_ext_ref: Optional function to resolve external references
2256 object_format: Optional object format. Required by subclasses
2257 that materialise objects (e.g. PackInflater) when iterating
2258 packs in a non-default hash algorithm such as SHA-256.
2259 """
2260 self._file = file_obj
2261 self.hash_func = hash_func
2262 self._object_format = object_format
2263 self._resolve_ext_ref = resolve_ext_ref
2264 self._pending_ofs: dict[int, list[int]] = defaultdict(list)
2265 self._pending_ref: dict[bytes, list[int]] = defaultdict(list)
2266 self._full_ofs: list[tuple[int, int]] = []
2267 self._ext_refs: list[RawObjectID] = []
2269 @classmethod
2270 def for_pack_data(
2271 cls, pack_data: PackData, resolve_ext_ref: ResolveExtRefFn | None = None
2272 ) -> "DeltaChainIterator[T]":
2273 """Create a DeltaChainIterator from pack data.
2275 Args:
2276 pack_data: PackData object to iterate
2277 resolve_ext_ref: Optional function to resolve external refs
2279 Returns:
2280 DeltaChainIterator instance
2281 """
2282 walker = cls(
2283 None,
2284 pack_data.object_format.hash_func,
2285 resolve_ext_ref=resolve_ext_ref,
2286 object_format=pack_data.object_format,
2287 )
2288 walker.set_pack_data(pack_data)
2289 for unpacked in pack_data.iter_unpacked(include_comp=False):
2290 walker.record(unpacked)
2291 return walker
2293 @classmethod
2294 def for_pack_subset(
2295 cls,
2296 pack: "Pack",
2297 shas: Iterable[ObjectID | RawObjectID],
2298 *,
2299 allow_missing: bool = False,
2300 resolve_ext_ref: ResolveExtRefFn | None = None,
2301 ) -> "DeltaChainIterator[T]":
2302 """Create a DeltaChainIterator for a subset of objects.
2304 Args:
2305 pack: Pack object containing the data
2306 shas: Iterable of object SHAs to include
2307 allow_missing: If True, skip missing objects
2308 resolve_ext_ref: Optional function to resolve external refs
2310 Returns:
2311 DeltaChainIterator instance
2312 """
2313 walker = cls(
2314 None,
2315 pack.object_format.hash_func,
2316 resolve_ext_ref=resolve_ext_ref,
2317 object_format=pack.object_format,
2318 )
2319 walker.set_pack_data(pack.data)
2320 todo = set()
2321 for sha in shas:
2322 try:
2323 off = pack.index.object_offset(sha)
2324 except KeyError:
2325 if not allow_missing:
2326 raise
2327 else:
2328 todo.add(off)
2329 done = set()
2330 while todo:
2331 off = todo.pop()
2332 unpacked = pack.data.get_unpacked_object_at(off)
2333 walker.record(unpacked)
2334 done.add(off)
2335 base_ofs = None
2336 if unpacked.pack_type_num == OFS_DELTA:
2337 assert unpacked.offset is not None
2338 assert unpacked.delta_base is not None
2339 assert isinstance(unpacked.delta_base, int)
2340 base_ofs = unpacked.offset - unpacked.delta_base
2341 elif unpacked.pack_type_num == REF_DELTA:
2342 with suppress(KeyError):
2343 assert isinstance(unpacked.delta_base, bytes)
2344 base_ofs = pack.index.object_offset(
2345 RawObjectID(unpacked.delta_base)
2346 )
2347 if base_ofs is not None and base_ofs not in done:
2348 todo.add(base_ofs)
2349 return walker
2351 def record(self, unpacked: UnpackedObject) -> None:
2352 """Record an unpacked object for later processing.
2354 Args:
2355 unpacked: UnpackedObject to record
2356 """
2357 type_num = unpacked.pack_type_num
2358 offset = unpacked.offset
2359 assert offset is not None
2360 if type_num == OFS_DELTA:
2361 assert unpacked.delta_base is not None
2362 assert isinstance(unpacked.delta_base, int)
2363 base_offset = offset - unpacked.delta_base
2364 self._pending_ofs[base_offset].append(offset)
2365 elif type_num == REF_DELTA:
2366 assert isinstance(unpacked.delta_base, bytes)
2367 self._pending_ref[unpacked.delta_base].append(offset)
2368 else:
2369 self._full_ofs.append((offset, type_num))
2371 def set_pack_data(self, pack_data: PackData) -> None:
2372 """Set the pack data for iteration.
2374 Args:
2375 pack_data: PackData object to use
2376 """
2377 self._file = pack_data._file
2379 def _walk_all_chains(self) -> Iterator[T]:
2380 for offset, type_num in self._full_ofs:
2381 yield from self._follow_chain(offset, type_num, None)
2382 yield from self._walk_ref_chains()
2383 assert not self._pending_ofs, repr(self._pending_ofs)
2385 def _ensure_no_pending(self) -> None:
2386 if self._pending_ref:
2387 raise UnresolvedDeltas(
2388 [sha_to_hex(RawObjectID(s)) for s in self._pending_ref]
2389 )
2391 def _walk_ref_chains(self) -> Iterator[T]:
2392 if not self._resolve_ext_ref:
2393 self._ensure_no_pending()
2394 return
2396 for base_sha, pending in sorted(self._pending_ref.items()):
2397 if base_sha not in self._pending_ref:
2398 continue
2399 try:
2400 type_num, chunks = self._resolve_ext_ref(RawObjectID(base_sha))
2401 except KeyError:
2402 # Not an external ref, but may depend on one. Either it will
2403 # get popped via a _follow_chain call, or we will raise an
2404 # error below.
2405 continue
2406 self._ext_refs.append(RawObjectID(base_sha))
2407 self._pending_ref.pop(base_sha)
2408 for new_offset in pending:
2409 yield from self._follow_chain(new_offset, type_num, chunks)
2411 self._ensure_no_pending()
2413 def _result(self, unpacked: UnpackedObject) -> T:
2414 raise NotImplementedError
2416 def _resolve_object(
2417 self,
2418 offset: int,
2419 obj_type_num: int,
2420 base_chunks: bytes | list[bytes] | None,
2421 ) -> UnpackedObject:
2422 assert self._file is not None
2423 self._file.seek(offset)
2424 unpacked, _ = unpack_object(
2425 self._file.read,
2426 self.hash_func,
2427 read_some=None,
2428 compute_crc32=self._compute_crc32,
2429 include_comp=self._include_comp,
2430 )
2431 unpacked.offset = offset
2432 if base_chunks is None:
2433 assert unpacked.pack_type_num == obj_type_num
2434 else:
2435 assert unpacked.pack_type_num in DELTA_TYPES
2436 unpacked.obj_type_num = obj_type_num
2437 unpacked.obj_chunks = apply_delta(base_chunks, unpacked.decomp_chunks)
2438 # A delta that resolves to a zero-byte payload for a
2439 # commit/tree/tag is malformed: ``_parse_message`` /
2440 # ``parse_tree`` accept the empty input silently, so without
2441 # this guard a too-short delta could materialise an
2442 # otherwise-valid SHA pointing at an empty commit object
2443 # (which ``git fsck`` rejects). Only blobs may legitimately
2444 # be empty, and an empty blob would never be stored as a
2445 # delta in practice.
2446 # Blob.type_num == 3 (avoid the import cycle).
2447 if obj_type_num != 3 and chunks_length(unpacked.obj_chunks) == 0:
2448 raise ApplyDeltaError(
2449 f"delta resolved to empty payload for type {obj_type_num}"
2450 )
2451 return unpacked
2453 def _follow_chain(
2454 self,
2455 offset: int,
2456 obj_type_num: int,
2457 base_chunks: bytes | list[bytes] | None,
2458 ) -> Iterator[T]:
2459 # Unlike PackData.get_object_at, there is no need to cache offsets as
2460 # this approach by design inflates each object exactly once.
2461 todo = [(offset, obj_type_num, base_chunks)]
2462 while todo:
2463 (offset, obj_type_num, base_chunks) = todo.pop()
2464 unpacked = self._resolve_object(offset, obj_type_num, base_chunks)
2465 yield self._result(unpacked)
2467 assert unpacked.offset is not None
2468 unblocked = chain(
2469 self._pending_ofs.pop(unpacked.offset, []),
2470 self._pending_ref.pop(unpacked.sha(), []),
2471 )
2472 todo.extend(
2473 (new_offset, unpacked.obj_type_num, unpacked.obj_chunks) # type: ignore
2474 for new_offset in unblocked
2475 )
2477 def __iter__(self) -> Iterator[T]:
2478 """Iterate over objects in the pack."""
2479 return self._walk_all_chains()
2481 def ext_refs(self) -> list[RawObjectID]:
2482 """Return external references."""
2483 return self._ext_refs
2486class UnpackedObjectIterator(DeltaChainIterator[UnpackedObject]):
2487 """Delta chain iterator that yield unpacked objects."""
2489 def _result(self, unpacked: UnpackedObject) -> UnpackedObject:
2490 """Return the unpacked object.
2492 Args:
2493 unpacked: The unpacked object
2495 Returns:
2496 The unpacked object unchanged
2497 """
2498 return unpacked
2501class PackIndexer(DeltaChainIterator[PackIndexEntry]):
2502 """Delta chain iterator that yields index entries."""
2504 _compute_crc32 = True
2506 def _result(self, unpacked: UnpackedObject) -> PackIndexEntry:
2507 """Convert unpacked object to pack index entry.
2509 Args:
2510 unpacked: The unpacked object
2512 Returns:
2513 Tuple of (sha, offset, crc32) for index entry
2514 """
2515 assert unpacked.offset is not None
2516 return unpacked.sha(), unpacked.offset, unpacked.crc32
2519class PackInflater(DeltaChainIterator[ShaFile]):
2520 """Delta chain iterator that yields ShaFile objects."""
2522 def _result(self, unpacked: UnpackedObject) -> ShaFile:
2523 """Convert unpacked object to ShaFile.
2525 Args:
2526 unpacked: The unpacked object
2528 Returns:
2529 ShaFile object from the unpacked data
2530 """
2531 assert unpacked.obj_type_num is not None and unpacked.obj_chunks is not None
2532 return ShaFile.from_raw_chunks(
2533 unpacked.obj_type_num,
2534 unpacked.obj_chunks,
2535 object_format=self._object_format,
2536 )
2539class SHA1Reader(BinaryIO):
2540 """Wrapper for file-like object that remembers the SHA1 of its data."""
2542 def __init__(self, f: IO[bytes]) -> None:
2543 """Initialize SHA1Reader.
2545 Args:
2546 f: File-like object to wrap
2547 """
2548 self.f = f
2549 self.sha1 = sha1(b"")
2551 def read(self, size: int = -1) -> bytes:
2552 """Read bytes and update SHA1.
2554 Args:
2555 size: Number of bytes to read, -1 for all
2557 Returns:
2558 Bytes read from file
2559 """
2560 data = self.f.read(size)
2561 self.sha1.update(data)
2562 return data
2564 def check_sha(self, allow_empty: bool = False) -> None:
2565 """Check if the SHA1 matches the expected value.
2567 Args:
2568 allow_empty: Allow empty SHA1 hash
2570 Raises:
2571 ChecksumMismatch: If SHA1 doesn't match
2572 """
2573 stored = self.f.read(20)
2574 # If git option index.skipHash is set the index will be empty
2575 if stored != self.sha1.digest() and (
2576 not allow_empty
2577 or (
2578 len(stored) == 20
2579 and sha_to_hex(RawObjectID(stored))
2580 != b"0000000000000000000000000000000000000000"
2581 )
2582 ):
2583 raise ChecksumMismatch(
2584 self.sha1.hexdigest(),
2585 sha_to_hex(RawObjectID(stored)) if stored else b"",
2586 )
2588 def close(self) -> None:
2589 """Close the underlying file."""
2590 return self.f.close()
2592 def tell(self) -> int:
2593 """Return current file position."""
2594 return self.f.tell()
2596 # BinaryIO abstract methods
2597 def readable(self) -> bool:
2598 """Check if file is readable."""
2599 return True
2601 def writable(self) -> bool:
2602 """Check if file is writable."""
2603 return False
2605 def seekable(self) -> bool:
2606 """Check if file is seekable."""
2607 return getattr(self.f, "seekable", lambda: False)()
2609 def seek(self, offset: int, whence: int = 0) -> int:
2610 """Seek to position in file.
2612 Args:
2613 offset: Position offset
2614 whence: Reference point (0=start, 1=current, 2=end)
2616 Returns:
2617 New file position
2618 """
2619 return self.f.seek(offset, whence)
2621 def flush(self) -> None:
2622 """Flush the file buffer."""
2623 if hasattr(self.f, "flush"):
2624 self.f.flush()
2626 def readline(self, size: int = -1) -> bytes:
2627 """Read a line from the file.
2629 Args:
2630 size: Maximum bytes to read
2632 Returns:
2633 Line read from file
2634 """
2635 return self.f.readline(size)
2637 def readlines(self, hint: int = -1) -> list[bytes]:
2638 """Read all lines from the file.
2640 Args:
2641 hint: Approximate number of bytes to read
2643 Returns:
2644 List of lines
2645 """
2646 return self.f.readlines(hint)
2648 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
2649 """Write multiple lines to the file (not supported)."""
2650 raise UnsupportedOperation("writelines")
2652 def write(self, data: bytes, /) -> int: # type: ignore[override]
2653 """Write data to the file (not supported)."""
2654 raise UnsupportedOperation("write")
2656 def __enter__(self) -> Self:
2657 """Enter context manager."""
2658 return self
2660 def __exit__(
2661 self,
2662 type: type | None,
2663 value: BaseException | None,
2664 traceback: TracebackType | None,
2665 ) -> None:
2666 """Exit context manager and close file."""
2667 self.close()
2669 def __iter__(self) -> "SHA1Reader":
2670 """Return iterator for reading file lines."""
2671 return self
2673 def __next__(self) -> bytes:
2674 """Get next line from file.
2676 Returns:
2677 Next line
2679 Raises:
2680 StopIteration: When no more lines
2681 """
2682 line = self.readline()
2683 if not line:
2684 raise StopIteration
2685 return line
2687 def fileno(self) -> int:
2688 """Return file descriptor number."""
2689 return self.f.fileno()
2691 def isatty(self) -> bool:
2692 """Check if file is a terminal."""
2693 return getattr(self.f, "isatty", lambda: False)()
2695 def truncate(self, size: int | None = None) -> int:
2696 """Not supported for read-only file.
2698 Raises:
2699 UnsupportedOperation: Always raised
2700 """
2701 raise UnsupportedOperation("truncate")
2704class SHA1Writer(BinaryIO):
2705 """Wrapper for file-like object that remembers the SHA1 of its data."""
2707 def __init__(self, f: BinaryIO | IO[bytes]) -> None:
2708 """Initialize SHA1Writer.
2710 Args:
2711 f: File-like object to wrap
2712 """
2713 self.f = f
2714 self.length = 0
2715 self.sha1 = sha1(b"")
2716 self.digest: bytes | None = None
2718 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
2719 """Write data and update SHA1.
2721 Args:
2722 data: Data to write
2724 Returns:
2725 Number of bytes written
2726 """
2727 self.sha1.update(data)
2728 written = self.f.write(data)
2729 self.length += written
2730 return written
2732 def write_sha(self) -> bytes:
2733 """Write the SHA1 digest to the file.
2735 Returns:
2736 The SHA1 digest bytes
2737 """
2738 sha = self.sha1.digest()
2739 assert len(sha) == 20
2740 self.f.write(sha)
2741 self.length += len(sha)
2742 return sha
2744 def close(self) -> None:
2745 """Close the pack file and finalize the SHA."""
2746 self.digest = self.write_sha()
2747 self.f.close()
2749 def offset(self) -> int:
2750 """Get the total number of bytes written.
2752 Returns:
2753 Total bytes written
2754 """
2755 return self.length
2757 def tell(self) -> int:
2758 """Return current file position."""
2759 return self.f.tell()
2761 # BinaryIO abstract methods
2762 def readable(self) -> bool:
2763 """Check if file is readable."""
2764 return False
2766 def writable(self) -> bool:
2767 """Check if file is writable."""
2768 return True
2770 def seekable(self) -> bool:
2771 """Check if file is seekable."""
2772 return getattr(self.f, "seekable", lambda: False)()
2774 def seek(self, offset: int, whence: int = 0) -> int:
2775 """Seek to position in file.
2777 Args:
2778 offset: Position offset
2779 whence: Reference point (0=start, 1=current, 2=end)
2781 Returns:
2782 New file position
2783 """
2784 return self.f.seek(offset, whence)
2786 def flush(self) -> None:
2787 """Flush the file buffer."""
2788 if hasattr(self.f, "flush"):
2789 self.f.flush()
2791 def readline(self, size: int = -1) -> bytes:
2792 """Not supported for write-only file.
2794 Raises:
2795 UnsupportedOperation: Always raised
2796 """
2797 raise UnsupportedOperation("readline")
2799 def readlines(self, hint: int = -1) -> list[bytes]:
2800 """Not supported for write-only file.
2802 Raises:
2803 UnsupportedOperation: Always raised
2804 """
2805 raise UnsupportedOperation("readlines")
2807 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
2808 """Write multiple lines to the file.
2810 Args:
2811 lines: Iterable of lines to write
2812 """
2813 for line in lines:
2814 self.write(line)
2816 def read(self, size: int = -1) -> bytes:
2817 """Not supported for write-only file.
2819 Raises:
2820 UnsupportedOperation: Always raised
2821 """
2822 raise UnsupportedOperation("read")
2824 def __enter__(self) -> Self:
2825 """Enter context manager."""
2826 return self
2828 def __exit__(
2829 self,
2830 type: type | None,
2831 value: BaseException | None,
2832 traceback: TracebackType | None,
2833 ) -> None:
2834 """Exit context manager and close file."""
2835 self.f.close()
2837 def __iter__(self) -> "SHA1Writer":
2838 """Return iterator."""
2839 return self
2841 def __next__(self) -> bytes:
2842 """Not supported for write-only file.
2844 Raises:
2845 UnsupportedOperation: Always raised
2846 """
2847 raise UnsupportedOperation("__next__")
2849 def fileno(self) -> int:
2850 """Return file descriptor number."""
2851 return self.f.fileno()
2853 def isatty(self) -> bool:
2854 """Check if file is a terminal."""
2855 return getattr(self.f, "isatty", lambda: False)()
2857 def truncate(self, size: int | None = None) -> int:
2858 """Not supported for write-only file.
2860 Raises:
2861 UnsupportedOperation: Always raised
2862 """
2863 raise UnsupportedOperation("truncate")
2866class HashWriter(BinaryIO):
2867 """Wrapper for file-like object that computes hash of its data.
2869 This is a generic version that works with any hash algorithm.
2870 """
2872 def __init__(
2873 self, f: BinaryIO | IO[bytes], hash_func: Callable[[], "HashObject"]
2874 ) -> None:
2875 """Initialize HashWriter.
2877 Args:
2878 f: File-like object to wrap
2879 hash_func: Hash function (e.g., sha1, sha256)
2880 """
2881 self.f = f
2882 self.length = 0
2883 self.hash_obj = hash_func()
2884 self.digest: bytes | None = None
2886 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
2887 """Write data and update hash.
2889 Args:
2890 data: Data to write
2892 Returns:
2893 Number of bytes written
2894 """
2895 self.hash_obj.update(data)
2896 written = self.f.write(data)
2897 self.length += written
2898 return written
2900 def write_hash(self) -> bytes:
2901 """Write the hash digest to the file.
2903 Returns:
2904 The hash digest bytes
2905 """
2906 digest = self.hash_obj.digest()
2907 self.f.write(digest)
2908 self.length += len(digest)
2909 return digest
2911 def close(self) -> None:
2912 """Close the pack file and finalize the hash."""
2913 self.digest = self.write_hash()
2914 self.f.close()
2916 def offset(self) -> int:
2917 """Get the total number of bytes written.
2919 Returns:
2920 Total bytes written
2921 """
2922 return self.length
2924 def tell(self) -> int:
2925 """Return current file position."""
2926 return self.f.tell()
2928 # BinaryIO abstract methods
2929 def readable(self) -> bool:
2930 """Check if file is readable."""
2931 return False
2933 def writable(self) -> bool:
2934 """Check if file is writable."""
2935 return True
2937 def seekable(self) -> bool:
2938 """Check if file is seekable."""
2939 return getattr(self.f, "seekable", lambda: False)()
2941 def seek(self, offset: int, whence: int = 0) -> int:
2942 """Seek to position in file.
2944 Args:
2945 offset: Position offset
2946 whence: Reference point (0=start, 1=current, 2=end)
2948 Returns:
2949 New file position
2950 """
2951 return self.f.seek(offset, whence)
2953 def flush(self) -> None:
2954 """Flush the file buffer."""
2955 if hasattr(self.f, "flush"):
2956 self.f.flush()
2958 def readline(self, size: int = -1) -> bytes:
2959 """Not supported for write-only file.
2961 Raises:
2962 UnsupportedOperation: Always raised
2963 """
2964 raise UnsupportedOperation("readline")
2966 def readlines(self, hint: int = -1) -> list[bytes]:
2967 """Not supported for write-only file.
2969 Raises:
2970 UnsupportedOperation: Always raised
2971 """
2972 raise UnsupportedOperation("readlines")
2974 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
2975 """Write multiple lines to the file.
2977 Args:
2978 lines: Iterable of lines to write
2979 """
2980 for line in lines:
2981 self.write(line)
2983 def read(self, size: int = -1) -> bytes:
2984 """Not supported for write-only file.
2986 Raises:
2987 UnsupportedOperation: Always raised
2988 """
2989 raise UnsupportedOperation("read")
2991 def __enter__(self) -> Self:
2992 """Enter context manager."""
2993 return self
2995 def __exit__(
2996 self,
2997 type: type | None,
2998 value: BaseException | None,
2999 traceback: TracebackType | None,
3000 ) -> None:
3001 """Exit context manager and close file."""
3002 self.close()
3004 def __iter__(self) -> "HashWriter":
3005 """Return iterator."""
3006 return self
3008 def __next__(self) -> bytes:
3009 """Not supported for write-only file.
3011 Raises:
3012 UnsupportedOperation: Always raised
3013 """
3014 raise UnsupportedOperation("__next__")
3016 def fileno(self) -> int:
3017 """Return file descriptor number."""
3018 return self.f.fileno()
3020 def isatty(self) -> bool:
3021 """Check if file is a terminal."""
3022 return getattr(self.f, "isatty", lambda: False)()
3024 def truncate(self, size: int | None = None) -> int:
3025 """Not supported for write-only file.
3027 Raises:
3028 UnsupportedOperation: Always raised
3029 """
3030 raise UnsupportedOperation("truncate")
3033def pack_object_header(
3034 type_num: int,
3035 delta_base: bytes | int | None,
3036 size: int,
3037 object_format: "ObjectFormat",
3038) -> bytearray:
3039 """Create a pack object header for the given object info.
3041 Args:
3042 type_num: Numeric type of the object.
3043 delta_base: Delta base offset or ref, or None for whole objects.
3044 size: Uncompressed object size.
3045 object_format: Object format (hash algorithm) to use.
3046 Returns: A header for a packed object.
3047 """
3048 header = []
3049 c = (type_num << 4) | (size & 15)
3050 size >>= 4
3051 while size:
3052 header.append(c | 0x80)
3053 c = size & 0x7F
3054 size >>= 7
3055 header.append(c)
3056 if type_num == OFS_DELTA:
3057 assert isinstance(delta_base, int)
3058 ret = [delta_base & 0x7F]
3059 delta_base >>= 7
3060 while delta_base:
3061 delta_base -= 1
3062 ret.insert(0, 0x80 | (delta_base & 0x7F))
3063 delta_base >>= 7
3064 header.extend(ret)
3065 elif type_num == REF_DELTA:
3066 assert isinstance(delta_base, bytes)
3067 assert len(delta_base) == object_format.oid_length
3068 header += delta_base
3069 return bytearray(header)
3072def pack_object_chunks(
3073 type: int,
3074 object: list[bytes] | tuple[bytes | int, list[bytes]],
3075 object_format: "ObjectFormat",
3076 *,
3077 compression_level: int = -1,
3078) -> Iterator[bytes]:
3079 """Generate chunks for a pack object.
3081 Args:
3082 type: Numeric type of the object
3083 object: Object to write
3084 object_format: Object format (hash algorithm) to use
3085 compression_level: the zlib compression level
3086 Returns: Chunks
3087 """
3088 if type in DELTA_TYPES:
3089 if isinstance(object, tuple):
3090 delta_base, object = object
3091 else:
3092 raise TypeError("Delta types require a tuple of (delta_base, object)")
3093 else:
3094 delta_base = None
3096 # Convert object to list of bytes chunks
3097 if isinstance(object, bytes):
3098 chunks = [object]
3099 elif isinstance(object, list):
3100 chunks = object
3101 elif isinstance(object, ShaFile):
3102 chunks = object.as_raw_chunks()
3103 else:
3104 # Shouldn't reach here with proper typing
3105 raise TypeError(f"Unexpected object type: {object.__class__.__name__}")
3107 yield bytes(
3108 pack_object_header(
3109 type, delta_base, sum(map(len, chunks)), object_format=object_format
3110 )
3111 )
3112 compressor = zlib.compressobj(level=compression_level)
3113 for data in chunks:
3114 yield compressor.compress(data)
3115 yield compressor.flush()
3118def write_pack_object(
3119 write: Callable[[bytes], int],
3120 type: int,
3121 object: list[bytes] | tuple[bytes | int, list[bytes]],
3122 object_format: "ObjectFormat",
3123 *,
3124 sha: "HashObject | None" = None,
3125 compression_level: int = -1,
3126) -> int:
3127 """Write pack object to a file.
3129 Args:
3130 write: Write function to use
3131 type: Numeric type of the object
3132 object: Object to write
3133 object_format: Object format (hash algorithm) to use
3134 sha: Optional SHA-1 hasher to update
3135 compression_level: the zlib compression level
3136 Returns: CRC32 checksum of the written object
3137 """
3138 crc32 = 0
3139 for chunk in pack_object_chunks(
3140 type, object, compression_level=compression_level, object_format=object_format
3141 ):
3142 write(chunk)
3143 if sha is not None:
3144 sha.update(chunk)
3145 crc32 = binascii.crc32(chunk, crc32)
3146 return crc32 & 0xFFFFFFFF
3149def write_pack(
3150 filename: str,
3151 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3152 object_format: "ObjectFormat",
3153 *,
3154 deltify: bool | None = None,
3155 delta_window_size: int | None = None,
3156 compression_level: int = -1,
3157) -> tuple[bytes, bytes]:
3158 """Write a new pack data file.
3160 Args:
3161 filename: Path to the new pack file (without .pack extension)
3162 objects: Objects to write to the pack
3163 object_format: Object format
3164 delta_window_size: Delta window size
3165 deltify: Whether to deltify pack objects
3166 compression_level: the zlib compression level
3167 Returns: Tuple with checksum of pack file and index file
3168 """
3169 with GitFile(filename + ".pack", "wb") as f:
3170 entries, data_sum = write_pack_objects(
3171 f,
3172 objects,
3173 delta_window_size=delta_window_size,
3174 deltify=deltify,
3175 compression_level=compression_level,
3176 object_format=object_format,
3177 )
3178 entries_list = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
3179 with GitFile(filename + ".idx", "wb") as f:
3180 idx_sha = write_pack_index(f, entries_list, data_sum)
3181 return data_sum, idx_sha
3184def pack_header_chunks(num_objects: int) -> Iterator[bytes]:
3185 """Yield chunks for a pack header."""
3186 yield b"PACK" # Pack header
3187 yield struct.pack(b">L", 2) # Pack version
3188 yield struct.pack(b">L", num_objects) # Number of objects in pack
3191def write_pack_header(
3192 write: Callable[[bytes], int] | IO[bytes], num_objects: int
3193) -> None:
3194 """Write a pack header for the given number of objects."""
3195 if not callable(write):
3196 write_fn: Callable[[bytes], int] = write.write
3197 warnings.warn(
3198 "write_pack_header() now takes a write rather than file argument",
3199 DeprecationWarning,
3200 stacklevel=2,
3201 )
3202 else:
3203 write_fn = write
3204 for chunk in pack_header_chunks(num_objects):
3205 write_fn(chunk)
3208def find_reusable_deltas(
3209 container: PackedObjectContainer,
3210 object_ids: Set[ObjectID],
3211 *,
3212 other_haves: Set[ObjectID] | None = None,
3213 progress: Callable[..., None] | None = None,
3214) -> Iterator[UnpackedObject]:
3215 """Find deltas in a pack that can be reused.
3217 Args:
3218 container: Pack container to search for deltas
3219 object_ids: Set of object IDs to find deltas for
3220 other_haves: Set of other object IDs we have
3221 progress: Optional progress reporting callback
3223 Returns:
3224 Iterator of UnpackedObject entries that can be reused
3225 """
3226 if other_haves is None:
3227 other_haves = set()
3228 reused = 0
3229 for i, unpacked in enumerate(
3230 container.iter_unpacked_subset(
3231 object_ids, allow_missing=True, convert_ofs_delta=True
3232 )
3233 ):
3234 if progress is not None and i % 1000 == 0:
3235 progress(f"checking for reusable deltas: {i}/{len(object_ids)}\r".encode())
3236 if unpacked.pack_type_num == REF_DELTA:
3237 hexsha = sha_to_hex(unpacked.delta_base) # type: ignore
3238 if hexsha in object_ids or hexsha in other_haves:
3239 yield unpacked
3240 reused += 1
3241 if progress is not None:
3242 progress((f"found {reused} deltas to reuse\n").encode())
3245def deltify_pack_objects(
3246 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, bytes | None]],
3247 *,
3248 window_size: int | None = None,
3249 progress: Callable[..., None] | None = None,
3250) -> Iterator[UnpackedObject]:
3251 """Generate deltas for pack objects.
3253 Args:
3254 objects: An iterable of (object, path) tuples to deltify.
3255 window_size: Window size; None for default
3256 progress: Optional progress reporting callback
3257 Returns: Iterator over type_num, object id, delta_base, content
3258 delta_base is None for full text entries
3259 """
3261 def objects_with_hints() -> Iterator[tuple[ShaFile, tuple[int, bytes | None]]]:
3262 for e in objects:
3263 if isinstance(e, ShaFile):
3264 yield (e, (e.type_num, None))
3265 else:
3266 yield (e[0], (e[0].type_num, e[1]))
3268 sorted_objs = sort_objects_for_delta(objects_with_hints())
3269 yield from deltas_from_sorted_objects(
3270 sorted_objs,
3271 window_size=window_size,
3272 progress=progress,
3273 )
3276def sort_objects_for_delta(
3277 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, PackHint | None]],
3278) -> Iterator[tuple[ShaFile, bytes | None]]:
3279 """Sort objects for optimal delta compression.
3281 Args:
3282 objects: Iterator of objects or (object, hint) tuples
3284 Returns:
3285 Iterator of sorted (ShaFile, path) tuples
3286 """
3287 magic = []
3288 for entry in objects:
3289 if isinstance(entry, tuple):
3290 obj, hint = entry
3291 if hint is None:
3292 type_num = None
3293 path = None
3294 else:
3295 (type_num, path) = hint
3296 else:
3297 obj = entry
3298 type_num = None
3299 path = None
3300 magic.append((type_num, path, -obj.raw_length(), obj))
3301 # Build a list of objects ordered by the magic Linus heuristic
3302 # This helps us find good objects to diff against us
3303 magic.sort()
3304 return ((x[3], x[1]) for x in magic)
3307def deltas_from_sorted_objects(
3308 objects: Iterator[tuple[ShaFile, bytes | None]],
3309 window_size: int | None = None,
3310 progress: Callable[..., None] | None = None,
3311) -> Iterator[UnpackedObject]:
3312 """Create deltas from sorted objects.
3314 Args:
3315 objects: Iterator of sorted objects to deltify
3316 window_size: Delta window size; None for default
3317 progress: Optional progress reporting callback
3319 Returns:
3320 Iterator of UnpackedObject entries
3321 """
3322 # TODO(jelmer): Use threads
3323 if window_size is None:
3324 window_size = DEFAULT_PACK_DELTA_WINDOW_SIZE
3326 possible_bases: deque[tuple[bytes, int, bytes]] = deque()
3327 for i, (o, path) in enumerate(objects):
3328 if progress is not None and i % 1000 == 0:
3329 progress((f"generating deltas: {i}\r").encode())
3330 raw = o.as_raw_chunks()
3331 raw_bytes = b"".join(raw) # Join once for efficiency
3332 winner = raw
3333 winner_len = sum(map(len, winner))
3334 winner_base = None
3335 for base_id, base_type_num, base_bytes in possible_bases:
3336 if base_type_num != o.type_num:
3337 continue
3338 delta_len = 0
3339 delta = []
3340 for chunk in create_delta(base_bytes, raw_bytes):
3341 delta_len += len(chunk)
3342 if delta_len >= winner_len:
3343 break
3344 delta.append(chunk)
3345 else:
3346 winner_base = base_id
3347 winner = delta
3348 winner_len = sum(map(len, winner))
3349 yield UnpackedObject(
3350 o.type_num,
3351 sha=o.sha().digest(),
3352 delta_base=winner_base,
3353 decomp_len=winner_len,
3354 decomp_chunks=winner,
3355 )
3356 possible_bases.appendleft((o.sha().digest(), o.type_num, raw_bytes))
3357 while len(possible_bases) > window_size:
3358 possible_bases.pop()
3361def pack_objects_to_data(
3362 objects: Sequence[ShaFile]
3363 | Sequence[tuple[ShaFile, bytes | None]]
3364 | Sequence[tuple[ShaFile, PackHint | None]],
3365 *,
3366 deltify: bool | None = None,
3367 delta_window_size: int | None = None,
3368 ofs_delta: bool = True,
3369 progress: Callable[..., None] | None = None,
3370) -> tuple[int, Iterator[UnpackedObject]]:
3371 """Create pack data from objects.
3373 Args:
3374 objects: Pack objects
3375 deltify: Whether to deltify pack objects
3376 delta_window_size: Delta window size
3377 ofs_delta: Whether to use offset deltas
3378 progress: Optional progress reporting callback
3379 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3380 """
3381 count = len(objects)
3382 if deltify is None:
3383 # PERFORMANCE/TODO(jelmer): This should be enabled but the python
3384 # implementation is *much* too slow at the moment.
3385 # Maybe consider enabling it just if the rust extension is available?
3386 deltify = False
3387 if deltify:
3388 return (
3389 count,
3390 deltify_pack_objects(
3391 iter(objects), # type: ignore
3392 window_size=delta_window_size,
3393 progress=progress,
3394 ),
3395 )
3396 else:
3398 def iter_without_path() -> Iterator[UnpackedObject]:
3399 for o in objects:
3400 if isinstance(o, tuple):
3401 yield full_unpacked_object(o[0])
3402 else:
3403 yield full_unpacked_object(o)
3405 return (count, iter_without_path())
3408def generate_unpacked_objects(
3409 container: PackedObjectContainer,
3410 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3411 delta_window_size: int | None = None,
3412 deltify: bool | None = None,
3413 reuse_deltas: bool = True,
3414 ofs_delta: bool = True,
3415 other_haves: set[ObjectID] | None = None,
3416 progress: Callable[..., None] | None = None,
3417) -> Iterator[UnpackedObject]:
3418 """Create pack data from objects.
3420 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3421 """
3422 todo = dict(object_ids)
3423 if reuse_deltas:
3424 for unpack in find_reusable_deltas(
3425 container, set(todo), other_haves=other_haves, progress=progress
3426 ):
3427 del todo[sha_to_hex(RawObjectID(unpack.sha()))]
3428 yield unpack
3429 if deltify is None:
3430 # PERFORMANCE/TODO(jelmer): This should be enabled but is *much* too
3431 # slow at the moment.
3432 deltify = False
3433 if deltify:
3434 objects_to_delta = container.iterobjects_subset(
3435 todo.keys(), allow_missing=False
3436 )
3437 sorted_objs = sort_objects_for_delta((o, todo[o.id]) for o in objects_to_delta)
3438 yield from deltas_from_sorted_objects(
3439 sorted_objs,
3440 window_size=delta_window_size,
3441 progress=progress,
3442 )
3443 else:
3444 for oid in todo:
3445 yield full_unpacked_object(container[oid])
3448def full_unpacked_object(o: ShaFile) -> UnpackedObject:
3449 """Create an UnpackedObject from a ShaFile.
3451 Args:
3452 o: ShaFile object to convert
3454 Returns:
3455 UnpackedObject with full object data
3456 """
3457 return UnpackedObject(
3458 o.type_num,
3459 delta_base=None,
3460 crc32=None,
3461 decomp_chunks=o.as_raw_chunks(),
3462 sha=o.sha().digest(),
3463 )
3466def write_pack_from_container(
3467 write: Callable[[bytes], None]
3468 | Callable[[bytes | bytearray | memoryview], int]
3469 | IO[bytes],
3470 container: PackedObjectContainer,
3471 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3472 object_format: "ObjectFormat",
3473 *,
3474 delta_window_size: int | None = None,
3475 deltify: bool | None = None,
3476 reuse_deltas: bool = True,
3477 compression_level: int = -1,
3478 other_haves: set[ObjectID] | None = None,
3479) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3480 """Write a new pack data file.
3482 Args:
3483 write: write function to use
3484 container: PackedObjectContainer
3485 object_ids: Sequence of (object_id, hint) tuples to write
3486 object_format: Object format (hash algorithm) to use
3487 delta_window_size: Sliding window size for searching for deltas;
3488 Set to None for default window size.
3489 deltify: Whether to deltify objects
3490 reuse_deltas: Whether to reuse existing deltas
3491 compression_level: the zlib compression level to use
3492 other_haves: Set of additional object IDs the receiver has
3493 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3494 """
3495 pack_contents_count = len(object_ids)
3496 pack_contents = generate_unpacked_objects(
3497 container,
3498 object_ids,
3499 delta_window_size=delta_window_size,
3500 deltify=deltify,
3501 reuse_deltas=reuse_deltas,
3502 other_haves=other_haves,
3503 )
3505 return write_pack_data(
3506 write,
3507 pack_contents,
3508 num_records=pack_contents_count,
3509 compression_level=compression_level,
3510 object_format=object_format,
3511 )
3514def write_pack_objects(
3515 write: Callable[[bytes], None] | IO[bytes],
3516 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3517 object_format: "ObjectFormat",
3518 *,
3519 delta_window_size: int | None = None,
3520 deltify: bool | None = None,
3521 compression_level: int = -1,
3522) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3523 """Write a new pack data file.
3525 Args:
3526 write: write function to use
3527 objects: Sequence of (object, path) tuples to write
3528 object_format: Object format (hash algorithm) to use
3529 delta_window_size: Sliding window size for searching for deltas;
3530 Set to None for default window size.
3531 deltify: Whether to deltify objects
3532 compression_level: the zlib compression level to use
3533 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3534 """
3535 pack_contents_count, pack_contents = pack_objects_to_data(objects, deltify=deltify)
3537 return write_pack_data(
3538 write,
3539 pack_contents,
3540 num_records=pack_contents_count,
3541 compression_level=compression_level,
3542 object_format=object_format,
3543 )
3546class PackChunkGenerator:
3547 """Generator for pack data chunks."""
3549 def __init__(
3550 self,
3551 object_format: "ObjectFormat",
3552 num_records: int | None = None,
3553 records: Iterator[UnpackedObject] | None = None,
3554 progress: Callable[..., None] | None = None,
3555 compression_level: int = -1,
3556 reuse_compressed: bool = True,
3557 ) -> None:
3558 """Initialize PackChunkGenerator.
3560 Args:
3561 num_records: Expected number of records
3562 records: Iterator of pack records
3563 progress: Optional progress callback
3564 compression_level: Compression level (-1 for default)
3565 reuse_compressed: Whether to reuse compressed chunks
3566 object_format: Object format (hash algorithm) to use
3567 """
3568 self.object_format = object_format
3569 self.cs = object_format.new_hash()
3570 self.entries: dict[bytes, tuple[int, int]] = {}
3571 if records is None:
3572 records = iter([]) # Empty iterator if None
3573 self._it = self._pack_data_chunks(
3574 records=records,
3575 num_records=num_records,
3576 progress=progress,
3577 compression_level=compression_level,
3578 reuse_compressed=reuse_compressed,
3579 )
3581 def sha1digest(self) -> bytes:
3582 """Return the SHA1 digest of the pack data."""
3583 return self.cs.digest()
3585 def __iter__(self) -> Iterator[bytes]:
3586 """Iterate over pack data chunks."""
3587 return self._it
3589 def _pack_data_chunks(
3590 self,
3591 records: Iterator[UnpackedObject],
3592 *,
3593 num_records: int | None = None,
3594 progress: Callable[..., None] | None = None,
3595 compression_level: int = -1,
3596 reuse_compressed: bool = True,
3597 ) -> Iterator[bytes]:
3598 """Iterate pack data file chunks.
3600 Args:
3601 records: Iterator over UnpackedObject
3602 num_records: Number of records (defaults to len(records) if not specified)
3603 progress: Function to report progress to
3604 compression_level: the zlib compression level
3605 reuse_compressed: Whether to reuse compressed chunks
3606 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3607 """
3608 # Write the pack
3609 if num_records is None:
3610 num_records = len(records) # type: ignore
3611 offset = 0
3612 for chunk in pack_header_chunks(num_records):
3613 yield chunk
3614 self.cs.update(chunk)
3615 offset += len(chunk)
3616 actual_num_records = 0
3617 for i, unpacked in enumerate(records):
3618 type_num = unpacked.pack_type_num
3619 if progress is not None and i % 1000 == 0:
3620 progress((f"writing pack data: {i}/{num_records}\r").encode("ascii"))
3621 raw: list[bytes] | tuple[int, list[bytes]] | tuple[bytes, list[bytes]]
3622 if unpacked.delta_base is not None:
3623 assert isinstance(unpacked.delta_base, bytes), (
3624 f"Expected bytes, got {type(unpacked.delta_base)}"
3625 )
3626 try:
3627 base_offset, _base_crc32 = self.entries[unpacked.delta_base]
3628 except KeyError:
3629 type_num = REF_DELTA
3630 assert isinstance(unpacked.delta_base, bytes)
3631 raw = (unpacked.delta_base, unpacked.decomp_chunks)
3632 else:
3633 type_num = OFS_DELTA
3634 raw = (offset - base_offset, unpacked.decomp_chunks)
3635 else:
3636 raw = unpacked.decomp_chunks
3637 chunks: list[bytes] | Iterator[bytes]
3638 if unpacked.comp_chunks is not None and reuse_compressed:
3639 chunks = unpacked.comp_chunks
3640 else:
3641 chunks = pack_object_chunks(
3642 type_num,
3643 raw,
3644 compression_level=compression_level,
3645 object_format=self.object_format,
3646 )
3647 crc32 = 0
3648 object_size = 0
3649 for chunk in chunks:
3650 yield chunk
3651 crc32 = binascii.crc32(chunk, crc32)
3652 self.cs.update(chunk)
3653 object_size += len(chunk)
3654 actual_num_records += 1
3655 self.entries[unpacked.sha()] = (offset, crc32)
3656 offset += object_size
3657 if actual_num_records != num_records:
3658 raise AssertionError(
3659 f"actual records written differs: {actual_num_records} != {num_records}"
3660 )
3662 yield self.cs.digest()
3665def write_pack_data(
3666 write: Callable[[bytes], None]
3667 | Callable[[bytes | bytearray | memoryview], int]
3668 | IO[bytes],
3669 records: Iterator[UnpackedObject],
3670 object_format: "ObjectFormat",
3671 *,
3672 num_records: int | None = None,
3673 progress: Callable[..., None] | None = None,
3674 compression_level: int = -1,
3675) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3676 """Write a new pack data file.
3678 Args:
3679 write: Write function to use
3680 num_records: Number of records (defaults to len(records) if None)
3681 records: Iterator over type_num, object_id, delta_base, raw
3682 object_format: Object format (hash algorithm) to use
3683 progress: Function to report progress to
3684 compression_level: the zlib compression level
3685 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3686 """
3687 chunk_generator = PackChunkGenerator(
3688 num_records=num_records,
3689 records=records,
3690 progress=progress,
3691 compression_level=compression_level,
3692 object_format=object_format,
3693 )
3694 for chunk in chunk_generator:
3695 if callable(write):
3696 write(chunk)
3697 else:
3698 write.write(chunk)
3699 return chunk_generator.entries, chunk_generator.sha1digest()
3702def write_pack_index_v1(
3703 f: IO[bytes],
3704 entries: Iterable[tuple[bytes, int, int | None]],
3705 pack_checksum: bytes,
3706) -> bytes:
3707 """Write a new pack index file.
3709 Args:
3710 f: A file-like object to write to
3711 entries: List of tuples with object name (sha), offset_in_pack,
3712 and crc32_checksum.
3713 pack_checksum: Checksum of the pack file.
3714 Returns: The SHA of the written index file
3715 """
3716 f = SHA1Writer(f)
3717 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
3718 for name, _offset, _entry_checksum in entries:
3719 fan_out_table[ord(name[:1])] += 1
3720 # Fan-out table
3721 for i in range(0x100):
3722 f.write(struct.pack(">L", fan_out_table[i]))
3723 fan_out_table[i + 1] += fan_out_table[i]
3724 for name, offset, _entry_checksum in entries:
3725 if len(name) != 20:
3726 raise TypeError("pack index v1 only supports SHA-1 names")
3727 if not (offset <= 0xFFFFFFFF):
3728 raise TypeError("pack format 1 only supports offsets < 2Gb")
3729 f.write(struct.pack(">L20s", offset, name))
3730 assert len(pack_checksum) == 20
3731 f.write(pack_checksum)
3732 return f.write_sha()
3735def _delta_encode_size(size: int) -> bytes:
3736 ret = bytearray()
3737 c = size & 0x7F
3738 size >>= 7
3739 while size:
3740 ret.append(c | 0x80)
3741 c = size & 0x7F
3742 size >>= 7
3743 ret.append(c)
3744 return bytes(ret)
3747# The length of delta compression copy operations in version 2 packs is limited
3748# to 64K. To copy more, we use several copy operations. Version 3 packs allow
3749# 24-bit lengths in copy operations, but we always make version 2 packs.
3750_MAX_COPY_LEN = 0xFFFF
3753def _encode_copy_operation(start: int, length: int) -> bytes:
3754 scratch = bytearray([0x80])
3755 for i in range(4):
3756 if start & 0xFF << i * 8:
3757 scratch.append((start >> i * 8) & 0xFF)
3758 scratch[0] |= 1 << i
3759 for i in range(2):
3760 if length & 0xFF << i * 8:
3761 scratch.append((length >> i * 8) & 0xFF)
3762 scratch[0] |= 1 << (4 + i)
3763 return bytes(scratch)
3766def _create_delta_py(
3767 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
3768) -> Iterator[bytes]:
3769 """Use python difflib to work out how to transform base_buf to target_buf.
3771 Args:
3772 base_buf: Base buffer
3773 target_buf: Target buffer
3774 """
3775 if isinstance(base_buf, list):
3776 base_buf = b"".join(base_buf)
3777 if isinstance(target_buf, list):
3778 target_buf = b"".join(target_buf)
3779 # write delta header
3780 yield _delta_encode_size(len(base_buf))
3781 yield _delta_encode_size(len(target_buf))
3782 # write out delta opcodes
3783 seq = SequenceMatcher(isjunk=None, a=base_buf, b=target_buf)
3784 for opcode, i1, i2, j1, j2 in seq.get_opcodes():
3785 # Git patch opcodes don't care about deletes!
3786 # if opcode == 'replace' or opcode == 'delete':
3787 # pass
3788 if opcode == "equal":
3789 # If they are equal, unpacker will use data from base_buf
3790 # Write out an opcode that says what range to use
3791 copy_start = i1
3792 copy_len = i2 - i1
3793 while copy_len > 0:
3794 to_copy = min(copy_len, _MAX_COPY_LEN)
3795 yield _encode_copy_operation(copy_start, to_copy)
3796 copy_start += to_copy
3797 copy_len -= to_copy
3798 if opcode == "replace" or opcode == "insert":
3799 # If we are replacing a range or adding one, then we just
3800 # output it to the stream (prefixed by its size)
3801 s = j2 - j1
3802 o = j1
3803 while s > 127:
3804 yield bytes([127])
3805 yield bytes(memoryview(target_buf)[o : o + 127])
3806 s -= 127
3807 o += 127
3808 yield bytes([s])
3809 yield bytes(memoryview(target_buf)[o : o + s])
3812# Default to pure Python implementation
3813create_delta = _create_delta_py
3816def apply_delta(
3817 src_buf: bytes | list[bytes], delta: bytes | list[bytes]
3818) -> list[bytes]:
3819 """Based on the similar function in git's patch-delta.c.
3821 Args:
3822 src_buf: Source buffer
3823 delta: Delta instructions
3824 """
3825 if not isinstance(src_buf, bytes):
3826 src_buf = b"".join(src_buf)
3827 if not isinstance(delta, bytes):
3828 delta = b"".join(delta)
3829 out = []
3830 index = 0
3831 delta_length = len(delta)
3833 def get_delta_header_size(delta: bytes, index: int) -> tuple[int, int]:
3834 size = 0
3835 i = 0
3836 while True:
3837 # Bound-check explicitly: ``delta[index:index+1]`` silently
3838 # returns b"" past the end, which would crash with TypeError
3839 # in ``ord`` and leave the caller unable to distinguish a
3840 # truncated delta from a programming bug.
3841 if index >= delta_length:
3842 raise ApplyDeltaError("delta truncated in size header")
3843 cmd = ord(delta[index : index + 1])
3844 index += 1
3845 size |= (cmd & ~0x80) << i
3846 i += 7
3847 if not cmd & 0x80:
3848 break
3849 return size, index
3851 def read_byte(delta: bytes) -> int:
3852 nonlocal index
3853 # Bound-check explicitly: ``delta[index:index+1]`` silently returns
3854 # b"" past the end, which would crash with TypeError in ``ord`` and
3855 # leave the caller unable to distinguish a truncated delta from a
3856 # programming bug.
3857 if index >= delta_length:
3858 raise ApplyDeltaError("delta truncated in copy op")
3859 index += 1
3860 return ord(delta[index - 1 : index])
3862 src_size, index = get_delta_header_size(delta, index)
3863 dest_size, index = get_delta_header_size(delta, index)
3864 if src_size != len(src_buf):
3865 raise ApplyDeltaError(
3866 f"Unexpected source buffer size: {src_size} vs {len(src_buf)}"
3867 )
3868 while index < delta_length:
3869 cmd = ord(delta[index : index + 1])
3870 index += 1
3871 if cmd & 0x80:
3872 cp_off = 0
3873 for i in range(4):
3874 if cmd & (1 << i):
3875 x = read_byte(delta)
3876 cp_off |= x << (i * 8)
3877 cp_size = 0
3878 # Version 3 packs can contain copy sizes larger than 64K.
3879 for i in range(3):
3880 if cmd & (1 << (4 + i)):
3881 x = read_byte(delta)
3882 cp_size |= x << (i * 8)
3883 if cp_size == 0:
3884 cp_size = 0x10000
3885 if (
3886 cp_off + cp_size < cp_size
3887 or cp_off + cp_size > src_size
3888 or cp_size > dest_size
3889 ):
3890 break
3891 out.append(src_buf[cp_off : cp_off + cp_size])
3892 elif cmd != 0:
3893 if index + cmd > delta_length:
3894 raise ApplyDeltaError("delta truncated in insert op")
3895 out.append(delta[index : index + cmd])
3896 index += cmd
3897 else:
3898 raise ApplyDeltaError("Invalid opcode 0")
3900 if index != delta_length:
3901 raise ApplyDeltaError(f"delta not empty: {delta[index:]!r}")
3903 if dest_size != chunks_length(out):
3904 raise ApplyDeltaError("dest size incorrect")
3906 return out
3909def write_pack_index_v2(
3910 f: IO[bytes],
3911 entries: Iterable[tuple[bytes, int, int | None]],
3912 pack_checksum: bytes,
3913) -> bytes:
3914 """Write a new pack index file.
3916 Args:
3917 f: File-like object to write to
3918 entries: List of tuples with object name (sha), offset_in_pack, and
3919 crc32_checksum.
3920 pack_checksum: Checksum of the pack file.
3921 Returns: The checksum of the index file written
3922 """
3923 # Determine hash algorithm from pack_checksum length
3924 if len(pack_checksum) == 20:
3925 hash_func = sha1
3926 elif len(pack_checksum) == 32:
3927 hash_func = sha256
3928 else:
3929 raise ValueError(f"Unsupported pack checksum length: {len(pack_checksum)}")
3931 f_writer = HashWriter(f, hash_func)
3932 f_writer.write(b"\377tOc") # Magic!
3933 f_writer.write(struct.pack(">L", 2))
3935 # Convert to list to allow multiple iterations
3936 entries_list = list(entries)
3938 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
3939 for name, offset, entry_checksum in entries_list:
3940 fan_out_table[ord(name[:1])] += 1
3942 if entries_list:
3943 hash_size = len(entries_list[0][0])
3944 else:
3945 hash_size = len(pack_checksum) # Use pack_checksum length as hash size
3947 # Fan-out table
3948 largetable: list[int] = []
3949 for i in range(0x100):
3950 f_writer.write(struct.pack(b">L", fan_out_table[i]))
3951 fan_out_table[i + 1] += fan_out_table[i]
3952 for name, offset, entry_checksum in entries_list:
3953 if len(name) != hash_size:
3954 raise TypeError(
3955 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
3956 )
3957 f_writer.write(name)
3958 for name, offset, entry_checksum in entries_list:
3959 f_writer.write(struct.pack(b">L", entry_checksum))
3960 for name, offset, entry_checksum in entries_list:
3961 if offset < 2**31:
3962 f_writer.write(struct.pack(b">L", offset))
3963 else:
3964 f_writer.write(struct.pack(b">L", 2**31 + len(largetable)))
3965 largetable.append(offset)
3966 for offset in largetable:
3967 f_writer.write(struct.pack(b">Q", offset))
3968 f_writer.write(pack_checksum)
3969 return f_writer.write_hash()
3972def write_pack_index_v3(
3973 f: IO[bytes],
3974 entries: Iterable[tuple[bytes, int, int | None]],
3975 pack_checksum: bytes,
3976 hash_format: int = 1,
3977) -> bytes:
3978 """Write a new pack index file in v3 format.
3980 Args:
3981 f: File-like object to write to
3982 entries: List of tuples with object name (sha), offset_in_pack, and
3983 crc32_checksum.
3984 pack_checksum: Checksum of the pack file.
3985 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
3986 Returns: The SHA of the index file written
3987 """
3988 if hash_format == 1:
3989 hash_size = 20 # SHA-1
3990 writer_cls = SHA1Writer
3991 elif hash_format == 2:
3992 hash_size = 32 # SHA-256
3993 # TODO: Add SHA256Writer when SHA-256 support is implemented
3994 raise NotImplementedError("SHA-256 support not yet implemented")
3995 else:
3996 raise ValueError(f"Unknown hash algorithm {hash_format}")
3998 # Convert entries to list to allow multiple iterations
3999 entries_list = list(entries)
4001 # Calculate shortest unambiguous prefix length for object names
4002 # For now, use full hash size (this could be optimized)
4003 shortened_oid_len = hash_size
4005 f = writer_cls(f)
4006 f.write(b"\377tOc") # Magic!
4007 f.write(struct.pack(">L", 3)) # Version 3
4008 f.write(struct.pack(">L", hash_format)) # Hash algorithm
4009 f.write(struct.pack(">L", shortened_oid_len)) # Shortened OID length
4011 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4012 for name, offset, entry_checksum in entries_list:
4013 if len(name) != hash_size:
4014 raise ValueError(
4015 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4016 )
4017 fan_out_table[ord(name[:1])] += 1
4019 # Fan-out table
4020 largetable: list[int] = []
4021 for i in range(0x100):
4022 f.write(struct.pack(b">L", fan_out_table[i]))
4023 fan_out_table[i + 1] += fan_out_table[i]
4025 # Object names table
4026 for name, offset, entry_checksum in entries_list:
4027 f.write(name)
4029 # CRC32 checksums table
4030 for name, offset, entry_checksum in entries_list:
4031 f.write(struct.pack(b">L", entry_checksum))
4033 # Offset table
4034 for name, offset, entry_checksum in entries_list:
4035 if offset < 2**31:
4036 f.write(struct.pack(b">L", offset))
4037 else:
4038 f.write(struct.pack(b">L", 2**31 + len(largetable)))
4039 largetable.append(offset)
4041 # Large offset table
4042 for offset in largetable:
4043 f.write(struct.pack(b">Q", offset))
4045 assert len(pack_checksum) == hash_size, (
4046 f"Pack checksum has wrong length: expected {hash_size}, got {len(pack_checksum)}"
4047 )
4048 f.write(pack_checksum)
4049 return f.write_sha()
4052def write_pack_index(
4053 f: IO[bytes],
4054 entries: Iterable[tuple[bytes, int, int | None]],
4055 pack_checksum: bytes,
4056 progress: Callable[..., None] | None = None,
4057 version: int | None = None,
4058) -> bytes:
4059 """Write a pack index file.
4061 Args:
4062 f: File-like object to write to.
4063 entries: List of (checksum, offset, crc32) tuples
4064 pack_checksum: Checksum of the pack file.
4065 progress: Progress function (not currently used)
4066 version: Pack index version to use (1, 2, or 3). If None, defaults to DEFAULT_PACK_INDEX_VERSION.
4068 Returns:
4069 SHA of the written index file
4071 Raises:
4072 ValueError: If an unsupported version is specified
4073 """
4074 if version is None:
4075 version = DEFAULT_PACK_INDEX_VERSION
4077 if version == 1:
4078 return write_pack_index_v1(f, entries, pack_checksum)
4079 elif version == 2:
4080 return write_pack_index_v2(f, entries, pack_checksum)
4081 elif version == 3:
4082 return write_pack_index_v3(f, entries, pack_checksum)
4083 else:
4084 raise ValueError(f"Unsupported pack index version: {version}")
4087class Pack:
4088 """A Git pack object."""
4090 _data_load: Callable[[], PackData] | None
4091 _idx_load: Callable[[], PackIndex] | None
4093 _data: PackData | None
4094 _idx: PackIndex | None
4095 _bitmap: "PackBitmap | None"
4097 def __init__(
4098 self,
4099 basename: str,
4100 *,
4101 object_format: ObjectFormat,
4102 resolve_ext_ref: ResolveExtRefFn | None = None,
4103 delta_window_size: int | None = None,
4104 window_memory: int | None = None,
4105 delta_cache_size: int | None = None,
4106 depth: int | None = None,
4107 threads: int | None = None,
4108 big_file_threshold: int | None = None,
4109 delta_base_cache_limit: int | None = None,
4110 ) -> None:
4111 """Initialize a Pack object.
4113 Args:
4114 basename: Base path for pack files (without .pack/.idx extension)
4115 object_format: Hash algorithm used by the repository
4116 resolve_ext_ref: Optional function to resolve external references
4117 delta_window_size: Size of the delta compression window
4118 window_memory: Memory limit for delta compression window
4119 delta_cache_size: Size of the delta cache
4120 depth: Maximum depth for delta chains
4121 threads: Number of threads to use for operations
4122 big_file_threshold: Size threshold for big file handling
4123 delta_base_cache_limit: Maximum bytes for delta base object cache
4124 """
4125 self._basename = basename
4126 self.object_format = object_format
4127 self._data = None
4128 self._idx = None
4129 self._bitmap = None
4130 self._idx_path = self._basename + ".idx"
4131 self._data_path = self._basename + ".pack"
4132 self._bitmap_path = self._basename + ".bitmap"
4133 self.delta_window_size = delta_window_size
4134 self.window_memory = window_memory
4135 self.delta_cache_size = delta_cache_size
4136 self.depth = depth
4137 self.threads = threads
4138 self.big_file_threshold = big_file_threshold
4139 self.delta_base_cache_limit = delta_base_cache_limit
4140 self._idx_load = lambda: load_pack_index(self._idx_path, object_format)
4141 self._data_load = lambda: PackData(
4142 self._data_path,
4143 delta_window_size=delta_window_size,
4144 window_memory=window_memory,
4145 delta_cache_size=delta_cache_size,
4146 depth=depth,
4147 threads=threads,
4148 big_file_threshold=big_file_threshold,
4149 delta_base_cache_limit=delta_base_cache_limit,
4150 object_format=object_format,
4151 )
4152 self.resolve_ext_ref = resolve_ext_ref
4154 @classmethod
4155 def from_lazy_objects(
4156 cls,
4157 data_fn: Callable[[], PackData],
4158 idx_fn: Callable[[], PackIndex],
4159 ) -> "Pack":
4160 """Create a new pack object from callables to load pack data and index objects."""
4161 # Load index to get object format
4162 idx = idx_fn()
4163 ret = cls("", object_format=idx.object_format)
4164 ret._data_load = data_fn
4165 ret._idx = idx
4166 ret._idx_load = None
4167 return ret
4169 @classmethod
4170 def from_objects(cls, data: PackData, idx: PackIndex) -> "Pack":
4171 """Create a new pack object from pack data and index objects."""
4172 ret = cls("", object_format=idx.object_format)
4173 ret._data = data
4174 ret._data_load = None
4175 ret._idx = idx
4176 ret._idx_load = None
4177 ret.check_length_and_checksum()
4178 return ret
4180 def name(self) -> bytes:
4181 """The SHA over the SHAs of the objects in this pack."""
4182 return self.index.objects_sha1()
4184 @property
4185 def data(self) -> PackData:
4186 """The pack data object being used."""
4187 if self._data is None:
4188 assert self._data_load
4189 try:
4190 self._data = self._data_load()
4191 except FileNotFoundError as exc:
4192 raise PackFileDisappeared(self) from exc
4193 self.check_length_and_checksum()
4194 return self._data
4196 @property
4197 def index(self) -> PackIndex:
4198 """The index being used.
4200 Note: This may be an in-memory index
4201 """
4202 if self._idx is None:
4203 assert self._idx_load
4204 try:
4205 self._idx = self._idx_load()
4206 except FileNotFoundError as exc:
4207 raise PackFileDisappeared(self) from exc
4208 return self._idx
4210 @property
4211 def bitmap(self) -> "PackBitmap | None":
4212 """The bitmap being used, if available.
4214 Returns:
4215 PackBitmap instance, or None if no bitmap exists or the bitmap
4216 was built for a different pack
4218 Raises:
4219 ValueError: If bitmap file is invalid or corrupt
4220 """
4221 if self._bitmap is None:
4222 from .bitmap import read_bitmap
4224 try:
4225 self._bitmap = read_bitmap(
4226 self._bitmap_path,
4227 pack_index=self.index,
4228 pack_checksum=self.get_stored_checksum(),
4229 )
4230 except ChecksumMismatch:
4231 # The bitmap records the checksum of the pack it was built for.
4232 # A mismatch means it is stale or was swapped in from another
4233 # pack, so its positions no longer describe this pack's objects.
4234 # Ignore it and let callers fall back to graph traversal, the
4235 # same as git.
4236 logger.warning(
4237 "Ignoring bitmap %s: checksum does not match pack",
4238 self._bitmap_path,
4239 )
4240 return None
4241 return self._bitmap
4243 def ensure_bitmap(
4244 self,
4245 object_store: "BaseObjectStore",
4246 refs: dict["Ref", "ObjectID"],
4247 commit_interval: int | None = None,
4248 progress: Callable[[str], None] | None = None,
4249 ) -> "PackBitmap":
4250 """Ensure a bitmap exists for this pack, generating one if needed.
4252 Args:
4253 object_store: Object store to read objects from
4254 refs: Dictionary of ref names to commit SHAs
4255 commit_interval: Include every Nth commit in bitmap index
4256 progress: Optional progress reporting callback
4258 Returns:
4259 PackBitmap instance (either existing or newly generated)
4260 """
4261 from .bitmap import generate_bitmap, write_bitmap
4263 # Check if bitmap already exists
4264 try:
4265 existing = self.bitmap
4266 if existing is not None:
4267 return existing
4268 except FileNotFoundError:
4269 pass # No bitmap, we'll generate one
4271 # Generate new bitmap
4272 if progress:
4273 progress(f"Generating bitmap for {self.name().decode('utf-8')}...\n")
4275 pack_bitmap = generate_bitmap(
4276 self.index,
4277 object_store,
4278 refs,
4279 self.get_stored_checksum(),
4280 commit_interval=commit_interval,
4281 progress=progress,
4282 )
4284 # Write bitmap file
4285 write_bitmap(self._bitmap_path, pack_bitmap)
4287 if progress:
4288 progress(f"Wrote {self._bitmap_path}\n")
4290 # Update cached bitmap
4291 self._bitmap = pack_bitmap
4293 return pack_bitmap
4295 @property
4296 def mmap_size(self) -> int:
4297 """Return the total mmapped memory usage of this pack.
4299 This includes the pack data file and index file sizes,
4300 but only for components that have been loaded (and thus mmapped).
4301 """
4302 total = 0
4303 if self._data is not None:
4304 total += self._data._get_size()
4305 if self._idx is not None and isinstance(self._idx, FilePackIndex):
4306 total += self._idx._size
4307 return total
4309 def close(self) -> None:
4310 """Close the pack file and index."""
4311 if self._data is not None:
4312 self._data.close()
4313 self._data = None
4314 if self._idx is not None:
4315 self._idx.close()
4316 self._idx = None
4318 def __del__(self) -> None:
4319 """Ensure pack file is closed when Pack is garbage collected."""
4320 if self._data is not None or self._idx is not None:
4321 import warnings
4323 warnings.warn(
4324 f"unclosed Pack {self!r}", ResourceWarning, stacklevel=2, source=self
4325 )
4326 try:
4327 self.close()
4328 except Exception:
4329 # Ignore errors during cleanup
4330 pass
4332 def __enter__(self) -> Self:
4333 """Enter context manager."""
4334 return self
4336 def __exit__(
4337 self,
4338 type: type | None,
4339 value: BaseException | None,
4340 traceback: TracebackType | None,
4341 ) -> None:
4342 """Exit context manager."""
4343 self.close()
4345 def __eq__(self, other: object) -> bool:
4346 """Check equality with another pack."""
4347 if not isinstance(other, Pack):
4348 return False
4349 return self.index == other.index
4351 def __len__(self) -> int:
4352 """Number of entries in this pack."""
4353 return len(self.index)
4355 def __repr__(self) -> str:
4356 """Return string representation of this pack."""
4357 return f"{self.__class__.__name__}({self._basename!r})"
4359 def __iter__(self) -> Iterator[ObjectID]:
4360 """Iterate over all the sha1s of the objects in this pack."""
4361 return iter(self.index)
4363 def check_length_and_checksum(self) -> None:
4364 """Sanity check the length and checksum of the pack index and data."""
4365 assert len(self.index) == len(self.data), (
4366 f"Length mismatch: {len(self.index)} (index) != {len(self.data)} (data)"
4367 )
4368 idx_stored_checksum = self.index.get_pack_checksum()
4369 data_stored_checksum = self.data.get_stored_checksum()
4370 if (
4371 idx_stored_checksum is not None
4372 and idx_stored_checksum != data_stored_checksum
4373 ):
4374 raise ChecksumMismatch(
4375 sha_to_hex(RawObjectID(idx_stored_checksum)),
4376 sha_to_hex(RawObjectID(data_stored_checksum)),
4377 )
4379 def check(self) -> None:
4380 """Check the integrity of this pack.
4382 Raises:
4383 ChecksumMismatch: if a checksum for the index or data is wrong
4384 """
4385 self.index.check()
4386 self.data.check()
4387 for obj in self.iterobjects():
4388 obj.check()
4389 # TODO: object connectivity checks
4391 def get_stored_checksum(self) -> bytes:
4392 """Return the stored checksum of the pack data."""
4393 return self.data.get_stored_checksum()
4395 def pack_tuples(self) -> list[tuple[ShaFile, None]]:
4396 """Return pack tuples for all objects in pack."""
4397 return [(o, None) for o in self.iterobjects()]
4399 def __contains__(self, sha1: ObjectID | RawObjectID) -> bool:
4400 """Check whether this pack contains a particular SHA1."""
4401 try:
4402 self.index.object_offset(sha1)
4403 return True
4404 except KeyError:
4405 return False
4407 def get_raw(self, sha1: RawObjectID | ObjectID) -> tuple[int, bytes]:
4408 """Get raw object data by SHA1."""
4409 offset = self.index.object_offset(sha1)
4410 obj_type, obj = self.data.get_object_at(offset)
4411 type_num, chunks = self.resolve_object(offset, obj_type, obj)
4412 return type_num, b"".join(chunks) # type: ignore[arg-type]
4414 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
4415 """Retrieve the specified SHA1."""
4416 type, uncomp = self.get_raw(sha1)
4417 return ShaFile.from_raw_string(type, uncomp, sha=sha1)
4419 def iterobjects(self) -> Iterator[ShaFile]:
4420 """Iterate over the objects in this pack."""
4421 return iter(
4422 PackInflater.for_pack_data(self.data, resolve_ext_ref=self.resolve_ext_ref)
4423 )
4425 def iterobjects_subset(
4426 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
4427 ) -> Iterator[ShaFile]:
4428 """Iterate over a subset of objects in this pack."""
4429 return (
4430 uo
4431 for uo in PackInflater.for_pack_subset(
4432 self,
4433 shas,
4434 allow_missing=allow_missing,
4435 resolve_ext_ref=self.resolve_ext_ref,
4436 )
4437 if uo.id in shas
4438 )
4440 def iter_unpacked_subset(
4441 self,
4442 shas: Iterable[ObjectID | RawObjectID],
4443 *,
4444 include_comp: bool = False,
4445 allow_missing: bool = False,
4446 convert_ofs_delta: bool = False,
4447 ) -> Iterator[UnpackedObject]:
4448 """Iterate over unpacked objects in subset."""
4449 ofs_pending: dict[int, list[UnpackedObject]] = defaultdict(list)
4450 ofs: dict[int, bytes] = {}
4451 todo: set[ObjectID | RawObjectID] = set(shas)
4452 for unpacked in self.iter_unpacked(include_comp=include_comp):
4453 sha = unpacked.sha()
4454 if unpacked.offset is not None:
4455 ofs[unpacked.offset] = sha
4456 hexsha = sha_to_hex(RawObjectID(sha))
4457 if hexsha in todo:
4458 if unpacked.pack_type_num == OFS_DELTA:
4459 assert isinstance(unpacked.delta_base, int)
4460 assert unpacked.offset is not None
4461 base_offset = unpacked.offset - unpacked.delta_base
4462 try:
4463 unpacked.delta_base = ofs[base_offset]
4464 except KeyError:
4465 ofs_pending[base_offset].append(unpacked)
4466 continue
4467 else:
4468 unpacked.pack_type_num = REF_DELTA
4469 yield unpacked
4470 todo.remove(hexsha)
4471 if unpacked.offset is not None:
4472 for child in ofs_pending.pop(unpacked.offset, []):
4473 child.pack_type_num = REF_DELTA
4474 child.delta_base = sha
4475 yield child
4476 assert not ofs_pending
4477 if not allow_missing and todo:
4478 raise UnresolvedDeltas(list(todo))
4480 def iter_unpacked(self, include_comp: bool = False) -> Iterator[UnpackedObject]:
4481 """Iterate over all unpacked objects in this pack."""
4482 ofs_to_entries = {
4483 ofs: (sha, crc32) for (sha, ofs, crc32) in self.index.iterentries()
4484 }
4485 for unpacked in self.data.iter_unpacked(include_comp=include_comp):
4486 assert unpacked.offset is not None
4487 (sha, crc32) = ofs_to_entries[unpacked.offset]
4488 unpacked._sha = sha
4489 unpacked.crc32 = crc32
4490 yield unpacked
4492 def keep(self, msg: bytes | None = None) -> str:
4493 """Add a .keep file for the pack, preventing git from garbage collecting it.
4495 Args:
4496 msg: A message written inside the .keep file; can be used later
4497 to determine whether or not a .keep file is obsolete.
4498 Returns: The path of the .keep file, as a string.
4499 """
4500 keepfile_name = f"{self._basename}.keep"
4501 with GitFile(keepfile_name, "wb") as keepfile:
4502 if msg:
4503 keepfile.write(msg)
4504 keepfile.write(b"\n")
4505 return keepfile_name
4507 def get_ref(
4508 self, sha: RawObjectID | ObjectID
4509 ) -> tuple[int | None, int, OldUnpackedObject]:
4510 """Get the object for a ref SHA, only looking in this pack."""
4511 # TODO: cache these results
4512 try:
4513 offset = self.index.object_offset(sha)
4514 except KeyError:
4515 offset = None
4516 if offset:
4517 type, obj = self.data.get_object_at(offset)
4518 elif self.resolve_ext_ref:
4519 type, obj = self.resolve_ext_ref(sha)
4520 else:
4521 raise KeyError(sha)
4522 return offset, type, obj
4524 def resolve_object(
4525 self,
4526 offset: int,
4527 type: int,
4528 obj: OldUnpackedObject,
4529 get_ref: Callable[
4530 [RawObjectID | ObjectID], tuple[int | None, int, OldUnpackedObject]
4531 ]
4532 | None = None,
4533 ) -> tuple[int, OldUnpackedObject]:
4534 """Resolve an object, possibly resolving deltas when necessary.
4536 Returns: Tuple with object type and contents.
4537 """
4538 # Walk down the delta chain, building a stack of deltas to reach
4539 # the requested object.
4540 base_offset: int | None = offset
4541 base_type = type
4542 base_obj = obj
4543 delta_stack = []
4544 while base_type in DELTA_TYPES:
4545 prev_offset = base_offset
4546 if get_ref is None:
4547 get_ref = self.get_ref
4548 assert isinstance(base_obj, tuple), (
4549 f"Expected delta tuple, got {base_obj.__class__.__name__}"
4550 )
4551 if base_type == OFS_DELTA:
4552 (delta_offset, delta) = base_obj
4553 # TODO: clean up asserts and replace with nicer error messages
4554 assert isinstance(delta_offset, int), (
4555 f"Expected int, got {delta_offset.__class__}"
4556 )
4557 assert base_offset is not None
4558 base_offset = base_offset - delta_offset
4559 base_type, base_obj = self.data.get_object_at(base_offset)
4560 assert isinstance(base_type, int)
4561 elif base_type == REF_DELTA:
4562 (basename, delta) = base_obj
4563 assert (
4564 isinstance(basename, bytes)
4565 and len(basename) == self.object_format.oid_length
4566 )
4567 base_offset_temp, base_type, base_obj = get_ref(RawObjectID(basename))
4568 assert isinstance(base_type, int)
4569 # base_offset_temp can be None for thin packs (external references)
4570 base_offset = base_offset_temp
4571 if base_offset == prev_offset: # object is based on itself
4572 raise UnresolvedDeltas([basename])
4573 else:
4574 raise AssertionError(f"Unexpected delta type: {base_type}")
4575 delta_stack.append((prev_offset, base_type, delta))
4577 # Now grab the base object (mustn't be a delta) and apply the
4578 # deltas all the way up the stack.
4579 chunks = base_obj
4580 for prev_offset, _delta_type, delta in reversed(delta_stack):
4581 # Convert chunks to bytes for apply_delta if needed
4582 if isinstance(chunks, list):
4583 chunks_bytes = b"".join(chunks)
4584 elif isinstance(chunks, tuple):
4585 # For tuple type, second element is the actual data
4586 _, chunk_data = chunks
4587 if isinstance(chunk_data, list):
4588 chunks_bytes = b"".join(chunk_data)
4589 else:
4590 chunks_bytes = chunk_data
4591 else:
4592 chunks_bytes = chunks
4594 # Apply delta and get result as list
4595 chunks = apply_delta(chunks_bytes, delta)
4597 if prev_offset is not None:
4598 self.data._offset_cache[prev_offset] = base_type, chunks
4599 return base_type, chunks
4601 def entries(
4602 self, progress: Callable[[int, int], None] | None = None
4603 ) -> Iterator[PackIndexEntry]:
4604 """Yield entries summarizing the contents of this pack.
4606 Args:
4607 progress: Progress function, called with current and total
4608 object count.
4609 Returns: iterator of tuples with (sha, offset, crc32)
4610 """
4611 return self.data.iterentries(
4612 progress=progress, resolve_ext_ref=self.resolve_ext_ref
4613 )
4615 def sorted_entries(
4616 self, progress: Callable[[int, int], None] | None = None
4617 ) -> Iterator[PackIndexEntry]:
4618 """Return entries in this pack, sorted by SHA.
4620 Args:
4621 progress: Progress function, called with current and total
4622 object count
4623 Returns: Iterator of tuples with (sha, offset, crc32)
4624 """
4625 return iter(
4626 self.data.sorted_entries(
4627 progress=progress, resolve_ext_ref=self.resolve_ext_ref
4628 )
4629 )
4631 def get_unpacked_object(
4632 self,
4633 sha: ObjectID | RawObjectID,
4634 *,
4635 include_comp: bool = False,
4636 convert_ofs_delta: bool = True,
4637 ) -> UnpackedObject:
4638 """Get the unpacked object for a sha.
4640 Args:
4641 sha: SHA of object to fetch
4642 include_comp: Whether to include compression data in UnpackedObject
4643 convert_ofs_delta: Whether to convert offset deltas to ref deltas
4644 """
4645 offset = self.index.object_offset(sha)
4646 unpacked = self.data.get_unpacked_object_at(offset, include_comp=include_comp)
4647 if unpacked.pack_type_num == OFS_DELTA and convert_ofs_delta:
4648 assert isinstance(unpacked.delta_base, int)
4649 unpacked.delta_base = self.index.object_sha1(offset - unpacked.delta_base)
4650 unpacked.pack_type_num = REF_DELTA
4651 return unpacked
4654def extend_pack(
4655 f: BinaryIO,
4656 object_ids: Set["RawObjectID"],
4657 get_raw: Callable[["RawObjectID | ObjectID"], tuple[int, bytes]],
4658 object_format: "ObjectFormat",
4659 *,
4660 compression_level: int = -1,
4661 progress: Callable[[bytes], None] | None = None,
4662) -> tuple[bytes, list[tuple[RawObjectID, int, int]]]:
4663 """Extend a pack file with more objects.
4665 The caller should make sure that object_ids does not contain any objects
4666 that are already in the pack
4667 """
4668 # Update the header with the new number of objects.
4669 f.seek(0)
4670 _version, num_objects = read_pack_header(f.read)
4672 if object_ids:
4673 f.seek(0)
4674 write_pack_header(f.write, num_objects + len(object_ids))
4676 # Must flush before reading (http://bugs.python.org/issue3207)
4677 f.flush()
4679 # Rescan the rest of the pack, computing the SHA with the new header.
4680 new_sha = compute_file_sha(
4681 f, hash_func=object_format.hash_func, end_ofs=-object_format.oid_length
4682 )
4684 # Must reposition before writing (http://bugs.python.org/issue3207)
4685 f.seek(0, os.SEEK_CUR)
4687 extra_entries = []
4689 # Complete the pack.
4690 for i, object_id in enumerate(object_ids):
4691 if progress is not None:
4692 progress(
4693 (f"writing extra base objects: {i}/{len(object_ids)}\r").encode("ascii")
4694 )
4695 assert len(object_id) == object_format.oid_length
4696 type_num, data = get_raw(object_id)
4697 offset = f.tell()
4698 crc32 = write_pack_object(
4699 f.write,
4700 type_num,
4701 [data], # Convert bytes to list[bytes]
4702 sha=new_sha,
4703 compression_level=compression_level,
4704 object_format=object_format,
4705 )
4706 extra_entries.append((object_id, offset, crc32))
4707 pack_sha = new_sha.digest()
4708 f.write(pack_sha)
4709 return pack_sha, extra_entries
4712try:
4713 from dulwich._pack import ( # type: ignore
4714 apply_delta,
4715 bisect_find_sha,
4716 )
4717except ImportError:
4718 pass
4720# Try to import the Rust version of create_delta
4721try:
4722 from dulwich._pack import create_delta as _create_delta_rs
4723except ImportError:
4724 pass
4725else:
4726 # Wrap the Rust version to match the Python API (returns bytes instead of Iterator)
4727 def _create_delta_rs_wrapper(
4728 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
4729 ) -> Iterator[bytes]:
4730 """Wrapper for Rust create_delta to match Python API."""
4731 if isinstance(base_buf, list):
4732 base_buf = b"".join(base_buf)
4733 if isinstance(target_buf, list):
4734 target_buf = b"".join(target_buf)
4735 yield _create_delta_rs(base_buf, target_buf)
4737 create_delta = _create_delta_rs_wrapper