Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/pack.py: 22%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# pack.py -- For dealing with packed git objects.
2# Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
3# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
23"""Classes for dealing with packed git objects.
25A pack is a compact representation of a bunch of objects, stored
26using deltas where possible.
28They have two parts, the pack file, which stores the data, and an index
29that tells you where the data is.
31To find an object you look in all of the index files 'til you find a
32match for the object name. You then use the pointer got from this as
33a pointer in to the corresponding packfile.
34"""
36__all__ = [
37 "DEFAULT_PACK_DELTA_WINDOW_SIZE",
38 "DEFAULT_PACK_INDEX_VERSION",
39 "DELTA_TYPES",
40 "OFS_DELTA",
41 "PACK_SPOOL_FILE_MAX_SIZE",
42 "REF_DELTA",
43 "DeltaChainIterator",
44 "FilePackIndex",
45 "MemoryPackIndex",
46 "ObjectContainer",
47 "Pack",
48 "PackChunkGenerator",
49 "PackData",
50 "PackFileDisappeared",
51 "PackHint",
52 "PackIndex",
53 "PackIndex1",
54 "PackIndex2",
55 "PackIndex3",
56 "PackIndexEntry",
57 "PackIndexer",
58 "PackInflater",
59 "PackStreamCopier",
60 "PackStreamReader",
61 "PackedObjectContainer",
62 "SHA1Reader",
63 "SHA1Writer",
64 "UnpackedObject",
65 "UnpackedObjectIterator",
66 "UnpackedObjectStream",
67 "UnresolvedDeltas",
68 "apply_delta",
69 "bisect_find_sha",
70 "chunks_length",
71 "compute_buffer_sha",
72 "compute_file_sha",
73 "deltas_from_sorted_objects",
74 "deltify_pack_objects",
75 "extend_pack",
76 "find_reusable_deltas",
77 "full_unpacked_object",
78 "generate_unpacked_objects",
79 "iter_sha1",
80 "load_pack_index",
81 "load_pack_index_file",
82 "obj_sha",
83 "pack_header_chunks",
84 "pack_object_chunks",
85 "pack_object_header",
86 "pack_objects_to_data",
87 "read_pack_header",
88 "read_pack_header_at",
89 "read_zlib_chunks",
90 "read_zlib_chunks_at",
91 "sort_objects_for_delta",
92 "take_msb_bytes",
93 "take_msb_bytes_at",
94 "unpack_object",
95 "unpack_object_at",
96 "verify_and_read",
97 "write_pack",
98 "write_pack_data",
99 "write_pack_from_container",
100 "write_pack_header",
101 "write_pack_index",
102 "write_pack_object",
103 "write_pack_objects",
104]
106import binascii
107from collections import defaultdict, deque
108from contextlib import suppress
109from io import BytesIO, UnsupportedOperation
111try:
112 from cdifflib import CSequenceMatcher as SequenceMatcher
113except ModuleNotFoundError:
114 from difflib import SequenceMatcher
116import logging
117import os
118import struct
119import sys
120import threading
121import warnings
122import zlib
123from collections.abc import Callable, Iterable, Iterator, Sequence, Set
124from hashlib import sha1, sha256
125from itertools import chain
126from os import SEEK_END
127from struct import unpack_from
128from types import TracebackType
129from typing import (
130 IO,
131 TYPE_CHECKING,
132 Any,
133 BinaryIO,
134 Generic,
135 Protocol,
136 TypeVar,
137)
139if sys.version_info >= (3, 11):
140 from typing import Self
141else:
142 from typing_extensions import Self
144import mmap
146from .errors import ApplyDeltaError, ChecksumMismatch
147from .file import GitFile, _GitFile
148from .lru_cache import LRUSizeCache
149from .object_format import OBJECT_FORMAT_TYPE_NUMS, SHA1, ObjectFormat
150from .objects import (
151 ObjectID,
152 RawObjectID,
153 ShaFile,
154 hex_to_sha,
155 object_header,
156 sha_to_hex,
157)
159if TYPE_CHECKING:
160 from _hashlib import HASH as HashObject
162 from .bitmap import PackBitmap
163 from .commit_graph import CommitGraph
164 from .object_store import BaseObjectStore
165 from .refs import Ref
167logger = logging.getLogger(__name__)
169# Some platforms (e.g. plan9) don't support mmap properly
170has_mmap = sys.platform != "Plan9"
172OFS_DELTA = 6
173REF_DELTA = 7
175DELTA_TYPES = (OFS_DELTA, REF_DELTA)
178DEFAULT_PACK_DELTA_WINDOW_SIZE = 10
180# Keep pack files under 16Mb in memory, otherwise write them out to disk
181PACK_SPOOL_FILE_MAX_SIZE = 16 * 1024 * 1024
183# Default pack index version to use when none is specified
184DEFAULT_PACK_INDEX_VERSION = 2
187OldUnpackedObject = tuple[bytes | int, list[bytes]] | list[bytes] | bytes
188ResolveExtRefFn = Callable[[RawObjectID | ObjectID], tuple[int, bytes | list[bytes]]]
189ProgressFn = Callable[[int, str], None]
190PackHint = tuple[int, bytes | None]
193def verify_and_read(
194 read_func: Callable[[int], bytes],
195 expected_hash: bytes,
196 hash_algo: str,
197 progress: Callable[[bytes], None] | None = None,
198) -> Iterator[bytes]:
199 """Read from stream, verify hash, then yield verified chunks.
201 This function downloads data to a temporary file (in-memory for small files,
202 on-disk for large ones) while computing its hash. Only after the hash is
203 verified to match expected_hash will it yield any data. This prevents
204 corrupted or malicious data from reaching the caller.
206 Args:
207 read_func: Function to read bytes (like file.read or HTTP response reader)
208 expected_hash: Expected hash as hex string bytes (e.g., b'a3b2c1...')
209 hash_algo: Hash algorithm name ('sha1' or 'sha256')
210 progress: Optional progress callback
212 Yields:
213 Chunks of verified data (only after hash verification succeeds)
215 Raises:
216 ValueError: If hash doesn't match or algorithm unsupported
217 """
218 from tempfile import SpooledTemporaryFile
220 from .object_format import OBJECT_FORMATS
222 # Get the hash function for this algorithm
223 obj_format = OBJECT_FORMATS.get(hash_algo)
224 if obj_format is None:
225 raise ValueError(f"Unsupported hash algorithm: {hash_algo}")
227 hasher = obj_format.new_hash()
229 # Download to temporary file (memory or disk) while computing hash
230 with SpooledTemporaryFile(
231 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="dulwich-verify-"
232 ) as temp_file:
233 # Read data, hash it, and write to temp file
234 while True:
235 chunk = read_func(65536) # Read in 64KB chunks
236 if not chunk:
237 break
238 hasher.update(chunk)
239 temp_file.write(chunk)
241 # Verify hash BEFORE yielding any data
242 computed_hash = hasher.hexdigest().encode("ascii")
243 if computed_hash != expected_hash:
244 raise ValueError(
245 f"hash mismatch: expected {expected_hash.decode('ascii')}, "
246 f"got {computed_hash.decode('ascii')}"
247 )
249 # Hash verified! Now read from temp file and yield chunks
250 if progress:
251 progress(b"Hash verified, processing data\n")
253 temp_file.seek(0)
254 while True:
255 chunk = temp_file.read(65536)
256 if not chunk:
257 break
258 yield chunk
261class UnresolvedDeltas(Exception):
262 """Delta objects could not be resolved."""
264 def __init__(self, shas: list[bytes]) -> None:
265 """Initialize UnresolvedDeltas exception.
267 Args:
268 shas: List of SHA hashes for unresolved delta objects
269 """
270 self.shas = shas
273class DeltaCycle(UnresolvedDeltas):
274 """A pack's delta chain references itself and cannot be resolved."""
277class ObjectContainer(Protocol):
278 """Protocol for objects that can contain git objects."""
280 def add_object(self, obj: ShaFile) -> None:
281 """Add a single object to this object store."""
283 def add_objects(
284 self,
285 objects: Sequence[tuple[ShaFile, str | None]],
286 progress: Callable[..., None] | None = None,
287 ) -> "Pack | None":
288 """Add a set of objects to this object store.
290 Args:
291 objects: Iterable over a list of (object, path) tuples
292 progress: Progress callback for object insertion
293 Returns: Optional Pack object of the objects written.
294 """
296 def __contains__(self, sha1: "ObjectID") -> bool:
297 """Check if a hex sha is present."""
298 ...
300 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
301 """Retrieve an object."""
302 ...
304 def get_commit_graph(self) -> "CommitGraph | None":
305 """Get the commit graph for this object store.
307 Returns:
308 CommitGraph object if available, None otherwise
309 """
310 return None
313class PackedObjectContainer(ObjectContainer):
314 """Container for objects packed in a pack file."""
316 def get_unpacked_object(
317 self, sha1: "ObjectID | RawObjectID", *, include_comp: bool = False
318 ) -> "UnpackedObject":
319 """Get a raw unresolved object.
321 Args:
322 sha1: SHA-1 hash of the object
323 include_comp: Whether to include compressed data
325 Returns:
326 UnpackedObject instance
327 """
328 raise NotImplementedError(self.get_unpacked_object)
330 def iterobjects_subset(
331 self, shas: Iterable["ObjectID"], *, allow_missing: bool = False
332 ) -> Iterator[ShaFile]:
333 """Iterate over a subset of objects.
335 Args:
336 shas: Iterable of object SHAs to retrieve
337 allow_missing: If True, skip missing objects
339 Returns:
340 Iterator of ShaFile objects
341 """
342 raise NotImplementedError(self.iterobjects_subset)
344 def iter_unpacked_subset(
345 self,
346 shas: Iterable["ObjectID | RawObjectID"],
347 *,
348 include_comp: bool = False,
349 allow_missing: bool = False,
350 convert_ofs_delta: bool = True,
351 ) -> Iterator["UnpackedObject"]:
352 """Iterate over unpacked objects from a subset of SHAs.
354 Args:
355 shas: Set of object SHAs to retrieve
356 include_comp: Include compressed data if True
357 allow_missing: If True, skip missing objects
358 convert_ofs_delta: If True, convert offset deltas to ref deltas
360 Returns:
361 Iterator of UnpackedObject instances
362 """
363 raise NotImplementedError(self.iter_unpacked_subset)
366class UnpackedObjectStream:
367 """Abstract base class for a stream of unpacked objects."""
369 def __iter__(self) -> Iterator["UnpackedObject"]:
370 """Iterate over unpacked objects."""
371 raise NotImplementedError(self.__iter__)
373 def __len__(self) -> int:
374 """Return the number of objects in the stream."""
375 raise NotImplementedError(self.__len__)
378def take_msb_bytes(
379 read: Callable[[int], bytes], crc32: int | None = None
380) -> tuple[list[int], int | None]:
381 """Read bytes marked with most significant bit.
383 Args:
384 read: Read function
385 crc32: Optional CRC32 checksum to update
387 Returns:
388 Tuple of (list of bytes read, updated CRC32 or None)
389 """
390 ret: list[int] = []
391 while len(ret) == 0 or ret[-1] & 0x80:
392 b = read(1)
393 if crc32 is not None:
394 crc32 = binascii.crc32(b, crc32)
395 ret.append(ord(b[:1]))
396 return ret, crc32
399def take_msb_bytes_at(
400 contents: "bytes | mmap.mmap", offset: int, crc32: int | None = None
401) -> tuple[list[int], int, int | None]:
402 """Read bytes marked with most significant bit from a buffer at an offset.
404 Args:
405 contents: Buffer to read from
406 offset: Offset in contents to start reading at
407 crc32: Optional CRC32 checksum to update
409 Returns:
410 Tuple of (list of bytes read, offset just past them, updated CRC32 or None)
411 """
412 ret: list[int] = []
413 pos = offset
414 while len(ret) == 0 or ret[-1] & 0x80:
415 b = contents[pos : pos + 1]
416 if not b:
417 raise AssertionError(f"unexpected end of pack data at {pos}")
418 pos += 1
419 if crc32 is not None:
420 crc32 = binascii.crc32(b, crc32)
421 ret.append(ord(b))
422 return ret, pos, crc32
425class PackFileDisappeared(Exception):
426 """Raised when a pack file unexpectedly disappears.
428 This typically happens when a concurrent operation (e.g. ``git repack``
429 or ``git gc --auto``) removes a pack file between the moment dulwich
430 snapshots the pack directory and the moment it actually opens the
431 pack's ``.idx`` or ``.pack`` file.
433 The ``obj`` attribute holds the :class:`Pack` (or :class:`FilePackIndex`)
434 whose backing file vanished, so the caller can evict the stale object
435 from its cache and rescan the pack directory.
436 """
438 obj: "Pack | FilePackIndex"
440 def __init__(self, obj: "Pack | FilePackIndex") -> None:
441 """Initialize PackFileDisappeared exception.
443 Args:
444 obj: The pack or pack index that disappeared.
445 """
446 self.obj = obj
449class UnpackedObject:
450 """Class encapsulating an object unpacked from a pack file.
452 These objects should only be created from within unpack_object. Most
453 members start out as empty and are filled in at various points by
454 read_zlib_chunks, unpack_object, DeltaChainIterator, etc.
456 End users of this object should take care that the function they're getting
457 this object from is guaranteed to set the members they need.
458 """
460 __slots__ = [
461 "_sha", # Cached binary SHA.
462 "comp_chunks", # Compressed object chunks.
463 "crc32", # CRC32.
464 "decomp_chunks", # Decompressed object chunks.
465 "decomp_len", # Decompressed length of this object.
466 "delta_base", # Delta base offset or SHA.
467 "hash_func", # Hash function to use for computing object IDs.
468 "obj_chunks", # Decompressed and delta-resolved chunks.
469 "obj_type_num", # Type of this object.
470 "offset", # Offset in its pack.
471 "pack_type_num", # Type of this object in the pack (may be a delta).
472 ]
474 obj_type_num: int | None
475 obj_chunks: list[bytes] | None
476 delta_base: bytes | int | None
477 decomp_chunks: list[bytes]
478 comp_chunks: list[bytes] | None
479 decomp_len: int | None
480 crc32: int | None
481 offset: int | None
482 pack_type_num: int
483 _sha: bytes | None
484 hash_func: Callable[[], "HashObject"]
486 # TODO(dborowitz): read_zlib_chunks and unpack_object could very well be
487 # methods of this object.
488 def __init__(
489 self,
490 pack_type_num: int,
491 *,
492 delta_base: bytes | int | None = None,
493 decomp_len: int | None = None,
494 crc32: int | None = None,
495 sha: bytes | None = None,
496 decomp_chunks: list[bytes] | None = None,
497 offset: int | None = None,
498 hash_func: Callable[[], "HashObject"] = sha1,
499 ) -> None:
500 """Initialize an UnpackedObject.
502 Args:
503 pack_type_num: Type number of this object in the pack
504 delta_base: Delta base (offset or SHA) if this is a delta object
505 decomp_len: Decompressed length of this object
506 crc32: CRC32 checksum
507 sha: SHA hash of the object
508 decomp_chunks: Decompressed chunks
509 offset: Offset in the pack file
510 hash_func: Hash function to use (defaults to sha1)
511 """
512 self.offset = offset
513 self._sha = sha
514 self.pack_type_num = pack_type_num
515 self.delta_base = delta_base
516 self.comp_chunks = None
517 self.decomp_chunks: list[bytes] = decomp_chunks or []
518 if decomp_chunks is not None and decomp_len is None:
519 self.decomp_len = sum(map(len, decomp_chunks))
520 else:
521 self.decomp_len = decomp_len
522 self.crc32 = crc32
523 self.hash_func = hash_func
525 if pack_type_num in DELTA_TYPES:
526 self.obj_type_num = None
527 self.obj_chunks = None
528 else:
529 self.obj_type_num = pack_type_num
530 self.obj_chunks = self.decomp_chunks
531 self.delta_base = delta_base
533 def sha(self) -> RawObjectID:
534 """Return the binary SHA of this object."""
535 if self._sha is None:
536 assert self.obj_type_num is not None and self.obj_chunks is not None
537 self._sha = obj_sha(self.obj_type_num, self.obj_chunks, self.hash_func)
538 return RawObjectID(self._sha)
540 def sha_file(self) -> ShaFile:
541 """Return a ShaFile from this object."""
542 assert self.obj_type_num is not None and self.obj_chunks is not None
543 return ShaFile.from_raw_chunks(self.obj_type_num, self.obj_chunks)
545 # Only provided for backwards compatibility with code that expects either
546 # chunks or a delta tuple.
547 def _obj(self) -> OldUnpackedObject:
548 """Return the decompressed chunks, or (delta base, delta chunks)."""
549 if self.pack_type_num in DELTA_TYPES:
550 assert isinstance(self.delta_base, bytes | int)
551 return (self.delta_base, self.decomp_chunks)
552 else:
553 return self.decomp_chunks
555 def __eq__(self, other: object) -> bool:
556 """Check equality with another UnpackedObject."""
557 if not isinstance(other, UnpackedObject):
558 return False
559 for slot in self.__slots__:
560 if getattr(self, slot) != getattr(other, slot):
561 return False
562 return True
564 def __ne__(self, other: object) -> bool:
565 """Check inequality with another UnpackedObject."""
566 return not (self == other)
568 def __repr__(self) -> str:
569 """Return string representation of this UnpackedObject."""
570 data = [f"{s}={getattr(self, s)!r}" for s in self.__slots__]
571 return "{}({})".format(self.__class__.__name__, ", ".join(data))
574_ZLIB_BUFSIZE = 65536 # 64KB buffer for better I/O performance
576# Default maximum memory for caching delta base objects (matches Git's default
577# for core.deltaBaseCacheLimit).
578DEFAULT_DELTA_BASE_CACHE_LIMIT = 96 * 1024 * 1024 # 96 MiB
581def read_zlib_chunks(
582 read_some: Callable[[int], bytes],
583 unpacked: UnpackedObject,
584 include_comp: bool = False,
585 buffer_size: int = _ZLIB_BUFSIZE,
586) -> bytes:
587 """Read zlib data from a buffer.
589 This function requires that the buffer have additional data following the
590 compressed data, which is guaranteed to be the case for git pack files.
592 Args:
593 read_some: Read function that returns at least one byte, but may
594 return less than the requested size.
595 unpacked: An UnpackedObject to write result data to. If its crc32
596 attr is not None, the CRC32 of the compressed bytes will be computed
597 using this starting CRC32.
598 After this function, will have the following attrs set:
599 * comp_chunks (if include_comp is True)
600 * decomp_chunks
601 * decomp_len
602 * crc32
603 include_comp: If True, include compressed data in the result.
604 buffer_size: Size of the read buffer.
605 Returns: Leftover unused data from the decompression.
607 Raises:
608 zlib.error: if a decompression error occurred.
609 """
610 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
611 raise ValueError("non-negative zlib data stream size expected")
612 decomp_obj = zlib.decompressobj()
614 comp_chunks = []
615 decomp_chunks = unpacked.decomp_chunks
616 decomp_len = 0
617 crc32 = unpacked.crc32
618 max_decomp = unpacked.decomp_len
620 while True:
621 add = read_some(buffer_size)
622 if not add:
623 raise zlib.error("EOF before end of zlib stream")
624 comp_chunks.append(add)
625 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
626 remaining = max_decomp - decomp_len + 1
627 decomp = decomp_obj.decompress(add, remaining)
628 if decomp_obj.unconsumed_tail:
629 raise zlib.error("decompressed data exceeds expected size")
630 decomp_len += len(decomp)
631 decomp_chunks.append(decomp)
632 unused = decomp_obj.unused_data
633 if unused:
634 left = len(unused)
635 if crc32 is not None:
636 crc32 = binascii.crc32(add[:-left], crc32)
637 if include_comp:
638 comp_chunks[-1] = add[:-left]
639 break
640 elif crc32 is not None:
641 crc32 = binascii.crc32(add, crc32)
642 if crc32 is not None:
643 crc32 &= 0xFFFFFFFF
645 if decomp_len != unpacked.decomp_len:
646 raise zlib.error("decompressed data does not match expected size")
648 unpacked.crc32 = crc32
649 if include_comp:
650 unpacked.comp_chunks = comp_chunks
651 return unused
654def read_zlib_chunks_at(
655 contents: "bytes | mmap.mmap",
656 offset: int,
657 unpacked: UnpackedObject,
658 include_comp: bool = False,
659 buffer_size: int = _ZLIB_BUFSIZE,
660) -> int:
661 """Read zlib data from a buffer at a given offset.
663 Like :func:`read_zlib_chunks`, but indexes the buffer directly instead of
664 consuming a read callable, so concurrent readers do not share a position.
666 The buffer is fed to zlib in ``buffer_size`` slices rather than in one
667 piece. That bounds ``unused_data``, which zlib materialises as a copy of
668 everything it was handed past the end of the stream: passing the whole
669 mapping would copy the entire remainder of the pack for every object read.
671 Slices are taken as memoryviews, so the compressed data is decompressed
672 straight out of the mapping. With ``include_comp`` the chunks are kept on
673 ``unpacked`` and outlive the mapping, so those are copied.
675 Args:
676 contents: Buffer holding the compressed data.
677 offset: Offset in contents at which the zlib stream starts.
678 unpacked: An UnpackedObject to write result data to; see
679 :func:`read_zlib_chunks` for the attributes set on it.
680 include_comp: If True, include compressed data in the result.
681 buffer_size: Number of bytes to feed to zlib at a time.
682 Returns: Offset in contents just past the end of the zlib stream.
684 Raises:
685 zlib.error: if a decompression error occurred.
686 """
687 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
688 raise ValueError("non-negative zlib data stream size expected")
689 decomp_obj = zlib.decompressobj()
691 comp_chunks = []
692 decomp_chunks = unpacked.decomp_chunks
693 decomp_len = 0
694 crc32 = unpacked.crc32
695 max_decomp = unpacked.decomp_len
696 pos = offset
698 with memoryview(contents) as view:
699 while True:
700 add = view[pos : pos + buffer_size]
701 if not add:
702 raise zlib.error("EOF before end of zlib stream")
703 pos += len(add)
704 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
705 remaining = max_decomp - decomp_len + 1
706 decomp = decomp_obj.decompress(add, remaining)
707 if decomp_obj.unconsumed_tail:
708 raise zlib.error("decompressed data exceeds expected size")
709 decomp_len += len(decomp)
710 decomp_chunks.append(decomp)
711 unused = decomp_obj.unused_data
712 if unused:
713 left = len(unused)
714 pos -= left
715 add = add[:-left]
716 if crc32 is not None:
717 crc32 = binascii.crc32(add, crc32)
718 if include_comp:
719 comp_chunks.append(bytes(add))
720 if unused:
721 break
722 if crc32 is not None:
723 crc32 &= 0xFFFFFFFF
725 if decomp_len != unpacked.decomp_len:
726 raise zlib.error("decompressed data does not match expected size")
728 unpacked.crc32 = crc32
729 if include_comp:
730 unpacked.comp_chunks = comp_chunks
731 return pos
734def iter_sha1(iter: Iterable[bytes]) -> bytes:
735 """Return the hexdigest of the SHA1 over a set of names.
737 Args:
738 iter: Iterator over string objects
739 Returns: 40-byte hex sha1 digest
740 """
741 sha = sha1()
742 for name in iter:
743 sha.update(name)
744 return sha.hexdigest().encode("ascii")
747def load_pack_index(
748 path: str | os.PathLike[str], object_format: ObjectFormat
749) -> "PackIndex":
750 """Load an index file by path.
752 Args:
753 path: Path to the index file
754 object_format: Hash algorithm used by the repository
755 Returns: A PackIndex loaded from the given path
756 """
757 # Ownership of the file is transferred to the returned index, which mmaps
758 # it and closes it in PackIndex.close(). It must not be closed here: on
759 # Windows an mmap keeps the file locked, so closing the handle out from
760 # under a live mapping leaves the .idx undeletable until the index is GCed.
761 f = GitFile(path, "rb")
762 try:
763 return load_pack_index_file(path, f, object_format)
764 except BaseException:
765 f.close()
766 raise
769def _load_file_contents(
770 f: IO[bytes] | _GitFile, size: int | None = None
771) -> tuple[bytes | Any, int]:
772 """Load contents from a file, preferring mmap when possible.
774 Args:
775 f: File-like object to load
776 size: Expected size, or None to determine from file
777 Returns: Tuple of (contents, size)
778 """
779 # Avoid rolling a SpooledTemporaryFile to disk just to get a descriptor.
780 if getattr(f, "_rolled", True) is False:
781 fd = None
782 else:
783 try:
784 fd = f.fileno()
785 except (UnsupportedOperation, AttributeError):
786 fd = None
787 # Attempt to use mmap if possible
788 if fd is not None:
789 if size is None:
790 size = os.fstat(fd).st_size
791 if has_mmap:
792 try:
793 contents = mmap.mmap(fd, size, access=mmap.ACCESS_READ)
794 except (OSError, ValueError):
795 # Can't mmap - perhaps a socket or invalid file descriptor
796 pass
797 else:
798 return contents, size
799 contents_bytes = f.read()
800 size = len(contents_bytes)
801 return contents_bytes, size
804def _close_file_contents(contents: "bytes | mmap.mmap | None") -> None:
805 """Close contents returned by _load_file_contents, if closeable.
807 Callers must close the mapping before the file it maps: on Windows the
808 mapping holds a lock on the file, so the handle cannot be released while
809 it is alive.
810 """
811 close_fn = getattr(contents, "close", None)
812 if close_fn is not None:
813 close_fn()
816def load_pack_index_file(
817 path: str | os.PathLike[str],
818 f: IO[bytes] | _GitFile,
819 object_format: ObjectFormat,
820) -> "PackIndex":
821 """Load an index file from a file-like object.
823 Args:
824 path: Path for the index file
825 f: File-like object
826 object_format: Hash algorithm used by the repository
827 Returns: A PackIndex loaded from the given file
828 """
829 contents, size = _load_file_contents(f)
830 if contents[:4] == b"\377tOc":
831 version = struct.unpack(b">L", contents[4:8])[0]
832 if version == 2:
833 return PackIndex2(
834 path,
835 object_format,
836 file=f,
837 contents=contents,
838 size=size,
839 )
840 elif version == 3:
841 return PackIndex3(path, object_format, file=f, contents=contents, size=size)
842 else:
843 raise KeyError(f"Unknown pack index format {version}")
844 else:
845 return PackIndex1(path, object_format, file=f, contents=contents, size=size)
848def bisect_find_sha(
849 start: int, end: int, sha: bytes, unpack_name: Callable[[int], bytes]
850) -> int | None:
851 """Find a SHA in a data blob with sorted SHAs.
853 Args:
854 start: Start index of range to search
855 end: End index of range to search
856 sha: Sha to find
857 unpack_name: Callback to retrieve SHA by index
858 Returns: Index of the SHA, or None if it wasn't found
859 """
860 assert start <= end
861 while start <= end:
862 i = (start + end) // 2
863 file_sha = unpack_name(i)
864 if file_sha < sha:
865 start = i + 1
866 elif file_sha > sha:
867 end = i - 1
868 else:
869 return i
870 return None
873PackIndexEntry = tuple[RawObjectID, int, int | None]
876class PackIndex:
877 """An index in to a packfile.
879 Given a sha id of an object a pack index can tell you the location in the
880 packfile of that object if it has it.
881 """
883 object_format: "ObjectFormat"
885 def __eq__(self, other: object) -> bool:
886 """Check equality with another PackIndex."""
887 if not isinstance(other, PackIndex):
888 return False
890 for (name1, _, _), (name2, _, _) in zip(
891 self.iterentries(), other.iterentries()
892 ):
893 if name1 != name2:
894 return False
895 return True
897 def __ne__(self, other: object) -> bool:
898 """Check if this pack index is not equal to another."""
899 return not self.__eq__(other)
901 def __len__(self) -> int:
902 """Return the number of entries in this pack index."""
903 raise NotImplementedError(self.__len__)
905 def __iter__(self) -> Iterator[ObjectID]:
906 """Iterate over the SHAs in this pack."""
907 return map(lambda sha: sha_to_hex(RawObjectID(sha)), self._itersha())
909 def iterentries(self) -> Iterator[PackIndexEntry]:
910 """Iterate over the entries in this pack index.
912 Returns: iterator over tuples with object name, offset in packfile and
913 crc32 checksum.
914 """
915 raise NotImplementedError(self.iterentries)
917 def get_pack_checksum(self) -> bytes | None:
918 """Return the SHA1 checksum stored for the corresponding packfile.
920 Returns: 20-byte binary digest, or None if not available
921 """
922 raise NotImplementedError(self.get_pack_checksum)
924 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
925 """Return the offset in to the corresponding packfile for the object.
927 Given the name of an object it will return the offset that object
928 lives at within the corresponding pack file. If the pack file doesn't
929 have the object then None will be returned.
930 """
931 raise NotImplementedError(self.object_offset)
933 def object_sha1(self, index: int) -> bytes:
934 """Return the SHA1 corresponding to the index in the pack file."""
935 for name, offset, _crc32 in self.iterentries():
936 if offset == index:
937 return name
938 else:
939 raise KeyError(index)
941 def _object_offset(self, sha: bytes) -> int:
942 """See object_offset.
944 Args:
945 sha: A *binary* SHA string. (20 characters long)_
946 """
947 raise NotImplementedError(self._object_offset)
949 def objects_sha1(self) -> bytes:
950 """Return the hex SHA1 over all the shas of all objects in this pack.
952 Note: This is used for the filename of the pack.
953 """
954 return iter_sha1(self._itersha())
956 def _itersha(self) -> Iterator[bytes]:
957 """Yield all the SHA1's of the objects in the index, sorted."""
958 raise NotImplementedError(self._itersha)
960 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
961 """Iterate over all SHA1s with the given prefix.
963 Args:
964 prefix: Binary prefix to match
965 Returns: Iterator of matching SHA1s
966 """
967 # Default implementation for PackIndex classes that don't override
968 for sha, _, _ in self.iterentries():
969 if sha.startswith(prefix):
970 yield RawObjectID(sha)
972 def close(self) -> None:
973 """Close any open files."""
975 def check(self) -> None:
976 """Check the consistency of this pack index."""
979class MemoryPackIndex(PackIndex):
980 """Pack index that is stored entirely in memory."""
982 def __init__(
983 self,
984 entries: list[PackIndexEntry],
985 object_format: ObjectFormat,
986 pack_checksum: bytes | None = None,
987 ) -> None:
988 """Create a new MemoryPackIndex.
990 Args:
991 entries: Sequence of name, idx, crc32 (sorted)
992 object_format: Object format used by this index
993 pack_checksum: Optional pack checksum
994 """
995 self._by_sha = {}
996 self._by_offset = {}
997 for name, offset, _crc32 in entries:
998 self._by_sha[name] = offset
999 self._by_offset[offset] = name
1000 self._entries = entries
1001 self._pack_checksum = pack_checksum
1002 self.object_format = object_format
1004 def get_pack_checksum(self) -> bytes | None:
1005 """Return the SHA checksum stored for the corresponding packfile."""
1006 return self._pack_checksum
1008 def __len__(self) -> int:
1009 """Return the number of entries in this pack index."""
1010 return len(self._entries)
1012 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1013 """Return the offset for the given SHA.
1015 Args:
1016 sha: SHA to look up (binary or hex)
1017 Returns: Offset in the pack file
1018 """
1019 lookup_sha: RawObjectID
1020 if len(sha) == self.object_format.hex_length:
1021 lookup_sha = hex_to_sha(ObjectID(sha))
1022 else:
1023 lookup_sha = RawObjectID(sha)
1024 return self._by_sha[lookup_sha]
1026 def object_sha1(self, index: int) -> bytes:
1027 """Return the SHA1 for the object at the given offset."""
1028 return self._by_offset[index]
1030 def _itersha(self) -> Iterator[bytes]:
1031 """Iterate over all SHA1s in the index."""
1032 return iter(self._by_sha)
1034 def iterentries(self) -> Iterator[PackIndexEntry]:
1035 """Iterate over all index entries."""
1036 return iter(self._entries)
1038 @classmethod
1039 def for_pack(cls, pack_data: "PackData") -> "MemoryPackIndex":
1040 """Create a MemoryPackIndex from a PackData object."""
1041 return MemoryPackIndex(
1042 list(pack_data.sorted_entries()),
1043 pack_checksum=pack_data.get_stored_checksum(),
1044 object_format=pack_data.object_format,
1045 )
1047 @classmethod
1048 def clone(cls, other_index: "PackIndex") -> "MemoryPackIndex":
1049 """Create a copy of another PackIndex in memory."""
1050 return cls(
1051 list(other_index.iterentries()),
1052 other_index.object_format,
1053 other_index.get_pack_checksum(),
1054 )
1057class FilePackIndex(PackIndex):
1058 """Pack index that is based on a file.
1060 To do the loop it opens the file, and indexes first 256 4 byte groups
1061 with the first byte of the sha id. The value in the four byte group indexed
1062 is the end of the group that shares the same starting byte. Subtract one
1063 from the starting byte and index again to find the start of the group.
1064 The values are sorted by sha id within the group, so do the math to find
1065 the start and end offset and then bisect in to find if the value is
1066 present.
1067 """
1069 _fan_out_table: list[int]
1070 _file: IO[bytes] | _GitFile
1072 def __init__(
1073 self,
1074 filename: str | os.PathLike[str],
1075 file: IO[bytes] | _GitFile | None = None,
1076 contents: "bytes | mmap.mmap | None" = None,
1077 size: int | None = None,
1078 ) -> None:
1079 """Create a pack index object.
1081 Provide it with the name of the index file to consider, and it will map
1082 it whenever required.
1083 """
1084 self._filename = filename
1085 # Take the size now, so it can be checked each time we map the file to
1086 # ensure that it hasn't changed.
1087 if file is None:
1088 self._file = GitFile(filename, "rb")
1089 else:
1090 self._file = file
1091 if contents is None:
1092 self._contents, self._size = _load_file_contents(self._file, size)
1093 else:
1094 self._contents = contents
1095 self._size = size if size is not None else len(contents)
1097 @property
1098 def path(self) -> str:
1099 """Return the path to this index file."""
1100 return os.fspath(self._filename)
1102 def __eq__(self, other: object) -> bool:
1103 """Check equality with another FilePackIndex."""
1104 # Quick optimization:
1105 if (
1106 isinstance(other, FilePackIndex)
1107 and self._fan_out_table != other._fan_out_table
1108 ):
1109 return False
1111 return super().__eq__(other)
1113 def close(self) -> None:
1114 """Close the underlying file and any mmap."""
1115 _close_file_contents(self._contents)
1116 self._file.close()
1118 def __del__(self) -> None:
1119 """Ensure the file and mmap are closed when GCed."""
1120 if not getattr(self._file, "closed", True):
1121 import warnings
1123 warnings.warn(
1124 f"unclosed pack index {self!r}",
1125 ResourceWarning,
1126 stacklevel=2,
1127 source=self,
1128 )
1129 try:
1130 self.close()
1131 except Exception:
1132 # Ignore errors during cleanup
1133 pass
1135 def __enter__(self) -> Self:
1136 """Enter context manager."""
1137 return self
1139 def __exit__(
1140 self,
1141 type: type | None,
1142 value: BaseException | None,
1143 traceback: TracebackType | None,
1144 ) -> None:
1145 """Exit context manager."""
1146 self.close()
1148 def __len__(self) -> int:
1149 """Return the number of entries in this pack index."""
1150 return self._fan_out_table[-1]
1152 def _unpack_entry(self, i: int) -> PackIndexEntry:
1153 """Unpack the i-th entry in the index file.
1155 Returns: Tuple with object name (SHA), offset in pack file and CRC32
1156 checksum (if known).
1157 """
1158 raise NotImplementedError(self._unpack_entry)
1160 def _unpack_name(self, i: int) -> bytes:
1161 """Unpack the i-th name from the index file."""
1162 raise NotImplementedError(self._unpack_name)
1164 def _unpack_offset(self, i: int) -> int:
1165 """Unpack the i-th object offset from the index file."""
1166 raise NotImplementedError(self._unpack_offset)
1168 def _unpack_crc32_checksum(self, i: int) -> int | None:
1169 """Unpack the crc32 checksum for the i-th object from the index file."""
1170 raise NotImplementedError(self._unpack_crc32_checksum)
1172 def _itersha(self) -> Iterator[bytes]:
1173 """Iterate over all SHA1s in the index."""
1174 for i in range(len(self)):
1175 yield self._unpack_name(i)
1177 def iterentries(self) -> Iterator[PackIndexEntry]:
1178 """Iterate over the entries in this pack index.
1180 Returns: iterator over tuples with object name, offset in packfile and
1181 crc32 checksum.
1182 """
1183 for i in range(len(self)):
1184 yield self._unpack_entry(i)
1186 def _read_fan_out_table(self, start_offset: int) -> list[int]:
1187 """Read the fan-out table from the index.
1189 The fan-out table contains 256 entries mapping first byte values
1190 to the number of objects with SHA1s less than or equal to that byte.
1192 Args:
1193 start_offset: Offset in the file where the fan-out table starts
1194 Returns: List of 256 integers
1195 """
1196 ret = []
1197 for i in range(0x100):
1198 fanout_entry = self._contents[
1199 start_offset + i * 4 : start_offset + (i + 1) * 4
1200 ]
1201 ret.append(struct.unpack(">L", fanout_entry)[0])
1202 return ret
1204 def check(self) -> None:
1205 """Check that the stored checksum matches the actual checksum."""
1206 actual = self.calculate_checksum()
1207 stored = self.get_stored_checksum()
1208 if actual != stored:
1209 raise ChecksumMismatch(stored, actual)
1211 def calculate_checksum(self) -> bytes:
1212 """Calculate the SHA1 checksum over this pack index.
1214 Returns: This is a 20-byte binary digest
1215 """
1216 return sha1(self._contents[:-20]).digest()
1218 def get_pack_checksum(self) -> bytes:
1219 """Return the SHA1 checksum stored for the corresponding packfile.
1221 Returns: 20-byte binary digest
1222 """
1223 return bytes(self._contents[-40:-20])
1225 def get_stored_checksum(self) -> bytes:
1226 """Return the SHA1 checksum stored for this index.
1228 Returns: 20-byte binary digest
1229 """
1230 return bytes(self._contents[-20:])
1232 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1233 """Return the offset in to the corresponding packfile for the object.
1235 Given the name of an object it will return the offset that object
1236 lives at within the corresponding pack file. If the pack file doesn't
1237 have the object then None will be returned.
1238 """
1239 lookup_sha: RawObjectID
1240 if len(sha) == self.object_format.hex_length: # hex string
1241 lookup_sha = hex_to_sha(ObjectID(sha))
1242 else:
1243 lookup_sha = RawObjectID(sha)
1244 try:
1245 return self._object_offset(lookup_sha)
1246 except ValueError as exc:
1247 closed = getattr(self._contents, "closed", None)
1248 if closed in (None, True):
1249 raise PackFileDisappeared(self) from exc
1250 raise
1252 def _object_offset(self, sha: bytes) -> int:
1253 """See object_offset.
1255 Args:
1256 sha: A *binary* SHA string. (20 characters long)_
1257 """
1258 hash_size = getattr(self, "hash_size", 20) # Default to SHA1 for v1
1259 assert len(sha) == hash_size
1260 idx = ord(sha[:1])
1261 if idx == 0:
1262 start = 0
1263 else:
1264 start = self._fan_out_table[idx - 1]
1265 end = self._fan_out_table[idx]
1266 i = bisect_find_sha(start, end, sha, self._unpack_name)
1267 if i is None:
1268 raise KeyError(sha)
1269 return self._unpack_offset(i)
1271 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
1272 """Iterate over all SHA1s with the given prefix."""
1273 start = ord(prefix[:1])
1274 if start == 0:
1275 start = 0
1276 else:
1277 start = self._fan_out_table[start - 1]
1278 end = ord(prefix[:1]) + 1
1279 if end == 0x100:
1280 end = len(self)
1281 else:
1282 end = self._fan_out_table[end]
1283 assert start <= end
1284 started = False
1285 for i in range(start, end):
1286 name: bytes = self._unpack_name(i)
1287 if name.startswith(prefix):
1288 yield RawObjectID(name)
1289 started = True
1290 elif started:
1291 break
1294class PackIndex1(FilePackIndex):
1295 """Version 1 Pack Index file."""
1297 object_format = SHA1
1299 def __init__(
1300 self,
1301 filename: str | os.PathLike[str],
1302 object_format: ObjectFormat,
1303 file: IO[bytes] | _GitFile | None = None,
1304 contents: bytes | None = None,
1305 size: int | None = None,
1306 ) -> None:
1307 """Initialize a version 1 pack index.
1309 Args:
1310 filename: Path to the index file
1311 object_format: Object format used by the repository
1312 file: Optional file object
1313 contents: Optional mmap'd contents
1314 size: Optional size of the index
1315 """
1316 super().__init__(filename, file, contents, size)
1318 # PackIndex1 only supports SHA1
1319 if object_format != SHA1:
1320 raise AssertionError(
1321 f"PackIndex1 only supports SHA1, not {object_format.name}"
1322 )
1324 self.object_format = object_format
1325 self.version = 1
1326 self._fan_out_table = self._read_fan_out_table(0)
1327 self.hash_size = self.object_format.oid_length
1328 self._entry_size = 4 + self.hash_size
1330 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, None]:
1331 base_offset = (0x100 * 4) + (i * self._entry_size)
1332 offset = unpack_from(">L", self._contents, base_offset)[0]
1333 name = self._contents[base_offset + 4 : base_offset + 4 + self.hash_size]
1334 return (RawObjectID(name), offset, None)
1336 def _unpack_name(self, i: int) -> bytes:
1337 offset = (0x100 * 4) + (i * self._entry_size) + 4
1338 return self._contents[offset : offset + self.hash_size]
1340 def _unpack_offset(self, i: int) -> int:
1341 offset = (0x100 * 4) + (i * self._entry_size)
1342 return int(unpack_from(">L", self._contents, offset)[0])
1344 def _unpack_crc32_checksum(self, i: int) -> None:
1345 # Not stored in v1 index files
1346 return None
1349class PackIndex2(FilePackIndex):
1350 """Version 2 Pack Index file."""
1352 object_format = SHA1
1354 def __init__(
1355 self,
1356 filename: str | os.PathLike[str],
1357 object_format: ObjectFormat,
1358 file: IO[bytes] | _GitFile | None = None,
1359 contents: bytes | None = None,
1360 size: int | None = None,
1361 ) -> None:
1362 """Initialize a version 2 pack index.
1364 Args:
1365 filename: Path to the index file
1366 object_format: Object format used by the repository
1367 file: Optional file object
1368 contents: Optional mmap'd contents
1369 size: Optional size of the index
1370 """
1371 super().__init__(filename, file, contents, size)
1372 self.object_format = object_format
1373 if self._contents[:4] != b"\377tOc":
1374 raise AssertionError("Not a v2 pack index file")
1375 (self.version,) = unpack_from(b">L", self._contents, 4)
1376 if self.version != 2:
1377 raise AssertionError(f"Version was {self.version}")
1378 self._fan_out_table = self._read_fan_out_table(8)
1379 self.hash_size = self.object_format.oid_length
1380 self._name_table_offset = 8 + 0x100 * 4
1381 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1382 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1383 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1384 self
1385 )
1387 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1388 return (
1389 RawObjectID(self._unpack_name(i)),
1390 self._unpack_offset(i),
1391 self._unpack_crc32_checksum(i),
1392 )
1394 def _unpack_name(self, i: int) -> bytes:
1395 offset = self._name_table_offset + i * self.hash_size
1396 return self._contents[offset : offset + self.hash_size]
1398 def _unpack_offset(self, i: int) -> int:
1399 offset = self._pack_offset_table_offset + i * 4
1400 offset_val = int(unpack_from(">L", self._contents, offset)[0])
1401 if offset_val & (2**31):
1402 offset = (
1403 self._pack_offset_largetable_offset + (offset_val & (2**31 - 1)) * 8
1404 )
1405 offset_val = int(unpack_from(">Q", self._contents, offset)[0])
1406 return offset_val
1408 def _unpack_crc32_checksum(self, i: int) -> int:
1409 return int(
1410 unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1411 )
1413 def get_pack_checksum(self) -> bytes:
1414 """Return the checksum stored for the corresponding packfile.
1416 Returns: binary digest (size depends on hash algorithm)
1417 """
1418 # Index ends with: pack_checksum + index_checksum
1419 # Each checksum is hash_size bytes
1420 checksum_size = self.hash_size
1421 return bytes(self._contents[-2 * checksum_size : -checksum_size])
1423 def get_stored_checksum(self) -> bytes:
1424 """Return the checksum stored for this index.
1426 Returns: binary digest (size depends on hash algorithm)
1427 """
1428 checksum_size = self.hash_size
1429 return bytes(self._contents[-checksum_size:])
1431 def calculate_checksum(self) -> bytes:
1432 """Calculate the checksum over this pack index.
1434 Returns: binary digest (size depends on hash algorithm)
1435 """
1436 # Determine hash function based on hash_size
1437 if self.hash_size == 20:
1438 hash_func = sha1
1439 elif self.hash_size == 32:
1440 hash_func = sha256
1441 else:
1442 raise ValueError(f"Unsupported hash size: {self.hash_size}")
1444 return hash_func(self._contents[: -self.hash_size]).digest()
1447class PackIndex3(FilePackIndex):
1448 """Version 3 Pack Index file.
1450 Supports variable hash sizes for SHA-1 (20 bytes) and SHA-256 (32 bytes).
1451 """
1453 def __init__(
1454 self,
1455 filename: str | os.PathLike[str],
1456 object_format: ObjectFormat,
1457 file: IO[bytes] | _GitFile | None = None,
1458 contents: bytes | None = None,
1459 size: int | None = None,
1460 ) -> None:
1461 """Initialize a version 3 pack index.
1463 Args:
1464 filename: Path to the index file
1465 object_format: Object format used by the repository
1466 file: Optional file object
1467 contents: Optional mmap'd contents
1468 size: Optional size of the index
1469 """
1470 super().__init__(filename, file, contents, size)
1471 if self._contents[:4] != b"\377tOc":
1472 raise AssertionError("Not a v3 pack index file")
1473 (self.version,) = unpack_from(b">L", self._contents, 4)
1474 if self.version != 3:
1475 raise AssertionError(f"Version was {self.version}")
1477 # Read hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
1478 (self.hash_format,) = unpack_from(b">L", self._contents, 8)
1479 file_object_format = OBJECT_FORMAT_TYPE_NUMS[self.hash_format]
1481 # Verify provided object_format matches what's in the file
1482 if object_format != file_object_format:
1483 raise AssertionError(
1484 f"Object format mismatch: provided {object_format.name}, "
1485 f"but file contains {file_object_format.name}"
1486 )
1488 self.object_format = object_format
1489 self.hash_size = self.object_format.oid_length
1491 # Read length of shortened object names
1492 (self.shortened_oid_len,) = unpack_from(b">L", self._contents, 12)
1494 # Calculate offsets based on variable hash size
1495 self._fan_out_table = self._read_fan_out_table(
1496 16
1497 ) # After header (4 + 4 + 4 + 4)
1498 self._name_table_offset = 16 + 0x100 * 4
1499 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1500 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1501 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1502 self
1503 )
1505 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1506 return (
1507 RawObjectID(self._unpack_name(i)),
1508 self._unpack_offset(i),
1509 self._unpack_crc32_checksum(i),
1510 )
1512 def _unpack_name(self, i: int) -> bytes:
1513 offset = self._name_table_offset + i * self.hash_size
1514 return self._contents[offset : offset + self.hash_size]
1516 def _unpack_offset(self, i: int) -> int:
1517 offset_pos = self._pack_offset_table_offset + i * 4
1518 offset = unpack_from(">L", self._contents, offset_pos)[0]
1519 assert isinstance(offset, int)
1520 if offset & (2**31):
1521 large_offset_pos = (
1522 self._pack_offset_largetable_offset + (offset & (2**31 - 1)) * 8
1523 )
1524 offset = unpack_from(">Q", self._contents, large_offset_pos)[0]
1525 assert isinstance(offset, int)
1526 return offset
1528 def _unpack_crc32_checksum(self, i: int) -> int:
1529 result = unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1530 assert isinstance(result, int)
1531 return result
1534def read_pack_header_at(
1535 contents: "bytes | mmap.mmap", offset: int = 0
1536) -> tuple[int, int]:
1537 """Read the header of a pack file from a buffer.
1539 Args:
1540 contents: Buffer holding the pack
1541 offset: Offset in contents at which the header starts
1542 Returns: Tuple of (pack version, number of objects).
1543 """
1544 header = contents[offset : offset + 12]
1545 if not header:
1546 raise AssertionError("file too short to contain pack")
1547 if header[:4] != b"PACK":
1548 raise AssertionError(f"Invalid pack header {bytes(header)!r}")
1549 (version,) = unpack_from(b">L", header, 4)
1550 if version not in (2, 3):
1551 raise AssertionError(f"Version was {version}")
1552 (num_objects,) = unpack_from(b">L", header, 8)
1553 return (version, num_objects)
1556def read_pack_header(read: Callable[[int], bytes]) -> tuple[int, int]:
1557 """Read the header of a pack file.
1559 Args:
1560 read: Read function
1561 Returns: Tuple of (pack version, number of objects).
1562 """
1563 return read_pack_header_at(read(12))
1566def chunks_length(chunks: bytes | Iterable[bytes]) -> int:
1567 """Get the total length of a sequence of chunks.
1569 Args:
1570 chunks: Either a single bytes object or an iterable of bytes
1571 Returns: Total length in bytes
1572 """
1573 if isinstance(chunks, bytes):
1574 return len(chunks)
1575 else:
1576 return sum(map(len, chunks))
1579def _decode_object_header(raw: list[int]) -> tuple[int, int]:
1580 """Decode an object type and size from a pack object header."""
1581 type_num = (raw[0] >> 4) & 0x07
1582 size = raw[0] & 0x0F
1583 for i, byte in enumerate(raw[1:]):
1584 size += (byte & 0x7F) << ((i * 7) + 4)
1585 return type_num, size
1588def _decode_delta_base_offset(raw: list[int]) -> int:
1589 """Decode an OFS_DELTA base offset from its variable-length encoding."""
1590 if raw[-1] & 0x80:
1591 raise AssertionError
1592 delta_base_offset = raw[0] & 0x7F
1593 for byte in raw[1:]:
1594 delta_base_offset += 1
1595 delta_base_offset <<= 7
1596 delta_base_offset += byte & 0x7F
1597 if delta_base_offset == 0:
1598 # A zero offset makes the delta reference itself, which would
1599 # loop forever in resolve_object. git's C client rejects this
1600 # with "delta offset == 0 is invalid".
1601 raise ApplyDeltaError("OFS_DELTA has delta_base_offset of 0")
1602 return delta_base_offset
1605def unpack_object(
1606 read_all: Callable[[int], bytes],
1607 hash_func: Callable[[], "HashObject"],
1608 read_some: Callable[[int], bytes] | None = None,
1609 compute_crc32: bool = False,
1610 include_comp: bool = False,
1611 zlib_bufsize: int = _ZLIB_BUFSIZE,
1612) -> tuple[UnpackedObject, bytes]:
1613 """Unpack a Git object.
1615 Args:
1616 read_all: Read function that blocks until the number of requested
1617 bytes are read.
1618 hash_func: Hash function to use for computing object IDs.
1619 read_some: Read function that returns at least one byte, but may not
1620 return the number of bytes requested.
1621 compute_crc32: If True, compute the CRC32 of the compressed data. If
1622 False, the returned CRC32 will be None.
1623 include_comp: If True, include compressed data in the result.
1624 zlib_bufsize: An optional buffer size for zlib operations.
1625 Returns: A tuple of (unpacked, unused), where unused is the unused data
1626 leftover from decompression, and unpacked in an UnpackedObject with
1627 the following attrs set:
1629 * obj_chunks (for non-delta types)
1630 * pack_type_num
1631 * delta_base (for delta types)
1632 * comp_chunks (if include_comp is True)
1633 * decomp_chunks
1634 * decomp_len
1635 * crc32 (if compute_crc32 is True)
1636 """
1637 if read_some is None:
1638 read_some = read_all
1639 if compute_crc32:
1640 crc32 = 0
1641 else:
1642 crc32 = None
1644 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1645 type_num, size = _decode_object_header(raw)
1647 delta_base: int | bytes | None
1648 raw_base = len(raw)
1649 if type_num == OFS_DELTA:
1650 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1651 raw_base += len(raw)
1652 delta_base = _decode_delta_base_offset(raw)
1653 elif type_num == REF_DELTA:
1654 # Determine hash size from hash_func
1655 hash_size = len(hash_func().digest())
1656 delta_base_obj = read_all(hash_size)
1657 if crc32 is not None:
1658 crc32 = binascii.crc32(delta_base_obj, crc32)
1659 delta_base = delta_base_obj
1660 raw_base += hash_size
1661 else:
1662 delta_base = None
1664 unpacked = UnpackedObject(
1665 type_num,
1666 delta_base=delta_base,
1667 decomp_len=size,
1668 crc32=crc32,
1669 hash_func=hash_func,
1670 )
1671 unused = read_zlib_chunks(
1672 read_some,
1673 unpacked,
1674 buffer_size=zlib_bufsize,
1675 include_comp=include_comp,
1676 )
1677 return unpacked, unused
1680def unpack_object_at(
1681 contents: "bytes | mmap.mmap",
1682 offset: int,
1683 hash_func: Callable[[], "HashObject"],
1684 compute_crc32: bool = False,
1685 include_comp: bool = False,
1686 zlib_bufsize: int = _ZLIB_BUFSIZE,
1687) -> tuple[UnpackedObject, int]:
1688 """Unpack a Git object from a buffer at a given offset.
1690 Like :func:`unpack_object`, but indexes the buffer directly rather than
1691 consuming a read callable, so any number of readers can work on the same
1692 buffer concurrently.
1694 Args:
1695 contents: Buffer holding the pack.
1696 offset: Offset in contents at which the object starts.
1697 hash_func: Hash function to use for computing object IDs.
1698 compute_crc32: If True, compute the CRC32 of the compressed data.
1699 include_comp: If True, include compressed data in the result.
1700 zlib_bufsize: An optional buffer size for zlib operations.
1701 Returns: A tuple of (unpacked, end), where end is the offset just past
1702 the object and unpacked is an UnpackedObject with its ``offset`` set;
1703 see :func:`unpack_object` for the other attributes.
1704 """
1705 crc32: int | None = 0 if compute_crc32 else None
1707 raw, pos, crc32 = take_msb_bytes_at(contents, offset, crc32=crc32)
1708 type_num, size = _decode_object_header(raw)
1710 delta_base: int | bytes | None
1711 if type_num == OFS_DELTA:
1712 raw, pos, crc32 = take_msb_bytes_at(contents, pos, crc32=crc32)
1713 delta_base = _decode_delta_base_offset(raw)
1714 elif type_num == REF_DELTA:
1715 hash_size = len(hash_func().digest())
1716 delta_base_obj = bytes(contents[pos : pos + hash_size])
1717 if len(delta_base_obj) != hash_size:
1718 raise AssertionError(f"unexpected end of pack data at {pos}")
1719 pos += hash_size
1720 if crc32 is not None:
1721 crc32 = binascii.crc32(delta_base_obj, crc32)
1722 delta_base = delta_base_obj
1723 else:
1724 delta_base = None
1726 unpacked = UnpackedObject(
1727 type_num,
1728 delta_base=delta_base,
1729 decomp_len=size,
1730 crc32=crc32,
1731 hash_func=hash_func,
1732 )
1733 unpacked.offset = offset
1734 end = read_zlib_chunks_at(
1735 contents,
1736 pos,
1737 unpacked,
1738 buffer_size=zlib_bufsize,
1739 include_comp=include_comp,
1740 )
1741 return unpacked, end
1744def _compute_object_size(value: tuple[int, Any]) -> int:
1745 """Compute the size of a unresolved object for use with LRUSizeCache."""
1746 (num, obj) = value
1747 if num in DELTA_TYPES:
1748 return chunks_length(obj[1])
1749 return chunks_length(obj)
1752class PackStreamReader:
1753 """Class to read a pack stream.
1755 The pack is read from a ReceivableProtocol using read() or recv() as
1756 appropriate.
1757 """
1759 def __init__(
1760 self,
1761 hash_func: Callable[[], "HashObject"],
1762 read_all: Callable[[int], bytes],
1763 read_some: Callable[[int], bytes] | None = None,
1764 zlib_bufsize: int = _ZLIB_BUFSIZE,
1765 ) -> None:
1766 """Initialize pack stream reader.
1768 Args:
1769 hash_func: Hash function to use for computing object IDs
1770 read_all: Function to read all requested bytes
1771 read_some: Function to read some bytes (optional)
1772 zlib_bufsize: Buffer size for zlib decompression
1773 """
1774 self.read_all = read_all
1775 if read_some is None:
1776 self.read_some = read_all
1777 else:
1778 self.read_some = read_some
1779 self.hash_func = hash_func
1780 self.sha = hash_func()
1781 self._hash_size = len(hash_func().digest())
1782 self._offset = 0
1783 self._rbuf = BytesIO()
1784 # trailer is a deque to avoid memory allocation on small reads
1785 self._trailer: deque[int] = deque()
1786 self._zlib_bufsize = zlib_bufsize
1788 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1789 """Read up to size bytes using the given callback.
1791 As a side effect, update the verifier's hash (excluding the last
1792 hash_size bytes read, which is the pack checksum).
1794 Args:
1795 read: The read callback to read from.
1796 size: The maximum number of bytes to read; the particular
1797 behavior is callback-specific.
1798 Returns: Bytes read
1799 """
1800 data = read(size)
1802 # maintain a trailer of the last hash_size bytes we've read
1803 n = len(data)
1804 self._offset += n
1805 tn = len(self._trailer)
1806 if n >= self._hash_size:
1807 to_pop = tn
1808 to_add = self._hash_size
1809 else:
1810 to_pop = max(n + tn - self._hash_size, 0)
1811 to_add = n
1812 self.sha.update(
1813 bytes(bytearray([self._trailer.popleft() for _ in range(to_pop)]))
1814 )
1815 self._trailer.extend(data[-to_add:])
1817 # hash everything but the trailer
1818 self.sha.update(data[:-to_add])
1819 return data
1821 def _buf_len(self) -> int:
1822 buf = self._rbuf
1823 start = buf.tell()
1824 buf.seek(0, SEEK_END)
1825 end = buf.tell()
1826 buf.seek(start)
1827 return end - start
1829 @property
1830 def offset(self) -> int:
1831 """Return current offset in the stream."""
1832 return self._offset - self._buf_len()
1834 def read(self, size: int) -> bytes:
1835 """Read, blocking until size bytes are read."""
1836 buf_len = self._buf_len()
1837 if buf_len >= size:
1838 return self._rbuf.read(size)
1839 buf_data = self._rbuf.read()
1840 self._rbuf = BytesIO()
1841 return buf_data + self._read(self.read_all, size - buf_len)
1843 def recv(self, size: int) -> bytes:
1844 """Read up to size bytes, blocking until one byte is read."""
1845 buf_len = self._buf_len()
1846 if buf_len:
1847 data = self._rbuf.read(size)
1848 if size >= buf_len:
1849 self._rbuf = BytesIO()
1850 return data
1851 return self._read(self.read_some, size)
1853 def __len__(self) -> int:
1854 """Return the number of objects in this pack."""
1855 return self._num_objects
1857 def read_objects(self, compute_crc32: bool = False) -> Iterator[UnpackedObject]:
1858 """Read the objects in this pack file.
1860 Args:
1861 compute_crc32: If True, compute the CRC32 of the compressed
1862 data. If False, the returned CRC32 will be None.
1863 Returns: Iterator over UnpackedObjects with the following members set:
1864 offset
1865 obj_type_num
1866 obj_chunks (for non-delta types)
1867 delta_base (for delta types)
1868 decomp_chunks
1869 decomp_len
1870 crc32 (if compute_crc32 is True)
1872 Raises:
1873 ChecksumMismatch: if the checksum of the pack contents does not
1874 match the checksum in the pack trailer.
1875 zlib.error: if an error occurred during zlib decompression.
1876 IOError: if an error occurred writing to the output file.
1877 """
1878 _pack_version, self._num_objects = read_pack_header(self.read)
1880 for _ in range(self._num_objects):
1881 offset = self.offset
1882 unpacked, unused = unpack_object(
1883 self.read,
1884 self.hash_func,
1885 read_some=self.recv,
1886 compute_crc32=compute_crc32,
1887 zlib_bufsize=self._zlib_bufsize,
1888 )
1889 unpacked.offset = offset
1891 # prepend any unused data to current read buffer
1892 buf = BytesIO()
1893 buf.write(unused)
1894 buf.write(self._rbuf.read())
1895 buf.seek(0)
1896 self._rbuf = buf
1898 yield unpacked
1900 if self._buf_len() < self._hash_size:
1901 # If the read buffer is full, then the last read() got the whole
1902 # trailer off the wire. If not, it means there is still some of the
1903 # trailer to read. We need to read() all hash_size bytes; N come from the
1904 # read buffer and (hash_size - N) come from the wire.
1905 self.read(self._hash_size)
1907 pack_sha = bytearray(self._trailer)
1908 if pack_sha != self.sha.digest():
1909 raise ChecksumMismatch(
1910 sha_to_hex(RawObjectID(bytes(pack_sha))), self.sha.hexdigest()
1911 )
1914class PackStreamCopier(PackStreamReader):
1915 """Class to verify a pack stream as it is being read.
1917 The pack is read from a ReceivableProtocol using read() or recv() as
1918 appropriate and written out to the given file-like object.
1919 """
1921 def __init__(
1922 self,
1923 hash_func: Callable[[], "HashObject"],
1924 read_all: Callable[[int], bytes],
1925 read_some: Callable[[int], bytes] | None,
1926 outfile: IO[bytes],
1927 delta_iter: "DeltaChainIterator[UnpackedObject] | None" = None,
1928 ) -> None:
1929 """Initialize the copier.
1931 Args:
1932 hash_func: Hash function to use for computing object IDs
1933 read_all: Read function that blocks until the number of
1934 requested bytes are read.
1935 read_some: Read function that returns at least one byte, but may
1936 not return the number of bytes requested.
1937 outfile: File-like object to write output through.
1938 delta_iter: Optional DeltaChainIterator to record deltas as we
1939 read them.
1940 """
1941 super().__init__(hash_func, read_all, read_some=read_some)
1942 self.outfile = outfile
1943 self._delta_iter = delta_iter
1945 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1946 """Read data from the read callback and write it to the file."""
1947 data = super()._read(read, size)
1948 self.outfile.write(data)
1949 return data
1951 def verify(self, progress: Callable[..., None] | None = None) -> None:
1952 """Verify a pack stream and write it to the output file.
1954 See PackStreamReader.iterobjects for a list of exceptions this may
1955 throw.
1956 """
1957 i = 0 # default count of entries if read_objects() is empty
1958 for i, unpacked in enumerate(self.read_objects()):
1959 if self._delta_iter:
1960 self._delta_iter.record(unpacked)
1961 if progress is not None:
1962 progress(f"copying pack entries: {i}/{len(self)}\r".encode("ascii"))
1963 if progress is not None:
1964 progress(f"copied {i} pack entries\n".encode("ascii"))
1967def obj_sha(
1968 type: int,
1969 chunks: bytes | Iterable[bytes],
1970 hash_func: Callable[[], "HashObject"] = sha1,
1971) -> bytes:
1972 """Compute the SHA for a numeric type and object chunks.
1974 Args:
1975 type: Object type number
1976 chunks: Object data chunks
1977 hash_func: Hash function to use (defaults to sha1)
1979 Returns:
1980 Binary hash digest
1981 """
1982 sha = hash_func()
1983 sha.update(object_header(type, chunks_length(chunks)))
1984 if isinstance(chunks, bytes):
1985 sha.update(chunks)
1986 else:
1987 for chunk in chunks:
1988 sha.update(chunk)
1989 return sha.digest()
1992def compute_file_sha(
1993 f: IO[bytes],
1994 hash_func: Callable[[], "HashObject"],
1995 start_ofs: int = 0,
1996 end_ofs: int = 0,
1997 buffer_size: int = 1 << 16,
1998) -> "HashObject":
1999 """Hash a portion of a file into a new SHA.
2001 Args:
2002 f: A file-like object to read from that supports seek().
2003 hash_func: A callable that returns a new HashObject.
2004 start_ofs: The offset in the file to start reading at.
2005 end_ofs: The offset in the file to end reading at, relative to the
2006 end of the file.
2007 buffer_size: A buffer size for reading.
2008 Returns: A new SHA object updated with data read from the file.
2009 """
2010 sha = hash_func()
2011 f.seek(0, SEEK_END)
2012 length = f.tell()
2013 if start_ofs < 0:
2014 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
2015 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
2016 raise AssertionError(
2017 f"Attempt to read beyond file length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, file length: {length}"
2018 )
2019 todo = length + end_ofs - start_ofs
2020 f.seek(start_ofs)
2021 while todo:
2022 data = f.read(min(todo, buffer_size))
2023 sha.update(data)
2024 todo -= len(data)
2025 return sha
2028def compute_buffer_sha(
2029 contents: "bytes | mmap.mmap",
2030 hash_func: Callable[[], "HashObject"],
2031 start_ofs: int = 0,
2032 end_ofs: int = 0,
2033) -> "HashObject":
2034 """Hash a portion of a buffer into a new SHA.
2036 The region is hashed in one pass through a memoryview, so a mapped pack
2037 is never copied. The view is released before returning rather than left
2038 to the garbage collector, since ``mmap.close()`` raises BufferError while
2039 an export is alive.
2041 Args:
2042 contents: Buffer to hash.
2043 hash_func: A callable that returns a new HashObject.
2044 start_ofs: The offset in the buffer to start hashing at.
2045 end_ofs: The offset to end hashing at, relative to the end of the
2046 buffer.
2047 Returns: A new SHA object updated with data read from the buffer.
2048 """
2049 sha = hash_func()
2050 length = len(contents)
2051 if start_ofs < 0:
2052 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
2053 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
2054 raise AssertionError(
2055 f"Attempt to read beyond buffer length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, buffer length: {length}"
2056 )
2057 with memoryview(contents) as view:
2058 sha.update(view[start_ofs : length + end_ofs])
2059 return sha
2062class PackData:
2063 """The data contained in a packfile.
2065 Pack files can be accessed both sequentially for exploding a pack, and
2066 directly with the help of an index to retrieve a specific object.
2068 The objects within are either complete or a delta against another.
2070 The header is variable length. If the MSB of each byte is set then it
2071 indicates that the subsequent byte is still part of the header.
2072 For the first byte the next MS bits are the type, which tells you the type
2073 of object, and whether it is a delta. The LS byte is the lowest bits of the
2074 size. For each subsequent byte the LS 7 bits are the next MS bits of the
2075 size, i.e. the last byte of the header contains the MS bits of the size.
2077 For the complete objects the data is stored as zlib deflated data.
2078 The size in the header is the uncompressed object size, so to uncompress
2079 you need to just keep feeding data to zlib until you get an object back,
2080 or it errors on bad data. This is done here by reading from the mapped
2081 pack contents starting at the deflated object.
2083 Currently there are no integrity checks done. Also no attempt is made to
2084 try and detect the delta case, or a request for an object at the wrong
2085 position. It will all just throw a zlib or KeyError.
2086 """
2088 def __init__(
2089 self,
2090 filename: str | os.PathLike[str],
2091 object_format: ObjectFormat,
2092 file: IO[bytes] | None = None,
2093 size: int | None = None,
2094 *,
2095 delta_window_size: int | None = None,
2096 window_memory: int | None = None,
2097 delta_cache_size: int | None = None,
2098 depth: int | None = None,
2099 threads: int | None = None,
2100 big_file_threshold: int | None = None,
2101 delta_base_cache_limit: int | None = None,
2102 ) -> None:
2103 """Create a PackData object representing the pack in the given filename.
2105 The file must exist and stay readable until the object is disposed of.
2106 It must also stay the same size. It is mapped into memory on open, so
2107 reads index the mapping directly rather than sharing a file position.
2109 The size argument is not trusted for checksum offsets, which are
2110 derived from the mapped length instead. When given it is checked
2111 against that length, so a caller passing a stale size gets an error
2112 rather than silently wrong offsets.
2113 """
2114 self._filename = filename
2115 self.object_format = object_format
2116 self._header_size = 12
2117 self.delta_window_size = delta_window_size
2118 self.window_memory = window_memory
2119 self.delta_cache_size = delta_cache_size
2120 self.depth = depth
2121 self.threads = threads
2122 self.big_file_threshold = big_file_threshold
2123 self.delta_base_cache_limit = delta_base_cache_limit
2124 self._file: IO[bytes]
2125 self._contents: bytes | mmap.mmap | None = None
2127 if file is None:
2128 self._file = GitFile(self._filename, "rb")
2129 self._close_file = True
2130 else:
2131 # A caller-supplied file stays the caller's to close; it may well
2132 # keep writing to it after we are done reading.
2133 self._file = file
2134 self._close_file = False
2135 try:
2136 # Map the pack once; every read indexes this buffer at an explicit
2137 # offset, so concurrent reads never contend on a file position.
2138 self._contents, self._size = _load_file_contents(self._file)
2139 if size is not None and size != self._size:
2140 raise AssertionError(
2141 f"{self._filename} is {self._size} bytes, but caller said {size}"
2142 )
2143 minimum_size = self._header_size + self.object_format.oid_length
2144 if self._size < minimum_size:
2145 raise AssertionError(
2146 f"{self._filename} is too small for a packfile ({self._size} < {minimum_size})"
2147 )
2148 (_version, self._num_objects) = read_pack_header_at(self._contents)
2150 # Use delta_base_cache_limit, then delta_cache_size, then default
2151 cache_size = (
2152 delta_base_cache_limit
2153 or delta_cache_size
2154 or DEFAULT_DELTA_BASE_CACHE_LIMIT
2155 )
2156 self._init_offset_cache(cache_size)
2157 except BaseException:
2158 self.close()
2159 raise
2161 def _init_offset_cache(self, max_size: int) -> None:
2162 """Initialize the resolved object cache."""
2163 self._offset_cache = LRUSizeCache[int, tuple[int, OldUnpackedObject]](
2164 max_size, compute_size=_compute_object_size
2165 )
2166 # Cache hits update the LRU linked list, so reads need locking too.
2167 self._offset_cache_lock = threading.Lock()
2169 @property
2170 def filename(self) -> str:
2171 """Get the filename of the pack file.
2173 Returns:
2174 Base filename without directory path
2175 """
2176 return os.path.basename(self._filename)
2178 @property
2179 def path(self) -> str | os.PathLike[str]:
2180 """Get the full path of the pack file.
2182 Returns:
2183 Full path to the pack file
2184 """
2185 return self._filename
2187 @classmethod
2188 def from_file(
2189 cls,
2190 file: IO[bytes],
2191 object_format: ObjectFormat,
2192 size: int | None = None,
2193 ) -> "PackData":
2194 """Create a PackData object from an open file.
2196 Args:
2197 file: Open file object
2198 object_format: Object format
2199 size: Optional expected file size, checked against the mapped length
2201 Returns:
2202 PackData instance
2203 """
2204 return cls(str(file), object_format, file=file, size=size)
2206 @classmethod
2207 def from_path(
2208 cls,
2209 path: str | os.PathLike[str],
2210 object_format: ObjectFormat,
2211 ) -> "PackData":
2212 """Create a PackData object from a file path.
2214 Args:
2215 path: Path to the pack file
2216 object_format: Object format
2218 Returns:
2219 PackData instance
2220 """
2221 return cls(filename=path, object_format=object_format)
2223 def _buffer(self) -> "bytes | mmap.mmap":
2224 """Return the mapped pack contents."""
2225 contents = self._contents
2226 if contents is None:
2227 raise ValueError(f"read from closed PackData: {self._filename}")
2228 return contents
2230 def close(self) -> None:
2231 """Release the mapping, and the pack file if we opened it.
2233 Callers must drop the mapping before writing to or renaming the pack:
2234 on Windows a live mapping locks the file.
2235 """
2236 contents = self._contents
2237 self._contents = None
2238 _close_file_contents(contents)
2239 if self._file is not None:
2240 if self._close_file:
2241 self._file.close()
2242 self._file = None # type: ignore
2244 def __del__(self) -> None:
2245 """Ensure pack file is closed when PackData is garbage collected."""
2246 if getattr(self, "_file", None) is not None:
2247 import warnings
2249 warnings.warn(
2250 f"unclosed PackData {self!r}",
2251 ResourceWarning,
2252 stacklevel=2,
2253 source=self,
2254 )
2255 try:
2256 self.close()
2257 except Exception:
2258 # Ignore errors during cleanup
2259 pass
2261 def __enter__(self) -> Self:
2262 """Enter context manager."""
2263 return self
2265 def __exit__(
2266 self,
2267 type: type | None,
2268 value: BaseException | None,
2269 traceback: TracebackType | None,
2270 ) -> None:
2271 """Exit context manager."""
2272 self.close()
2274 def __eq__(self, other: object) -> bool:
2275 """Check equality with another object."""
2276 if isinstance(other, PackData):
2277 return self.get_stored_checksum() == other.get_stored_checksum()
2278 return False
2280 def __len__(self) -> int:
2281 """Returns the number of objects in this pack."""
2282 return self._num_objects
2284 def calculate_checksum(self) -> bytes:
2285 """Calculate the checksum for this pack.
2287 Returns: Binary digest (size depends on hash algorithm)
2288 """
2289 return compute_buffer_sha(
2290 self._buffer(),
2291 hash_func=self.object_format.hash_func,
2292 end_ofs=-self.object_format.oid_length,
2293 ).digest()
2295 def iter_unpacked(self, *, include_comp: bool = False) -> Iterator[UnpackedObject]:
2296 """Iterate over unpacked objects in the pack."""
2297 if self._num_objects is None:
2298 return
2300 contents = self._buffer()
2301 offset = self._header_size
2302 for _ in range(self._num_objects):
2303 unpacked, offset = unpack_object_at(
2304 contents,
2305 offset,
2306 self.object_format.hash_func,
2307 compute_crc32=False,
2308 include_comp=include_comp,
2309 )
2310 yield unpacked
2312 def iterentries(
2313 self,
2314 progress: Callable[[int, int], None] | None = None,
2315 resolve_ext_ref: ResolveExtRefFn | None = None,
2316 ) -> Iterator[PackIndexEntry]:
2317 """Yield entries summarizing the contents of this pack.
2319 Args:
2320 progress: Progress function, called with current and total
2321 object count.
2322 resolve_ext_ref: Optional function to resolve external references
2323 Returns: iterator of tuples with (sha, offset, crc32)
2324 """
2325 num_objects = self._num_objects
2326 indexer = PackIndexer.for_pack_data(self, resolve_ext_ref=resolve_ext_ref)
2327 for i, result in enumerate(indexer):
2328 if progress is not None:
2329 progress(i, num_objects)
2330 yield result
2332 def sorted_entries(
2333 self,
2334 progress: Callable[[int, int], None] | None = None,
2335 resolve_ext_ref: ResolveExtRefFn | None = None,
2336 ) -> list[tuple[RawObjectID, int, int]]:
2337 """Return entries in this pack, sorted by SHA.
2339 Args:
2340 progress: Progress function, called with current and total
2341 object count
2342 resolve_ext_ref: Optional function to resolve external references
2343 Returns: Iterator of tuples with (sha, offset, crc32)
2344 """
2345 return sorted(
2346 self.iterentries(progress=progress, resolve_ext_ref=resolve_ext_ref) # type: ignore
2347 )
2349 def create_index_v1(
2350 self,
2351 filename: str,
2352 progress: Callable[..., None] | None = None,
2353 resolve_ext_ref: ResolveExtRefFn | None = None,
2354 ) -> bytes:
2355 """Create a version 1 file for this data file.
2357 Args:
2358 filename: Index filename.
2359 progress: Progress report function
2360 resolve_ext_ref: Optional function to resolve external references
2361 Returns: Checksum of index file
2362 """
2363 entries = self.sorted_entries(
2364 progress=progress, resolve_ext_ref=resolve_ext_ref
2365 )
2366 checksum = self.calculate_checksum()
2367 with GitFile(filename, "wb") as f:
2368 write_pack_index_v1(
2369 f,
2370 entries,
2371 checksum,
2372 )
2373 return checksum
2375 def create_index_v2(
2376 self,
2377 filename: str,
2378 progress: Callable[..., None] | None = None,
2379 resolve_ext_ref: ResolveExtRefFn | None = None,
2380 ) -> bytes:
2381 """Create a version 2 index file for this data file.
2383 Args:
2384 filename: Index filename.
2385 progress: Progress report function
2386 resolve_ext_ref: Optional function to resolve external references
2387 Returns: Checksum of index file
2388 """
2389 entries = self.sorted_entries(
2390 progress=progress, resolve_ext_ref=resolve_ext_ref
2391 )
2392 with GitFile(filename, "wb") as f:
2393 return write_pack_index_v2(f, entries, self.calculate_checksum())
2395 def create_index_v3(
2396 self,
2397 filename: str,
2398 progress: Callable[..., None] | None = None,
2399 resolve_ext_ref: ResolveExtRefFn | None = None,
2400 hash_format: int | None = None,
2401 ) -> bytes:
2402 """Create a version 3 index file for this data file.
2404 Args:
2405 filename: Index filename.
2406 progress: Progress report function
2407 resolve_ext_ref: Function to resolve external references
2408 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
2409 Returns: Checksum of index file
2410 """
2411 entries = self.sorted_entries(
2412 progress=progress, resolve_ext_ref=resolve_ext_ref
2413 )
2414 with GitFile(filename, "wb") as f:
2415 if hash_format is None:
2416 hash_format = 1 # Default to SHA-1
2417 return write_pack_index_v3(
2418 f, entries, self.calculate_checksum(), hash_format=hash_format
2419 )
2421 def create_index(
2422 self,
2423 filename: str,
2424 progress: Callable[..., None] | None = None,
2425 version: int = 2,
2426 resolve_ext_ref: ResolveExtRefFn | None = None,
2427 hash_format: int | None = None,
2428 ) -> bytes:
2429 """Create an index file for this data file.
2431 Args:
2432 filename: Index filename.
2433 progress: Progress report function
2434 version: Index version (1, 2, or 3)
2435 resolve_ext_ref: Function to resolve external references
2436 hash_format: Hash algorithm identifier for v3 (1 = SHA-1, 2 = SHA-256)
2437 Returns: Checksum of index file
2438 """
2439 if version == 1:
2440 return self.create_index_v1(
2441 filename, progress, resolve_ext_ref=resolve_ext_ref
2442 )
2443 elif version == 2:
2444 return self.create_index_v2(
2445 filename, progress, resolve_ext_ref=resolve_ext_ref
2446 )
2447 elif version == 3:
2448 return self.create_index_v3(
2449 filename,
2450 progress,
2451 resolve_ext_ref=resolve_ext_ref,
2452 hash_format=hash_format,
2453 )
2454 else:
2455 raise ValueError(f"unknown index format {version}")
2457 def get_stored_checksum(self) -> bytes:
2458 """Return the expected checksum stored in this pack."""
2459 checksum_size = self.object_format.oid_length
2460 return bytes(self._buffer()[self._size - checksum_size :])
2462 def check(self) -> None:
2463 """Check the consistency of this pack."""
2464 actual = self.calculate_checksum()
2465 stored = self.get_stored_checksum()
2466 if actual != stored:
2467 raise ChecksumMismatch(stored, actual)
2469 def get_unpacked_object_at(
2470 self, offset: int, *, include_comp: bool = False
2471 ) -> UnpackedObject:
2472 """Given offset in the packfile return a UnpackedObject."""
2473 assert offset >= self._header_size
2474 unpacked, _ = unpack_object_at(
2475 self._buffer(),
2476 offset,
2477 self.object_format.hash_func,
2478 include_comp=include_comp,
2479 )
2480 return unpacked
2482 def _get_cached_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2483 """Return the cached object at offset, or raise KeyError."""
2484 # Hot path: acquire/release directly rather than using the context
2485 # manager, which measurably speeds up cache hits on get_object_at.
2486 self._offset_cache_lock.acquire()
2487 try:
2488 return self._offset_cache[offset]
2489 finally:
2490 self._offset_cache_lock.release()
2492 def _cache_object_at(
2493 self, offset: int, type_num: int, chunks: OldUnpackedObject
2494 ) -> None:
2495 """Cache a resolved object at offset."""
2496 with self._offset_cache_lock:
2497 self._offset_cache[offset] = (type_num, chunks)
2499 def get_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2500 """Given an offset in to the packfile return the object that is there.
2502 Using the associated index the location of an object can be looked up,
2503 and then the packfile can be asked directly for that object using this
2504 function.
2505 """
2506 try:
2507 return self._get_cached_object_at(offset)
2508 except KeyError:
2509 pass
2510 unpacked = self.get_unpacked_object_at(offset, include_comp=False)
2511 return (unpacked.pack_type_num, unpacked._obj())
2514T = TypeVar("T")
2517class DeltaChainIterator(Generic[T]):
2518 """Abstract iterator over pack data based on delta chains.
2520 Each object in the pack is guaranteed to be inflated exactly once,
2521 regardless of how many objects reference it as a delta base. As a result,
2522 memory usage is proportional to the length of the longest delta chain.
2524 Subclasses can override _result to define the result type of the iterator.
2525 By default, results are UnpackedObjects with the following members set:
2527 * offset
2528 * obj_type_num
2529 * obj_chunks
2530 * pack_type_num
2531 * delta_base (for delta types)
2532 * comp_chunks (if _include_comp is True)
2533 * decomp_chunks
2534 * decomp_len
2535 * crc32 (if _compute_crc32 is True)
2536 """
2538 _compute_crc32 = False
2539 _include_comp = False
2541 def __init__(
2542 self,
2543 file_obj: IO[bytes] | None,
2544 hash_func: Callable[[], "HashObject"],
2545 *,
2546 resolve_ext_ref: ResolveExtRefFn | None = None,
2547 object_format: "ObjectFormat | None" = None,
2548 ) -> None:
2549 """Initialize DeltaChainIterator.
2551 Args:
2552 file_obj: File object to read pack data from
2553 hash_func: Hash function to use for computing object IDs
2554 resolve_ext_ref: Optional function to resolve external references
2555 object_format: Optional object format. Required by subclasses
2556 that materialise objects (e.g. PackInflater) when iterating
2557 packs in a non-default hash algorithm such as SHA-256.
2558 """
2559 self._file = file_obj
2560 self._contents: bytes | mmap.mmap | None = None
2561 self.hash_func = hash_func
2562 self._object_format = object_format
2563 self._resolve_ext_ref = resolve_ext_ref
2564 self._pending_ofs: dict[int, list[int]] = defaultdict(list)
2565 self._pending_ref: dict[bytes, list[int]] = defaultdict(list)
2566 self._full_ofs: list[tuple[int, int]] = []
2567 self._ext_refs: list[RawObjectID] = []
2569 @classmethod
2570 def for_pack_data(
2571 cls, pack_data: PackData, resolve_ext_ref: ResolveExtRefFn | None = None
2572 ) -> "DeltaChainIterator[T]":
2573 """Create a DeltaChainIterator from pack data.
2575 Args:
2576 pack_data: PackData object to iterate
2577 resolve_ext_ref: Optional function to resolve external refs
2579 Returns:
2580 DeltaChainIterator instance
2581 """
2582 walker = cls(
2583 None,
2584 pack_data.object_format.hash_func,
2585 resolve_ext_ref=resolve_ext_ref,
2586 object_format=pack_data.object_format,
2587 )
2588 walker.set_pack_data(pack_data)
2589 for unpacked in pack_data.iter_unpacked(include_comp=False):
2590 walker.record(unpacked)
2591 return walker
2593 @classmethod
2594 def for_pack_subset(
2595 cls,
2596 pack: "Pack",
2597 shas: Iterable[ObjectID | RawObjectID],
2598 *,
2599 allow_missing: bool = False,
2600 resolve_ext_ref: ResolveExtRefFn | None = None,
2601 ) -> "DeltaChainIterator[T]":
2602 """Create a DeltaChainIterator for a subset of objects.
2604 Args:
2605 pack: Pack object containing the data
2606 shas: Iterable of object SHAs to include
2607 allow_missing: If True, skip missing objects
2608 resolve_ext_ref: Optional function to resolve external refs
2610 Returns:
2611 DeltaChainIterator instance
2612 """
2613 walker = cls(
2614 None,
2615 pack.object_format.hash_func,
2616 resolve_ext_ref=resolve_ext_ref,
2617 object_format=pack.object_format,
2618 )
2619 walker.set_pack_data(pack.data)
2620 todo = set()
2621 for sha in shas:
2622 try:
2623 off = pack.index.object_offset(sha)
2624 except KeyError:
2625 if not allow_missing:
2626 raise
2627 else:
2628 todo.add(off)
2629 done = set()
2630 while todo:
2631 off = todo.pop()
2632 unpacked = pack.data.get_unpacked_object_at(off)
2633 walker.record(unpacked)
2634 done.add(off)
2635 base_ofs = None
2636 if unpacked.pack_type_num == OFS_DELTA:
2637 assert unpacked.offset is not None
2638 assert unpacked.delta_base is not None
2639 assert isinstance(unpacked.delta_base, int)
2640 base_ofs = unpacked.offset - unpacked.delta_base
2641 elif unpacked.pack_type_num == REF_DELTA:
2642 with suppress(KeyError):
2643 assert isinstance(unpacked.delta_base, bytes)
2644 base_ofs = pack.index.object_offset(
2645 RawObjectID(unpacked.delta_base)
2646 )
2647 if base_ofs is not None and base_ofs not in done:
2648 todo.add(base_ofs)
2649 return walker
2651 def record(self, unpacked: UnpackedObject) -> None:
2652 """Record an unpacked object for later processing.
2654 Args:
2655 unpacked: UnpackedObject to record
2656 """
2657 type_num = unpacked.pack_type_num
2658 offset = unpacked.offset
2659 assert offset is not None
2660 if type_num == OFS_DELTA:
2661 assert unpacked.delta_base is not None
2662 assert isinstance(unpacked.delta_base, int)
2663 base_offset = offset - unpacked.delta_base
2664 self._pending_ofs[base_offset].append(offset)
2665 elif type_num == REF_DELTA:
2666 assert isinstance(unpacked.delta_base, bytes)
2667 self._pending_ref[unpacked.delta_base].append(offset)
2668 else:
2669 self._full_ofs.append((offset, type_num))
2671 def set_pack_data(self, pack_data: PackData) -> None:
2672 """Set the pack data for iteration.
2674 Args:
2675 pack_data: PackData object to use
2676 """
2677 self._file = None
2678 self._contents = pack_data._buffer()
2680 def _walk_all_chains(self) -> Iterator[T]:
2681 for offset, type_num in self._full_ofs:
2682 yield from self._follow_chain(offset, type_num, None)
2683 yield from self._walk_ref_chains()
2684 assert not self._pending_ofs, repr(self._pending_ofs)
2686 def _ensure_no_pending(self) -> None:
2687 if self._pending_ref:
2688 raise UnresolvedDeltas(
2689 [sha_to_hex(RawObjectID(s)) for s in self._pending_ref]
2690 )
2692 def _walk_ref_chains(self) -> Iterator[T]:
2693 if not self._resolve_ext_ref:
2694 self._ensure_no_pending()
2695 return
2697 for base_sha, pending in sorted(self._pending_ref.items()):
2698 if base_sha not in self._pending_ref:
2699 continue
2700 try:
2701 type_num, chunks = self._resolve_ext_ref(RawObjectID(base_sha))
2702 except KeyError:
2703 # Not an external ref, but may depend on one. Either it will
2704 # get popped via a _follow_chain call, or we will raise an
2705 # error below.
2706 continue
2707 self._ext_refs.append(RawObjectID(base_sha))
2708 self._pending_ref.pop(base_sha)
2709 for new_offset in pending:
2710 yield from self._follow_chain(new_offset, type_num, chunks)
2712 self._ensure_no_pending()
2714 def _result(self, unpacked: UnpackedObject) -> T:
2715 raise NotImplementedError
2717 def _resolve_object(
2718 self,
2719 offset: int,
2720 obj_type_num: int,
2721 base_chunks: bytes | list[bytes] | None,
2722 ) -> UnpackedObject:
2723 if self._contents is not None:
2724 unpacked, _ = unpack_object_at(
2725 self._contents,
2726 offset,
2727 self.hash_func,
2728 compute_crc32=self._compute_crc32,
2729 include_comp=self._include_comp,
2730 )
2731 else:
2732 # add_thin_pack may still be writing to this file, so it cannot be
2733 # mapped up front; read through the file position instead.
2734 assert self._file is not None
2735 self._file.seek(offset)
2736 unpacked, _ = unpack_object(
2737 self._file.read,
2738 self.hash_func,
2739 read_some=None,
2740 compute_crc32=self._compute_crc32,
2741 include_comp=self._include_comp,
2742 )
2743 unpacked.offset = offset
2744 if base_chunks is None:
2745 assert unpacked.pack_type_num == obj_type_num
2746 else:
2747 assert unpacked.pack_type_num in DELTA_TYPES
2748 unpacked.obj_type_num = obj_type_num
2749 unpacked.obj_chunks = apply_delta(base_chunks, unpacked.decomp_chunks)
2750 # A delta that resolves to a zero-byte payload for a
2751 # commit/tree/tag is malformed: ``_parse_message`` /
2752 # ``parse_tree`` accept the empty input silently, so without
2753 # this guard a too-short delta could materialise an
2754 # otherwise-valid SHA pointing at an empty commit object
2755 # (which ``git fsck`` rejects). Only blobs may legitimately
2756 # be empty, and an empty blob would never be stored as a
2757 # delta in practice.
2758 # Blob.type_num == 3 (avoid the import cycle).
2759 if obj_type_num != 3 and chunks_length(unpacked.obj_chunks) == 0:
2760 raise ApplyDeltaError(
2761 f"delta resolved to empty payload for type {obj_type_num}"
2762 )
2763 return unpacked
2765 def _follow_chain(
2766 self,
2767 offset: int,
2768 obj_type_num: int,
2769 base_chunks: bytes | list[bytes] | None,
2770 ) -> Iterator[T]:
2771 # Unlike PackData.get_object_at, there is no need to cache offsets as
2772 # this approach by design inflates each object exactly once.
2773 todo = [(offset, obj_type_num, base_chunks)]
2774 while todo:
2775 (offset, obj_type_num, base_chunks) = todo.pop()
2776 unpacked = self._resolve_object(offset, obj_type_num, base_chunks)
2777 yield self._result(unpacked)
2779 assert unpacked.offset is not None
2780 unblocked = chain(
2781 self._pending_ofs.pop(unpacked.offset, []),
2782 self._pending_ref.pop(unpacked.sha(), []),
2783 )
2784 todo.extend(
2785 (new_offset, unpacked.obj_type_num, unpacked.obj_chunks) # type: ignore
2786 for new_offset in unblocked
2787 )
2789 def __iter__(self) -> Iterator[T]:
2790 """Iterate over objects in the pack."""
2791 return self._walk_all_chains()
2793 def ext_refs(self) -> list[RawObjectID]:
2794 """Return external references."""
2795 return self._ext_refs
2798class UnpackedObjectIterator(DeltaChainIterator[UnpackedObject]):
2799 """Delta chain iterator that yield unpacked objects."""
2801 def _result(self, unpacked: UnpackedObject) -> UnpackedObject:
2802 """Return the unpacked object.
2804 Args:
2805 unpacked: The unpacked object
2807 Returns:
2808 The unpacked object unchanged
2809 """
2810 return unpacked
2813class PackIndexer(DeltaChainIterator[PackIndexEntry]):
2814 """Delta chain iterator that yields index entries."""
2816 _compute_crc32 = True
2818 def _result(self, unpacked: UnpackedObject) -> PackIndexEntry:
2819 """Convert unpacked object to pack index entry.
2821 Args:
2822 unpacked: The unpacked object
2824 Returns:
2825 Tuple of (sha, offset, crc32) for index entry
2826 """
2827 assert unpacked.offset is not None
2828 return unpacked.sha(), unpacked.offset, unpacked.crc32
2831class PackInflater(DeltaChainIterator[ShaFile]):
2832 """Delta chain iterator that yields ShaFile objects."""
2834 def _result(self, unpacked: UnpackedObject) -> ShaFile:
2835 """Convert unpacked object to ShaFile.
2837 Args:
2838 unpacked: The unpacked object
2840 Returns:
2841 ShaFile object from the unpacked data
2842 """
2843 assert unpacked.obj_type_num is not None and unpacked.obj_chunks is not None
2844 return ShaFile.from_raw_chunks(
2845 unpacked.obj_type_num,
2846 unpacked.obj_chunks,
2847 object_format=self._object_format,
2848 )
2851class SHA1Reader(BinaryIO):
2852 """Wrapper for file-like object that remembers the SHA1 of its data."""
2854 def __init__(self, f: IO[bytes]) -> None:
2855 """Initialize SHA1Reader.
2857 Args:
2858 f: File-like object to wrap
2859 """
2860 self.f = f
2861 self.sha1 = sha1(b"")
2863 def read(self, size: int = -1) -> bytes:
2864 """Read bytes and update SHA1.
2866 Args:
2867 size: Number of bytes to read, -1 for all
2869 Returns:
2870 Bytes read from file
2871 """
2872 data = self.f.read(size)
2873 self.sha1.update(data)
2874 return data
2876 def check_sha(self, allow_empty: bool = False) -> None:
2877 """Check if the SHA1 matches the expected value.
2879 Args:
2880 allow_empty: Allow empty SHA1 hash
2882 Raises:
2883 ChecksumMismatch: If SHA1 doesn't match
2884 """
2885 stored = self.f.read(20)
2886 # If git option index.skipHash is set the index will be empty
2887 if stored != self.sha1.digest() and (
2888 not allow_empty
2889 or (
2890 len(stored) == 20
2891 and sha_to_hex(RawObjectID(stored))
2892 != b"0000000000000000000000000000000000000000"
2893 )
2894 ):
2895 raise ChecksumMismatch(
2896 self.sha1.hexdigest(),
2897 sha_to_hex(RawObjectID(stored)) if stored else b"",
2898 )
2900 def close(self) -> None:
2901 """Close the underlying file."""
2902 return self.f.close()
2904 def tell(self) -> int:
2905 """Return current file position."""
2906 return self.f.tell()
2908 # BinaryIO abstract methods
2909 def readable(self) -> bool:
2910 """Check if file is readable."""
2911 return True
2913 def writable(self) -> bool:
2914 """Check if file is writable."""
2915 return False
2917 def seekable(self) -> bool:
2918 """Check if file is seekable."""
2919 return getattr(self.f, "seekable", lambda: False)()
2921 def seek(self, offset: int, whence: int = 0) -> int:
2922 """Seek to position in file.
2924 Args:
2925 offset: Position offset
2926 whence: Reference point (0=start, 1=current, 2=end)
2928 Returns:
2929 New file position
2930 """
2931 return self.f.seek(offset, whence)
2933 def flush(self) -> None:
2934 """Flush the file buffer."""
2935 if hasattr(self.f, "flush"):
2936 self.f.flush()
2938 def readline(self, size: int = -1) -> bytes:
2939 """Read a line from the file.
2941 Args:
2942 size: Maximum bytes to read
2944 Returns:
2945 Line read from file
2946 """
2947 return self.f.readline(size)
2949 def readlines(self, hint: int = -1) -> list[bytes]:
2950 """Read all lines from the file.
2952 Args:
2953 hint: Approximate number of bytes to read
2955 Returns:
2956 List of lines
2957 """
2958 return self.f.readlines(hint)
2960 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
2961 """Write multiple lines to the file (not supported)."""
2962 raise UnsupportedOperation("writelines")
2964 def write(self, data: bytes, /) -> int: # type: ignore[override]
2965 """Write data to the file (not supported)."""
2966 raise UnsupportedOperation("write")
2968 def __enter__(self) -> Self:
2969 """Enter context manager."""
2970 return self
2972 def __exit__(
2973 self,
2974 type: type | None,
2975 value: BaseException | None,
2976 traceback: TracebackType | None,
2977 ) -> None:
2978 """Exit context manager and close file."""
2979 self.close()
2981 def __iter__(self) -> "SHA1Reader":
2982 """Return iterator for reading file lines."""
2983 return self
2985 def __next__(self) -> bytes:
2986 """Get next line from file.
2988 Returns:
2989 Next line
2991 Raises:
2992 StopIteration: When no more lines
2993 """
2994 line = self.readline()
2995 if not line:
2996 raise StopIteration
2997 return line
2999 def fileno(self) -> int:
3000 """Return file descriptor number."""
3001 return self.f.fileno()
3003 def isatty(self) -> bool:
3004 """Check if file is a terminal."""
3005 return getattr(self.f, "isatty", lambda: False)()
3007 def truncate(self, size: int | None = None) -> int:
3008 """Not supported for read-only file.
3010 Raises:
3011 UnsupportedOperation: Always raised
3012 """
3013 raise UnsupportedOperation("truncate")
3016class SHA1Writer(BinaryIO):
3017 """Wrapper for file-like object that remembers the SHA1 of its data."""
3019 def __init__(self, f: BinaryIO | IO[bytes]) -> None:
3020 """Initialize SHA1Writer.
3022 Args:
3023 f: File-like object to wrap
3024 """
3025 self.f = f
3026 self.length = 0
3027 self.sha1 = sha1(b"")
3028 self.digest: bytes | None = None
3030 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
3031 """Write data and update SHA1.
3033 Args:
3034 data: Data to write
3036 Returns:
3037 Number of bytes written
3038 """
3039 self.sha1.update(data)
3040 written = self.f.write(data)
3041 self.length += written
3042 return written
3044 def write_sha(self) -> bytes:
3045 """Write the SHA1 digest to the file.
3047 Returns:
3048 The SHA1 digest bytes
3049 """
3050 sha = self.sha1.digest()
3051 assert len(sha) == 20
3052 self.f.write(sha)
3053 self.length += len(sha)
3054 return sha
3056 def close(self) -> None:
3057 """Close the pack file and finalize the SHA."""
3058 self.digest = self.write_sha()
3059 self.f.close()
3061 def offset(self) -> int:
3062 """Get the total number of bytes written.
3064 Returns:
3065 Total bytes written
3066 """
3067 return self.length
3069 def tell(self) -> int:
3070 """Return current file position."""
3071 return self.f.tell()
3073 # BinaryIO abstract methods
3074 def readable(self) -> bool:
3075 """Check if file is readable."""
3076 return False
3078 def writable(self) -> bool:
3079 """Check if file is writable."""
3080 return True
3082 def seekable(self) -> bool:
3083 """Check if file is seekable."""
3084 return getattr(self.f, "seekable", lambda: False)()
3086 def seek(self, offset: int, whence: int = 0) -> int:
3087 """Seek to position in file.
3089 Args:
3090 offset: Position offset
3091 whence: Reference point (0=start, 1=current, 2=end)
3093 Returns:
3094 New file position
3095 """
3096 return self.f.seek(offset, whence)
3098 def flush(self) -> None:
3099 """Flush the file buffer."""
3100 if hasattr(self.f, "flush"):
3101 self.f.flush()
3103 def readline(self, size: int = -1) -> bytes:
3104 """Not supported for write-only file.
3106 Raises:
3107 UnsupportedOperation: Always raised
3108 """
3109 raise UnsupportedOperation("readline")
3111 def readlines(self, hint: int = -1) -> list[bytes]:
3112 """Not supported for write-only file.
3114 Raises:
3115 UnsupportedOperation: Always raised
3116 """
3117 raise UnsupportedOperation("readlines")
3119 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3120 """Write multiple lines to the file.
3122 Args:
3123 lines: Iterable of lines to write
3124 """
3125 for line in lines:
3126 self.write(line)
3128 def read(self, size: int = -1) -> bytes:
3129 """Not supported for write-only file.
3131 Raises:
3132 UnsupportedOperation: Always raised
3133 """
3134 raise UnsupportedOperation("read")
3136 def __enter__(self) -> Self:
3137 """Enter context manager."""
3138 return self
3140 def __exit__(
3141 self,
3142 type: type | None,
3143 value: BaseException | None,
3144 traceback: TracebackType | None,
3145 ) -> None:
3146 """Exit context manager and close file."""
3147 self.f.close()
3149 def __iter__(self) -> "SHA1Writer":
3150 """Return iterator."""
3151 return self
3153 def __next__(self) -> bytes:
3154 """Not supported for write-only file.
3156 Raises:
3157 UnsupportedOperation: Always raised
3158 """
3159 raise UnsupportedOperation("__next__")
3161 def fileno(self) -> int:
3162 """Return file descriptor number."""
3163 return self.f.fileno()
3165 def isatty(self) -> bool:
3166 """Check if file is a terminal."""
3167 return getattr(self.f, "isatty", lambda: False)()
3169 def truncate(self, size: int | None = None) -> int:
3170 """Not supported for write-only file.
3172 Raises:
3173 UnsupportedOperation: Always raised
3174 """
3175 raise UnsupportedOperation("truncate")
3178class HashWriter(BinaryIO):
3179 """Wrapper for file-like object that computes hash of its data.
3181 This is a generic version that works with any hash algorithm.
3182 """
3184 def __init__(
3185 self, f: BinaryIO | IO[bytes], hash_func: Callable[[], "HashObject"]
3186 ) -> None:
3187 """Initialize HashWriter.
3189 Args:
3190 f: File-like object to wrap
3191 hash_func: Hash function (e.g., sha1, sha256)
3192 """
3193 self.f = f
3194 self.length = 0
3195 self.hash_obj = hash_func()
3196 self.digest: bytes | None = None
3198 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
3199 """Write data and update hash.
3201 Args:
3202 data: Data to write
3204 Returns:
3205 Number of bytes written
3206 """
3207 self.hash_obj.update(data)
3208 written = self.f.write(data)
3209 self.length += written
3210 return written
3212 def write_hash(self) -> bytes:
3213 """Write the hash digest to the file.
3215 Returns:
3216 The hash digest bytes
3217 """
3218 digest = self.hash_obj.digest()
3219 self.f.write(digest)
3220 self.length += len(digest)
3221 return digest
3223 def close(self) -> None:
3224 """Close the pack file and finalize the hash."""
3225 self.digest = self.write_hash()
3226 self.f.close()
3228 def offset(self) -> int:
3229 """Get the total number of bytes written.
3231 Returns:
3232 Total bytes written
3233 """
3234 return self.length
3236 def tell(self) -> int:
3237 """Return current file position."""
3238 return self.f.tell()
3240 # BinaryIO abstract methods
3241 def readable(self) -> bool:
3242 """Check if file is readable."""
3243 return False
3245 def writable(self) -> bool:
3246 """Check if file is writable."""
3247 return True
3249 def seekable(self) -> bool:
3250 """Check if file is seekable."""
3251 return getattr(self.f, "seekable", lambda: False)()
3253 def seek(self, offset: int, whence: int = 0) -> int:
3254 """Seek to position in file.
3256 Args:
3257 offset: Position offset
3258 whence: Reference point (0=start, 1=current, 2=end)
3260 Returns:
3261 New file position
3262 """
3263 return self.f.seek(offset, whence)
3265 def flush(self) -> None:
3266 """Flush the file buffer."""
3267 if hasattr(self.f, "flush"):
3268 self.f.flush()
3270 def readline(self, size: int = -1) -> bytes:
3271 """Not supported for write-only file.
3273 Raises:
3274 UnsupportedOperation: Always raised
3275 """
3276 raise UnsupportedOperation("readline")
3278 def readlines(self, hint: int = -1) -> list[bytes]:
3279 """Not supported for write-only file.
3281 Raises:
3282 UnsupportedOperation: Always raised
3283 """
3284 raise UnsupportedOperation("readlines")
3286 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3287 """Write multiple lines to the file.
3289 Args:
3290 lines: Iterable of lines to write
3291 """
3292 for line in lines:
3293 self.write(line)
3295 def read(self, size: int = -1) -> bytes:
3296 """Not supported for write-only file.
3298 Raises:
3299 UnsupportedOperation: Always raised
3300 """
3301 raise UnsupportedOperation("read")
3303 def __enter__(self) -> Self:
3304 """Enter context manager."""
3305 return self
3307 def __exit__(
3308 self,
3309 type: type | None,
3310 value: BaseException | None,
3311 traceback: TracebackType | None,
3312 ) -> None:
3313 """Exit context manager and close file."""
3314 self.close()
3316 def __iter__(self) -> "HashWriter":
3317 """Return iterator."""
3318 return self
3320 def __next__(self) -> bytes:
3321 """Not supported for write-only file.
3323 Raises:
3324 UnsupportedOperation: Always raised
3325 """
3326 raise UnsupportedOperation("__next__")
3328 def fileno(self) -> int:
3329 """Return file descriptor number."""
3330 return self.f.fileno()
3332 def isatty(self) -> bool:
3333 """Check if file is a terminal."""
3334 return getattr(self.f, "isatty", lambda: False)()
3336 def truncate(self, size: int | None = None) -> int:
3337 """Not supported for write-only file.
3339 Raises:
3340 UnsupportedOperation: Always raised
3341 """
3342 raise UnsupportedOperation("truncate")
3345def pack_object_header(
3346 type_num: int,
3347 delta_base: bytes | int | None,
3348 size: int,
3349 object_format: "ObjectFormat",
3350) -> bytearray:
3351 """Create a pack object header for the given object info.
3353 Args:
3354 type_num: Numeric type of the object.
3355 delta_base: Delta base offset or ref, or None for whole objects.
3356 size: Uncompressed object size.
3357 object_format: Object format (hash algorithm) to use.
3358 Returns: A header for a packed object.
3359 """
3360 header = []
3361 c = (type_num << 4) | (size & 15)
3362 size >>= 4
3363 while size:
3364 header.append(c | 0x80)
3365 c = size & 0x7F
3366 size >>= 7
3367 header.append(c)
3368 if type_num == OFS_DELTA:
3369 assert isinstance(delta_base, int)
3370 ret = [delta_base & 0x7F]
3371 delta_base >>= 7
3372 while delta_base:
3373 delta_base -= 1
3374 ret.insert(0, 0x80 | (delta_base & 0x7F))
3375 delta_base >>= 7
3376 header.extend(ret)
3377 elif type_num == REF_DELTA:
3378 assert isinstance(delta_base, bytes)
3379 assert len(delta_base) == object_format.oid_length
3380 header += delta_base
3381 return bytearray(header)
3384def pack_object_chunks(
3385 type: int,
3386 object: list[bytes] | tuple[bytes | int, list[bytes]],
3387 object_format: "ObjectFormat",
3388 *,
3389 compression_level: int = -1,
3390) -> Iterator[bytes]:
3391 """Generate chunks for a pack object.
3393 Args:
3394 type: Numeric type of the object
3395 object: Object to write
3396 object_format: Object format (hash algorithm) to use
3397 compression_level: the zlib compression level
3398 Returns: Chunks
3399 """
3400 if type in DELTA_TYPES:
3401 if isinstance(object, tuple):
3402 delta_base, object = object
3403 else:
3404 raise TypeError("Delta types require a tuple of (delta_base, object)")
3405 else:
3406 delta_base = None
3408 # Convert object to list of bytes chunks
3409 if isinstance(object, bytes):
3410 chunks = [object]
3411 elif isinstance(object, list):
3412 chunks = object
3413 elif isinstance(object, ShaFile):
3414 chunks = object.as_raw_chunks()
3415 else:
3416 # Shouldn't reach here with proper typing
3417 raise TypeError(f"Unexpected object type: {object.__class__.__name__}")
3419 yield bytes(
3420 pack_object_header(
3421 type, delta_base, sum(map(len, chunks)), object_format=object_format
3422 )
3423 )
3424 compressor = zlib.compressobj(level=compression_level)
3425 for data in chunks:
3426 yield compressor.compress(data)
3427 yield compressor.flush()
3430def write_pack_object(
3431 write: Callable[[bytes], int],
3432 type: int,
3433 object: list[bytes] | tuple[bytes | int, list[bytes]],
3434 object_format: "ObjectFormat",
3435 *,
3436 sha: "HashObject | None" = None,
3437 compression_level: int = -1,
3438) -> int:
3439 """Write pack object to a file.
3441 Args:
3442 write: Write function to use
3443 type: Numeric type of the object
3444 object: Object to write
3445 object_format: Object format (hash algorithm) to use
3446 sha: Optional SHA-1 hasher to update
3447 compression_level: the zlib compression level
3448 Returns: CRC32 checksum of the written object
3449 """
3450 crc32 = 0
3451 for chunk in pack_object_chunks(
3452 type, object, compression_level=compression_level, object_format=object_format
3453 ):
3454 write(chunk)
3455 if sha is not None:
3456 sha.update(chunk)
3457 crc32 = binascii.crc32(chunk, crc32)
3458 return crc32 & 0xFFFFFFFF
3461def write_pack(
3462 filename: str,
3463 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3464 object_format: "ObjectFormat",
3465 *,
3466 deltify: bool | None = None,
3467 delta_window_size: int | None = None,
3468 compression_level: int = -1,
3469) -> tuple[bytes, bytes]:
3470 """Write a new pack data file.
3472 Args:
3473 filename: Path to the new pack file (without .pack extension)
3474 objects: Objects to write to the pack
3475 object_format: Object format
3476 delta_window_size: Delta window size
3477 deltify: Whether to deltify pack objects
3478 compression_level: the zlib compression level
3479 Returns: Tuple with checksum of pack file and index file
3480 """
3481 with GitFile(filename + ".pack", "wb") as f:
3482 entries, data_sum = write_pack_objects(
3483 f,
3484 objects,
3485 delta_window_size=delta_window_size,
3486 deltify=deltify,
3487 compression_level=compression_level,
3488 object_format=object_format,
3489 )
3490 entries_list = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
3491 with GitFile(filename + ".idx", "wb") as f:
3492 idx_sha = write_pack_index(f, entries_list, data_sum)
3493 return data_sum, idx_sha
3496def pack_header_chunks(num_objects: int) -> Iterator[bytes]:
3497 """Yield chunks for a pack header."""
3498 yield b"PACK" # Pack header
3499 yield struct.pack(b">L", 2) # Pack version
3500 yield struct.pack(b">L", num_objects) # Number of objects in pack
3503def write_pack_header(
3504 write: Callable[[bytes], int] | IO[bytes], num_objects: int
3505) -> None:
3506 """Write a pack header for the given number of objects."""
3507 if not callable(write):
3508 write_fn: Callable[[bytes], int] = write.write
3509 warnings.warn(
3510 "write_pack_header() now takes a write rather than file argument",
3511 DeprecationWarning,
3512 stacklevel=2,
3513 )
3514 else:
3515 write_fn = write
3516 for chunk in pack_header_chunks(num_objects):
3517 write_fn(chunk)
3520def find_reusable_deltas(
3521 container: PackedObjectContainer,
3522 object_ids: Set[ObjectID],
3523 *,
3524 other_haves: Set[ObjectID] | None = None,
3525 progress: Callable[..., None] | None = None,
3526) -> Iterator[UnpackedObject]:
3527 """Find deltas in a pack that can be reused.
3529 Args:
3530 container: Pack container to search for deltas
3531 object_ids: Set of object IDs to find deltas for
3532 other_haves: Set of other object IDs we have
3533 progress: Optional progress reporting callback
3535 Returns:
3536 Iterator of UnpackedObject entries that can be reused
3537 """
3538 if other_haves is None:
3539 other_haves = set()
3540 reused = 0
3541 for i, unpacked in enumerate(
3542 container.iter_unpacked_subset(
3543 object_ids, allow_missing=True, convert_ofs_delta=True
3544 )
3545 ):
3546 if progress is not None and i % 1000 == 0:
3547 progress(f"checking for reusable deltas: {i}/{len(object_ids)}\r".encode())
3548 if unpacked.pack_type_num == REF_DELTA:
3549 hexsha = sha_to_hex(unpacked.delta_base) # type: ignore
3550 if hexsha in object_ids or hexsha in other_haves:
3551 yield unpacked
3552 reused += 1
3553 if progress is not None:
3554 progress((f"found {reused} deltas to reuse\n").encode())
3557def deltify_pack_objects(
3558 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, bytes | None]],
3559 *,
3560 window_size: int | None = None,
3561 progress: Callable[..., None] | None = None,
3562) -> Iterator[UnpackedObject]:
3563 """Generate deltas for pack objects.
3565 Args:
3566 objects: An iterable of (object, path) tuples to deltify.
3567 window_size: Window size; None for default
3568 progress: Optional progress reporting callback
3569 Returns: Iterator over type_num, object id, delta_base, content
3570 delta_base is None for full text entries
3571 """
3573 def objects_with_hints() -> Iterator[tuple[ShaFile, tuple[int, bytes | None]]]:
3574 for e in objects:
3575 if isinstance(e, ShaFile):
3576 yield (e, (e.type_num, None))
3577 else:
3578 yield (e[0], (e[0].type_num, e[1]))
3580 sorted_objs = sort_objects_for_delta(objects_with_hints())
3581 yield from deltas_from_sorted_objects(
3582 sorted_objs,
3583 window_size=window_size,
3584 progress=progress,
3585 )
3588def sort_objects_for_delta(
3589 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, PackHint | None]],
3590) -> Iterator[tuple[ShaFile, bytes | None]]:
3591 """Sort objects for optimal delta compression.
3593 Args:
3594 objects: Iterator of objects or (object, hint) tuples
3596 Returns:
3597 Iterator of sorted (ShaFile, path) tuples
3598 """
3599 magic = []
3600 for entry in objects:
3601 if isinstance(entry, tuple):
3602 obj, hint = entry
3603 if hint is None:
3604 type_num = None
3605 path = None
3606 else:
3607 (type_num, path) = hint
3608 else:
3609 obj = entry
3610 type_num = None
3611 path = None
3612 magic.append((type_num, path, -obj.raw_length(), obj))
3613 # Build a list of objects ordered by the magic Linus heuristic
3614 # This helps us find good objects to diff against us
3615 magic.sort()
3616 return ((x[3], x[1]) for x in magic)
3619def deltas_from_sorted_objects(
3620 objects: Iterator[tuple[ShaFile, bytes | None]],
3621 window_size: int | None = None,
3622 progress: Callable[..., None] | None = None,
3623) -> Iterator[UnpackedObject]:
3624 """Create deltas from sorted objects.
3626 Args:
3627 objects: Iterator of sorted objects to deltify
3628 window_size: Delta window size; None for default
3629 progress: Optional progress reporting callback
3631 Returns:
3632 Iterator of UnpackedObject entries
3633 """
3634 # TODO(jelmer): Use threads
3635 if window_size is None:
3636 window_size = DEFAULT_PACK_DELTA_WINDOW_SIZE
3638 possible_bases: deque[tuple[bytes, int, bytes]] = deque()
3639 for i, (o, path) in enumerate(objects):
3640 if progress is not None and i % 1000 == 0:
3641 progress((f"generating deltas: {i}\r").encode())
3642 raw = o.as_raw_chunks()
3643 raw_bytes = b"".join(raw) # Join once for efficiency
3644 winner = raw
3645 winner_len = sum(map(len, winner))
3646 winner_base = None
3647 for base_id, base_type_num, base_bytes in possible_bases:
3648 if base_type_num != o.type_num:
3649 continue
3650 delta_len = 0
3651 delta = []
3652 for chunk in create_delta(base_bytes, raw_bytes):
3653 delta_len += len(chunk)
3654 if delta_len >= winner_len:
3655 break
3656 delta.append(chunk)
3657 else:
3658 winner_base = base_id
3659 winner = delta
3660 winner_len = sum(map(len, winner))
3661 yield UnpackedObject(
3662 o.type_num,
3663 sha=o.sha().digest(),
3664 delta_base=winner_base,
3665 decomp_len=winner_len,
3666 decomp_chunks=winner,
3667 )
3668 possible_bases.appendleft((o.sha().digest(), o.type_num, raw_bytes))
3669 while len(possible_bases) > window_size:
3670 possible_bases.pop()
3673def pack_objects_to_data(
3674 objects: Sequence[ShaFile]
3675 | Sequence[tuple[ShaFile, bytes | None]]
3676 | Sequence[tuple[ShaFile, PackHint | None]],
3677 *,
3678 deltify: bool | None = None,
3679 delta_window_size: int | None = None,
3680 ofs_delta: bool = True,
3681 progress: Callable[..., None] | None = None,
3682) -> tuple[int, Iterator[UnpackedObject]]:
3683 """Create pack data from objects.
3685 Args:
3686 objects: Pack objects
3687 deltify: Whether to deltify pack objects
3688 delta_window_size: Delta window size
3689 ofs_delta: Whether to use offset deltas
3690 progress: Optional progress reporting callback
3691 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3692 """
3693 count = len(objects)
3694 if deltify is None:
3695 # PERFORMANCE/TODO(jelmer): This should be enabled but the python
3696 # implementation is *much* too slow at the moment.
3697 # Maybe consider enabling it just if the rust extension is available?
3698 deltify = False
3699 if deltify:
3700 return (
3701 count,
3702 deltify_pack_objects(
3703 iter(objects), # type: ignore
3704 window_size=delta_window_size,
3705 progress=progress,
3706 ),
3707 )
3708 else:
3710 def iter_without_path() -> Iterator[UnpackedObject]:
3711 for o in objects:
3712 if isinstance(o, tuple):
3713 yield full_unpacked_object(o[0])
3714 else:
3715 yield full_unpacked_object(o)
3717 return (count, iter_without_path())
3720def generate_unpacked_objects(
3721 container: PackedObjectContainer,
3722 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3723 delta_window_size: int | None = None,
3724 deltify: bool | None = None,
3725 reuse_deltas: bool = True,
3726 ofs_delta: bool = True,
3727 other_haves: set[ObjectID] | None = None,
3728 progress: Callable[..., None] | None = None,
3729) -> Iterator[UnpackedObject]:
3730 """Create pack data from objects.
3732 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3733 """
3734 todo = dict(object_ids)
3735 if reuse_deltas:
3736 for unpack in find_reusable_deltas(
3737 container, set(todo), other_haves=other_haves, progress=progress
3738 ):
3739 del todo[sha_to_hex(RawObjectID(unpack.sha()))]
3740 yield unpack
3741 if deltify is None:
3742 # PERFORMANCE/TODO(jelmer): This should be enabled but is *much* too
3743 # slow at the moment.
3744 deltify = False
3745 if deltify:
3746 objects_to_delta = container.iterobjects_subset(
3747 todo.keys(), allow_missing=False
3748 )
3749 sorted_objs = sort_objects_for_delta((o, todo[o.id]) for o in objects_to_delta)
3750 yield from deltas_from_sorted_objects(
3751 sorted_objs,
3752 window_size=delta_window_size,
3753 progress=progress,
3754 )
3755 else:
3756 for oid in todo:
3757 yield full_unpacked_object(container[oid])
3760def full_unpacked_object(o: ShaFile) -> UnpackedObject:
3761 """Create an UnpackedObject from a ShaFile.
3763 Args:
3764 o: ShaFile object to convert
3766 Returns:
3767 UnpackedObject with full object data
3768 """
3769 return UnpackedObject(
3770 o.type_num,
3771 delta_base=None,
3772 crc32=None,
3773 decomp_chunks=o.as_raw_chunks(),
3774 sha=o.sha().digest(),
3775 )
3778def write_pack_from_container(
3779 write: Callable[[bytes], None]
3780 | Callable[[bytes | bytearray | memoryview], int]
3781 | IO[bytes],
3782 container: PackedObjectContainer,
3783 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3784 object_format: "ObjectFormat",
3785 *,
3786 delta_window_size: int | None = None,
3787 deltify: bool | None = None,
3788 reuse_deltas: bool = True,
3789 compression_level: int = -1,
3790 other_haves: set[ObjectID] | None = None,
3791) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3792 """Write a new pack data file.
3794 Args:
3795 write: write function to use
3796 container: PackedObjectContainer
3797 object_ids: Sequence of (object_id, hint) tuples to write
3798 object_format: Object format (hash algorithm) to use
3799 delta_window_size: Sliding window size for searching for deltas;
3800 Set to None for default window size.
3801 deltify: Whether to deltify objects
3802 reuse_deltas: Whether to reuse existing deltas
3803 compression_level: the zlib compression level to use
3804 other_haves: Set of additional object IDs the receiver has
3805 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3806 """
3807 pack_contents_count = len(object_ids)
3808 pack_contents = generate_unpacked_objects(
3809 container,
3810 object_ids,
3811 delta_window_size=delta_window_size,
3812 deltify=deltify,
3813 reuse_deltas=reuse_deltas,
3814 other_haves=other_haves,
3815 )
3817 return write_pack_data(
3818 write,
3819 pack_contents,
3820 num_records=pack_contents_count,
3821 compression_level=compression_level,
3822 object_format=object_format,
3823 )
3826def write_pack_objects(
3827 write: Callable[[bytes], None] | IO[bytes],
3828 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3829 object_format: "ObjectFormat",
3830 *,
3831 delta_window_size: int | None = None,
3832 deltify: bool | None = None,
3833 compression_level: int = -1,
3834) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3835 """Write a new pack data file.
3837 Args:
3838 write: write function to use
3839 objects: Sequence of (object, path) tuples to write
3840 object_format: Object format (hash algorithm) to use
3841 delta_window_size: Sliding window size for searching for deltas;
3842 Set to None for default window size.
3843 deltify: Whether to deltify objects
3844 compression_level: the zlib compression level to use
3845 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3846 """
3847 pack_contents_count, pack_contents = pack_objects_to_data(objects, deltify=deltify)
3849 return write_pack_data(
3850 write,
3851 pack_contents,
3852 num_records=pack_contents_count,
3853 compression_level=compression_level,
3854 object_format=object_format,
3855 )
3858class PackChunkGenerator:
3859 """Generator for pack data chunks."""
3861 def __init__(
3862 self,
3863 object_format: "ObjectFormat",
3864 num_records: int | None = None,
3865 records: Iterator[UnpackedObject] | None = None,
3866 progress: Callable[..., None] | None = None,
3867 compression_level: int = -1,
3868 reuse_compressed: bool = True,
3869 ) -> None:
3870 """Initialize PackChunkGenerator.
3872 Args:
3873 num_records: Expected number of records
3874 records: Iterator of pack records
3875 progress: Optional progress callback
3876 compression_level: Compression level (-1 for default)
3877 reuse_compressed: Whether to reuse compressed chunks
3878 object_format: Object format (hash algorithm) to use
3879 """
3880 self.object_format = object_format
3881 self.cs = object_format.new_hash()
3882 self.entries: dict[bytes, tuple[int, int]] = {}
3883 if records is None:
3884 records = iter([]) # Empty iterator if None
3885 self._it = self._pack_data_chunks(
3886 records=records,
3887 num_records=num_records,
3888 progress=progress,
3889 compression_level=compression_level,
3890 reuse_compressed=reuse_compressed,
3891 )
3893 def sha1digest(self) -> bytes:
3894 """Return the SHA1 digest of the pack data."""
3895 return self.cs.digest()
3897 def __iter__(self) -> Iterator[bytes]:
3898 """Iterate over pack data chunks."""
3899 return self._it
3901 def _pack_data_chunks(
3902 self,
3903 records: Iterator[UnpackedObject],
3904 *,
3905 num_records: int | None = None,
3906 progress: Callable[..., None] | None = None,
3907 compression_level: int = -1,
3908 reuse_compressed: bool = True,
3909 ) -> Iterator[bytes]:
3910 """Iterate pack data file chunks.
3912 Args:
3913 records: Iterator over UnpackedObject
3914 num_records: Number of records (defaults to len(records) if not specified)
3915 progress: Function to report progress to
3916 compression_level: the zlib compression level
3917 reuse_compressed: Whether to reuse compressed chunks
3918 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3919 """
3920 # Write the pack
3921 if num_records is None:
3922 num_records = len(records) # type: ignore
3923 offset = 0
3924 for chunk in pack_header_chunks(num_records):
3925 yield chunk
3926 self.cs.update(chunk)
3927 offset += len(chunk)
3928 actual_num_records = 0
3929 for i, unpacked in enumerate(records):
3930 type_num = unpacked.pack_type_num
3931 if progress is not None and i % 1000 == 0:
3932 progress((f"writing pack data: {i}/{num_records}\r").encode("ascii"))
3933 raw: list[bytes] | tuple[int, list[bytes]] | tuple[bytes, list[bytes]]
3934 if unpacked.delta_base is not None:
3935 assert isinstance(unpacked.delta_base, bytes), (
3936 f"Expected bytes, got {type(unpacked.delta_base)}"
3937 )
3938 try:
3939 base_offset, _base_crc32 = self.entries[unpacked.delta_base]
3940 except KeyError:
3941 type_num = REF_DELTA
3942 assert isinstance(unpacked.delta_base, bytes)
3943 raw = (unpacked.delta_base, unpacked.decomp_chunks)
3944 else:
3945 type_num = OFS_DELTA
3946 raw = (offset - base_offset, unpacked.decomp_chunks)
3947 else:
3948 raw = unpacked.decomp_chunks
3949 chunks: list[bytes] | Iterator[bytes]
3950 if unpacked.comp_chunks is not None and reuse_compressed:
3951 chunks = unpacked.comp_chunks
3952 else:
3953 chunks = pack_object_chunks(
3954 type_num,
3955 raw,
3956 compression_level=compression_level,
3957 object_format=self.object_format,
3958 )
3959 crc32 = 0
3960 object_size = 0
3961 for chunk in chunks:
3962 yield chunk
3963 crc32 = binascii.crc32(chunk, crc32)
3964 self.cs.update(chunk)
3965 object_size += len(chunk)
3966 actual_num_records += 1
3967 self.entries[unpacked.sha()] = (offset, crc32)
3968 offset += object_size
3969 if actual_num_records != num_records:
3970 raise AssertionError(
3971 f"actual records written differs: {actual_num_records} != {num_records}"
3972 )
3974 yield self.cs.digest()
3977def write_pack_data(
3978 write: Callable[[bytes], None]
3979 | Callable[[bytes | bytearray | memoryview], int]
3980 | IO[bytes],
3981 records: Iterator[UnpackedObject],
3982 object_format: "ObjectFormat",
3983 *,
3984 num_records: int | None = None,
3985 progress: Callable[..., None] | None = None,
3986 compression_level: int = -1,
3987) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3988 """Write a new pack data file.
3990 Args:
3991 write: Write function to use
3992 num_records: Number of records (defaults to len(records) if None)
3993 records: Iterator over type_num, object_id, delta_base, raw
3994 object_format: Object format (hash algorithm) to use
3995 progress: Function to report progress to
3996 compression_level: the zlib compression level
3997 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3998 """
3999 chunk_generator = PackChunkGenerator(
4000 num_records=num_records,
4001 records=records,
4002 progress=progress,
4003 compression_level=compression_level,
4004 object_format=object_format,
4005 )
4006 for chunk in chunk_generator:
4007 if callable(write):
4008 write(chunk)
4009 else:
4010 write.write(chunk)
4011 return chunk_generator.entries, chunk_generator.sha1digest()
4014def write_pack_index_v1(
4015 f: IO[bytes],
4016 entries: Iterable[tuple[bytes, int, int | None]],
4017 pack_checksum: bytes,
4018) -> bytes:
4019 """Write a new pack index file.
4021 Args:
4022 f: A file-like object to write to
4023 entries: List of tuples with object name (sha), offset_in_pack,
4024 and crc32_checksum.
4025 pack_checksum: Checksum of the pack file.
4026 Returns: The SHA of the written index file
4027 """
4028 f = SHA1Writer(f)
4029 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4030 for name, _offset, _entry_checksum in entries:
4031 fan_out_table[ord(name[:1])] += 1
4032 # Fan-out table
4033 for i in range(0x100):
4034 f.write(struct.pack(">L", fan_out_table[i]))
4035 fan_out_table[i + 1] += fan_out_table[i]
4036 for name, offset, _entry_checksum in entries:
4037 if len(name) != 20:
4038 raise TypeError("pack index v1 only supports SHA-1 names")
4039 if not (offset <= 0xFFFFFFFF):
4040 raise TypeError("pack format 1 only supports offsets < 2Gb")
4041 f.write(struct.pack(">L20s", offset, name))
4042 assert len(pack_checksum) == 20
4043 f.write(pack_checksum)
4044 return f.write_sha()
4047def _delta_encode_size(size: int) -> bytes:
4048 ret = bytearray()
4049 c = size & 0x7F
4050 size >>= 7
4051 while size:
4052 ret.append(c | 0x80)
4053 c = size & 0x7F
4054 size >>= 7
4055 ret.append(c)
4056 return bytes(ret)
4059# The length of delta compression copy operations in version 2 packs is limited
4060# to 64K. To copy more, we use several copy operations. Version 3 packs allow
4061# 24-bit lengths in copy operations, but we always make version 2 packs.
4062_MAX_COPY_LEN = 0xFFFF
4065def _encode_copy_operation(start: int, length: int) -> bytes:
4066 scratch = bytearray([0x80])
4067 for i in range(4):
4068 if start & 0xFF << i * 8:
4069 scratch.append((start >> i * 8) & 0xFF)
4070 scratch[0] |= 1 << i
4071 for i in range(2):
4072 if length & 0xFF << i * 8:
4073 scratch.append((length >> i * 8) & 0xFF)
4074 scratch[0] |= 1 << (4 + i)
4075 return bytes(scratch)
4078def _create_delta_py(
4079 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
4080) -> Iterator[bytes]:
4081 """Use python difflib to work out how to transform base_buf to target_buf.
4083 Args:
4084 base_buf: Base buffer
4085 target_buf: Target buffer
4086 """
4087 if isinstance(base_buf, list):
4088 base_buf = b"".join(base_buf)
4089 if isinstance(target_buf, list):
4090 target_buf = b"".join(target_buf)
4091 # write delta header
4092 yield _delta_encode_size(len(base_buf))
4093 yield _delta_encode_size(len(target_buf))
4094 # write out delta opcodes
4095 seq = SequenceMatcher(isjunk=None, a=base_buf, b=target_buf)
4096 for opcode, i1, i2, j1, j2 in seq.get_opcodes():
4097 # Git patch opcodes don't care about deletes!
4098 # if opcode == 'replace' or opcode == 'delete':
4099 # pass
4100 if opcode == "equal":
4101 # If they are equal, unpacker will use data from base_buf
4102 # Write out an opcode that says what range to use
4103 copy_start = i1
4104 copy_len = i2 - i1
4105 while copy_len > 0:
4106 to_copy = min(copy_len, _MAX_COPY_LEN)
4107 yield _encode_copy_operation(copy_start, to_copy)
4108 copy_start += to_copy
4109 copy_len -= to_copy
4110 if opcode == "replace" or opcode == "insert":
4111 # If we are replacing a range or adding one, then we just
4112 # output it to the stream (prefixed by its size)
4113 s = j2 - j1
4114 o = j1
4115 while s > 127:
4116 yield bytes([127])
4117 yield bytes(memoryview(target_buf)[o : o + 127])
4118 s -= 127
4119 o += 127
4120 yield bytes([s])
4121 yield bytes(memoryview(target_buf)[o : o + s])
4124# Default to pure Python implementation
4125create_delta = _create_delta_py
4128def apply_delta(
4129 src_buf: bytes | list[bytes], delta: bytes | list[bytes]
4130) -> list[bytes]:
4131 """Based on the similar function in git's patch-delta.c.
4133 Args:
4134 src_buf: Source buffer
4135 delta: Delta instructions
4136 """
4137 if not isinstance(src_buf, bytes):
4138 src_buf = b"".join(src_buf)
4139 if not isinstance(delta, bytes):
4140 delta = b"".join(delta)
4141 out = []
4142 index = 0
4143 delta_length = len(delta)
4145 def get_delta_header_size(delta: bytes, index: int) -> tuple[int, int]:
4146 size = 0
4147 i = 0
4148 while True:
4149 # Bound-check explicitly: ``delta[index:index+1]`` silently
4150 # returns b"" past the end, which would crash with TypeError
4151 # in ``ord`` and leave the caller unable to distinguish a
4152 # truncated delta from a programming bug.
4153 if index >= delta_length:
4154 raise ApplyDeltaError("delta truncated in size header")
4155 cmd = ord(delta[index : index + 1])
4156 index += 1
4157 size |= (cmd & ~0x80) << i
4158 i += 7
4159 if not cmd & 0x80:
4160 break
4161 return size, index
4163 def read_byte(delta: bytes) -> int:
4164 nonlocal index
4165 # Bound-check explicitly: ``delta[index:index+1]`` silently returns
4166 # b"" past the end, which would crash with TypeError in ``ord`` and
4167 # leave the caller unable to distinguish a truncated delta from a
4168 # programming bug.
4169 if index >= delta_length:
4170 raise ApplyDeltaError("delta truncated in copy op")
4171 index += 1
4172 return ord(delta[index - 1 : index])
4174 src_size, index = get_delta_header_size(delta, index)
4175 dest_size, index = get_delta_header_size(delta, index)
4176 if src_size != len(src_buf):
4177 raise ApplyDeltaError(
4178 f"Unexpected source buffer size: {src_size} vs {len(src_buf)}"
4179 )
4180 while index < delta_length:
4181 cmd = ord(delta[index : index + 1])
4182 index += 1
4183 if cmd & 0x80:
4184 cp_off = 0
4185 for i in range(4):
4186 if cmd & (1 << i):
4187 x = read_byte(delta)
4188 cp_off |= x << (i * 8)
4189 cp_size = 0
4190 # Version 3 packs can contain copy sizes larger than 64K.
4191 for i in range(3):
4192 if cmd & (1 << (4 + i)):
4193 x = read_byte(delta)
4194 cp_size |= x << (i * 8)
4195 if cp_size == 0:
4196 cp_size = 0x10000
4197 if (
4198 cp_off + cp_size < cp_size
4199 or cp_off + cp_size > src_size
4200 or cp_size > dest_size
4201 ):
4202 break
4203 out.append(src_buf[cp_off : cp_off + cp_size])
4204 elif cmd != 0:
4205 if index + cmd > delta_length:
4206 raise ApplyDeltaError("delta truncated in insert op")
4207 out.append(delta[index : index + cmd])
4208 index += cmd
4209 else:
4210 raise ApplyDeltaError("Invalid opcode 0")
4212 if index != delta_length:
4213 raise ApplyDeltaError(f"delta not empty: {delta[index:]!r}")
4215 if dest_size != chunks_length(out):
4216 raise ApplyDeltaError("dest size incorrect")
4218 return out
4221def write_pack_index_v2(
4222 f: IO[bytes],
4223 entries: Iterable[tuple[bytes, int, int | None]],
4224 pack_checksum: bytes,
4225) -> bytes:
4226 """Write a new pack index file.
4228 Args:
4229 f: File-like object to write to
4230 entries: List of tuples with object name (sha), offset_in_pack, and
4231 crc32_checksum.
4232 pack_checksum: Checksum of the pack file.
4233 Returns: The checksum of the index file written
4234 """
4235 # Determine hash algorithm from pack_checksum length
4236 if len(pack_checksum) == 20:
4237 hash_func = sha1
4238 elif len(pack_checksum) == 32:
4239 hash_func = sha256
4240 else:
4241 raise ValueError(f"Unsupported pack checksum length: {len(pack_checksum)}")
4243 f_writer = HashWriter(f, hash_func)
4244 f_writer.write(b"\377tOc") # Magic!
4245 f_writer.write(struct.pack(">L", 2))
4247 # Convert to list to allow multiple iterations
4248 entries_list = list(entries)
4250 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4251 for name, offset, entry_checksum in entries_list:
4252 fan_out_table[ord(name[:1])] += 1
4254 if entries_list:
4255 hash_size = len(entries_list[0][0])
4256 else:
4257 hash_size = len(pack_checksum) # Use pack_checksum length as hash size
4259 # Fan-out table
4260 largetable: list[int] = []
4261 for i in range(0x100):
4262 f_writer.write(struct.pack(b">L", fan_out_table[i]))
4263 fan_out_table[i + 1] += fan_out_table[i]
4264 for name, offset, entry_checksum in entries_list:
4265 if len(name) != hash_size:
4266 raise TypeError(
4267 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4268 )
4269 f_writer.write(name)
4270 for name, offset, entry_checksum in entries_list:
4271 f_writer.write(struct.pack(b">L", entry_checksum))
4272 for name, offset, entry_checksum in entries_list:
4273 if offset < 2**31:
4274 f_writer.write(struct.pack(b">L", offset))
4275 else:
4276 f_writer.write(struct.pack(b">L", 2**31 + len(largetable)))
4277 largetable.append(offset)
4278 for offset in largetable:
4279 f_writer.write(struct.pack(b">Q", offset))
4280 f_writer.write(pack_checksum)
4281 return f_writer.write_hash()
4284def write_pack_index_v3(
4285 f: IO[bytes],
4286 entries: Iterable[tuple[bytes, int, int | None]],
4287 pack_checksum: bytes,
4288 hash_format: int = 1,
4289) -> bytes:
4290 """Write a new pack index file in v3 format.
4292 Args:
4293 f: File-like object to write to
4294 entries: List of tuples with object name (sha), offset_in_pack, and
4295 crc32_checksum.
4296 pack_checksum: Checksum of the pack file.
4297 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
4298 Returns: The SHA of the index file written
4299 """
4300 if hash_format == 1:
4301 hash_size = 20 # SHA-1
4302 writer_cls = SHA1Writer
4303 elif hash_format == 2:
4304 hash_size = 32 # SHA-256
4305 # TODO: Add SHA256Writer when SHA-256 support is implemented
4306 raise NotImplementedError("SHA-256 support not yet implemented")
4307 else:
4308 raise ValueError(f"Unknown hash algorithm {hash_format}")
4310 # Convert entries to list to allow multiple iterations
4311 entries_list = list(entries)
4313 # Calculate shortest unambiguous prefix length for object names
4314 # For now, use full hash size (this could be optimized)
4315 shortened_oid_len = hash_size
4317 f = writer_cls(f)
4318 f.write(b"\377tOc") # Magic!
4319 f.write(struct.pack(">L", 3)) # Version 3
4320 f.write(struct.pack(">L", hash_format)) # Hash algorithm
4321 f.write(struct.pack(">L", shortened_oid_len)) # Shortened OID length
4323 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4324 for name, offset, entry_checksum in entries_list:
4325 if len(name) != hash_size:
4326 raise ValueError(
4327 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4328 )
4329 fan_out_table[ord(name[:1])] += 1
4331 # Fan-out table
4332 largetable: list[int] = []
4333 for i in range(0x100):
4334 f.write(struct.pack(b">L", fan_out_table[i]))
4335 fan_out_table[i + 1] += fan_out_table[i]
4337 # Object names table
4338 for name, offset, entry_checksum in entries_list:
4339 f.write(name)
4341 # CRC32 checksums table
4342 for name, offset, entry_checksum in entries_list:
4343 f.write(struct.pack(b">L", entry_checksum))
4345 # Offset table
4346 for name, offset, entry_checksum in entries_list:
4347 if offset < 2**31:
4348 f.write(struct.pack(b">L", offset))
4349 else:
4350 f.write(struct.pack(b">L", 2**31 + len(largetable)))
4351 largetable.append(offset)
4353 # Large offset table
4354 for offset in largetable:
4355 f.write(struct.pack(b">Q", offset))
4357 assert len(pack_checksum) == hash_size, (
4358 f"Pack checksum has wrong length: expected {hash_size}, got {len(pack_checksum)}"
4359 )
4360 f.write(pack_checksum)
4361 return f.write_sha()
4364def write_pack_index(
4365 f: IO[bytes],
4366 entries: Iterable[tuple[bytes, int, int | None]],
4367 pack_checksum: bytes,
4368 progress: Callable[..., None] | None = None,
4369 version: int | None = None,
4370) -> bytes:
4371 """Write a pack index file.
4373 Args:
4374 f: File-like object to write to.
4375 entries: List of (checksum, offset, crc32) tuples
4376 pack_checksum: Checksum of the pack file.
4377 progress: Progress function (not currently used)
4378 version: Pack index version to use (1, 2, or 3). If None, defaults to DEFAULT_PACK_INDEX_VERSION.
4380 Returns:
4381 SHA of the written index file
4383 Raises:
4384 ValueError: If an unsupported version is specified
4385 """
4386 if version is None:
4387 version = DEFAULT_PACK_INDEX_VERSION
4389 if version == 1:
4390 return write_pack_index_v1(f, entries, pack_checksum)
4391 elif version == 2:
4392 return write_pack_index_v2(f, entries, pack_checksum)
4393 elif version == 3:
4394 return write_pack_index_v3(f, entries, pack_checksum)
4395 else:
4396 raise ValueError(f"Unsupported pack index version: {version}")
4399class Pack:
4400 """A Git pack object."""
4402 _data_load: Callable[[], PackData] | None
4403 _idx_load: Callable[[], PackIndex] | None
4405 _data: PackData | None
4406 _idx: PackIndex | None
4407 _bitmap: "PackBitmap | None"
4409 def __init__(
4410 self,
4411 basename: str,
4412 *,
4413 object_format: ObjectFormat,
4414 resolve_ext_ref: ResolveExtRefFn | None = None,
4415 delta_window_size: int | None = None,
4416 window_memory: int | None = None,
4417 delta_cache_size: int | None = None,
4418 depth: int | None = None,
4419 threads: int | None = None,
4420 big_file_threshold: int | None = None,
4421 delta_base_cache_limit: int | None = None,
4422 ) -> None:
4423 """Initialize a Pack object.
4425 Args:
4426 basename: Base path for pack files (without .pack/.idx extension)
4427 object_format: Hash algorithm used by the repository
4428 resolve_ext_ref: Optional function to resolve external references
4429 delta_window_size: Size of the delta compression window
4430 window_memory: Memory limit for delta compression window
4431 delta_cache_size: Size of the delta cache
4432 depth: Maximum depth for delta chains
4433 threads: Number of threads to use for operations
4434 big_file_threshold: Size threshold for big file handling
4435 delta_base_cache_limit: Maximum bytes for delta base object cache
4436 """
4437 self._basename = basename
4438 self.object_format = object_format
4439 self._data = None
4440 self._idx = None
4441 self._bitmap = None
4442 self._idx_path = self._basename + ".idx"
4443 self._data_path = self._basename + ".pack"
4444 self._bitmap_path = self._basename + ".bitmap"
4445 self.delta_window_size = delta_window_size
4446 self.window_memory = window_memory
4447 self.delta_cache_size = delta_cache_size
4448 self.depth = depth
4449 self.threads = threads
4450 self.big_file_threshold = big_file_threshold
4451 self.delta_base_cache_limit = delta_base_cache_limit
4452 self._idx_load = lambda: load_pack_index(self._idx_path, object_format)
4453 self._data_load = lambda: PackData(
4454 self._data_path,
4455 delta_window_size=delta_window_size,
4456 window_memory=window_memory,
4457 delta_cache_size=delta_cache_size,
4458 depth=depth,
4459 threads=threads,
4460 big_file_threshold=big_file_threshold,
4461 delta_base_cache_limit=delta_base_cache_limit,
4462 object_format=object_format,
4463 )
4464 self.resolve_ext_ref = resolve_ext_ref
4466 @classmethod
4467 def from_lazy_objects(
4468 cls,
4469 data_fn: Callable[[], PackData],
4470 idx_fn: Callable[[], PackIndex],
4471 ) -> "Pack":
4472 """Create a new pack object from callables to load pack data and index objects."""
4473 # Load index to get object format
4474 idx = idx_fn()
4475 ret = cls("", object_format=idx.object_format)
4476 ret._data_load = data_fn
4477 ret._idx = idx
4478 ret._idx_load = None
4479 return ret
4481 @classmethod
4482 def from_objects(cls, data: PackData, idx: PackIndex) -> "Pack":
4483 """Create a new pack object from pack data and index objects."""
4484 ret = cls("", object_format=idx.object_format)
4485 ret._data = data
4486 ret._data_load = None
4487 ret._idx = idx
4488 ret._idx_load = None
4489 ret.check_length_and_checksum()
4490 return ret
4492 def name(self) -> bytes:
4493 """The SHA over the SHAs of the objects in this pack."""
4494 return self.index.objects_sha1()
4496 @property
4497 def data(self) -> PackData:
4498 """The pack data object being used."""
4499 if self._data is None:
4500 assert self._data_load
4501 try:
4502 self._data = self._data_load()
4503 except FileNotFoundError as exc:
4504 raise PackFileDisappeared(self) from exc
4505 self.check_length_and_checksum()
4506 return self._data
4508 @property
4509 def index(self) -> PackIndex:
4510 """The index being used.
4512 Note: This may be an in-memory index
4513 """
4514 if self._idx is None:
4515 assert self._idx_load
4516 try:
4517 self._idx = self._idx_load()
4518 except FileNotFoundError as exc:
4519 raise PackFileDisappeared(self) from exc
4520 return self._idx
4522 @property
4523 def bitmap(self) -> "PackBitmap | None":
4524 """The bitmap being used, if available.
4526 Returns:
4527 PackBitmap instance, or None if no bitmap exists or the bitmap
4528 was built for a different pack
4530 Raises:
4531 ValueError: If bitmap file is invalid or corrupt
4532 """
4533 if self._bitmap is None:
4534 from .bitmap import read_bitmap
4536 try:
4537 self._bitmap = read_bitmap(
4538 self._bitmap_path,
4539 pack_index=self.index,
4540 pack_checksum=self.get_stored_checksum(),
4541 )
4542 except ChecksumMismatch:
4543 # The bitmap records the checksum of the pack it was built for.
4544 # A mismatch means it is stale or was swapped in from another
4545 # pack, so its positions no longer describe this pack's objects.
4546 # Ignore it and let callers fall back to graph traversal, the
4547 # same as git.
4548 logger.warning(
4549 "Ignoring bitmap %s: checksum does not match pack",
4550 self._bitmap_path,
4551 )
4552 return None
4553 return self._bitmap
4555 def ensure_bitmap(
4556 self,
4557 object_store: "BaseObjectStore",
4558 refs: dict["Ref", "ObjectID"],
4559 commit_interval: int | None = None,
4560 progress: Callable[[str], None] | None = None,
4561 ) -> "PackBitmap":
4562 """Ensure a bitmap exists for this pack, generating one if needed.
4564 Args:
4565 object_store: Object store to read objects from
4566 refs: Dictionary of ref names to commit SHAs
4567 commit_interval: Include every Nth commit in bitmap index
4568 progress: Optional progress reporting callback
4570 Returns:
4571 PackBitmap instance (either existing or newly generated)
4572 """
4573 from .bitmap import generate_bitmap, write_bitmap
4575 # Check if bitmap already exists
4576 try:
4577 existing = self.bitmap
4578 if existing is not None:
4579 return existing
4580 except FileNotFoundError:
4581 pass # No bitmap, we'll generate one
4583 # Generate new bitmap
4584 if progress:
4585 progress(f"Generating bitmap for {self.name().decode('utf-8')}...\n")
4587 pack_bitmap = generate_bitmap(
4588 self.index,
4589 object_store,
4590 refs,
4591 self.get_stored_checksum(),
4592 commit_interval=commit_interval,
4593 progress=progress,
4594 )
4596 # Write bitmap file
4597 write_bitmap(self._bitmap_path, pack_bitmap)
4599 if progress:
4600 progress(f"Wrote {self._bitmap_path}\n")
4602 # Update cached bitmap
4603 self._bitmap = pack_bitmap
4605 return pack_bitmap
4607 @property
4608 def mmap_size(self) -> int:
4609 """Return the total mmapped memory usage of this pack.
4611 This includes the pack data file and index file sizes,
4612 but only for components that have been loaded (and thus mmapped).
4613 """
4614 total = 0
4615 if self._data is not None:
4616 total += self._data._size
4617 if self._idx is not None and isinstance(self._idx, FilePackIndex):
4618 total += self._idx._size
4619 return total
4621 def close(self) -> None:
4622 """Close the pack file and index."""
4623 if self._data is not None:
4624 self._data.close()
4625 self._data = None
4626 if self._idx is not None:
4627 self._idx.close()
4628 self._idx = None
4630 def __del__(self) -> None:
4631 """Ensure pack file is closed when Pack is garbage collected."""
4632 if self._data is not None or self._idx is not None:
4633 import warnings
4635 warnings.warn(
4636 f"unclosed Pack {self!r}", ResourceWarning, stacklevel=2, source=self
4637 )
4638 try:
4639 self.close()
4640 except Exception:
4641 # Ignore errors during cleanup
4642 pass
4644 def __enter__(self) -> Self:
4645 """Enter context manager."""
4646 return self
4648 def __exit__(
4649 self,
4650 type: type | None,
4651 value: BaseException | None,
4652 traceback: TracebackType | None,
4653 ) -> None:
4654 """Exit context manager."""
4655 self.close()
4657 def __eq__(self, other: object) -> bool:
4658 """Check equality with another pack."""
4659 if not isinstance(other, Pack):
4660 return False
4661 return self.index == other.index
4663 def __len__(self) -> int:
4664 """Number of entries in this pack."""
4665 return len(self.index)
4667 def __repr__(self) -> str:
4668 """Return string representation of this pack."""
4669 return f"{self.__class__.__name__}({self._basename!r})"
4671 def __iter__(self) -> Iterator[ObjectID]:
4672 """Iterate over all the sha1s of the objects in this pack."""
4673 return iter(self.index)
4675 def check_length_and_checksum(self) -> None:
4676 """Sanity check the length and checksum of the pack index and data."""
4677 assert len(self.index) == len(self.data), (
4678 f"Length mismatch: {len(self.index)} (index) != {len(self.data)} (data)"
4679 )
4680 idx_stored_checksum = self.index.get_pack_checksum()
4681 data_stored_checksum = self.data.get_stored_checksum()
4682 if (
4683 idx_stored_checksum is not None
4684 and idx_stored_checksum != data_stored_checksum
4685 ):
4686 raise ChecksumMismatch(
4687 sha_to_hex(RawObjectID(idx_stored_checksum)),
4688 sha_to_hex(RawObjectID(data_stored_checksum)),
4689 )
4691 def check(self) -> None:
4692 """Check the integrity of this pack.
4694 Raises:
4695 ChecksumMismatch: if a checksum for the index or data is wrong
4696 """
4697 self.index.check()
4698 self.data.check()
4699 for obj in self.iterobjects():
4700 obj.check()
4701 # TODO: object connectivity checks
4703 def get_stored_checksum(self) -> bytes:
4704 """Return the stored checksum of the pack data."""
4705 return self.data.get_stored_checksum()
4707 def pack_tuples(self) -> list[tuple[ShaFile, None]]:
4708 """Return pack tuples for all objects in pack."""
4709 return [(o, None) for o in self.iterobjects()]
4711 def __contains__(self, sha1: ObjectID | RawObjectID) -> bool:
4712 """Check whether this pack contains a particular SHA1."""
4713 try:
4714 self.index.object_offset(sha1)
4715 return True
4716 except KeyError:
4717 return False
4719 def get_raw(self, sha1: RawObjectID | ObjectID) -> tuple[int, bytes]:
4720 """Get raw object data by SHA1."""
4721 offset = self.index.object_offset(sha1)
4722 obj_type, obj = self.data.get_object_at(offset)
4723 type_num, chunks = self.resolve_object(offset, obj_type, obj)
4724 return type_num, b"".join(chunks) # type: ignore[arg-type]
4726 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
4727 """Retrieve the specified SHA1."""
4728 type, uncomp = self.get_raw(sha1)
4729 return ShaFile.from_raw_string(type, uncomp, sha=sha1)
4731 def iterobjects(self) -> Iterator[ShaFile]:
4732 """Iterate over the objects in this pack."""
4733 return iter(
4734 PackInflater.for_pack_data(self.data, resolve_ext_ref=self.resolve_ext_ref)
4735 )
4737 def iterobjects_subset(
4738 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
4739 ) -> Iterator[ShaFile]:
4740 """Iterate over a subset of objects in this pack."""
4741 return (
4742 uo
4743 for uo in PackInflater.for_pack_subset(
4744 self,
4745 shas,
4746 allow_missing=allow_missing,
4747 resolve_ext_ref=self.resolve_ext_ref,
4748 )
4749 if uo.id in shas
4750 )
4752 def iter_unpacked_subset(
4753 self,
4754 shas: Iterable[ObjectID | RawObjectID],
4755 *,
4756 include_comp: bool = False,
4757 allow_missing: bool = False,
4758 convert_ofs_delta: bool = False,
4759 ) -> Iterator[UnpackedObject]:
4760 """Iterate over unpacked objects in subset."""
4761 ofs_pending: dict[int, list[UnpackedObject]] = defaultdict(list)
4762 ofs: dict[int, bytes] = {}
4763 todo: set[ObjectID | RawObjectID] = set(shas)
4764 for unpacked in self.iter_unpacked(include_comp=include_comp):
4765 sha = unpacked.sha()
4766 if unpacked.offset is not None:
4767 ofs[unpacked.offset] = sha
4768 hexsha = sha_to_hex(RawObjectID(sha))
4769 if hexsha in todo:
4770 if unpacked.pack_type_num == OFS_DELTA:
4771 assert isinstance(unpacked.delta_base, int)
4772 assert unpacked.offset is not None
4773 base_offset = unpacked.offset - unpacked.delta_base
4774 try:
4775 unpacked.delta_base = ofs[base_offset]
4776 except KeyError:
4777 ofs_pending[base_offset].append(unpacked)
4778 continue
4779 else:
4780 unpacked.pack_type_num = REF_DELTA
4781 yield unpacked
4782 todo.remove(hexsha)
4783 if unpacked.offset is not None:
4784 for child in ofs_pending.pop(unpacked.offset, []):
4785 child.pack_type_num = REF_DELTA
4786 child.delta_base = sha
4787 yield child
4788 assert not ofs_pending
4789 if not allow_missing and todo:
4790 raise UnresolvedDeltas(list(todo))
4792 def iter_unpacked(self, include_comp: bool = False) -> Iterator[UnpackedObject]:
4793 """Iterate over all unpacked objects in this pack."""
4794 ofs_to_entries = {
4795 ofs: (sha, crc32) for (sha, ofs, crc32) in self.index.iterentries()
4796 }
4797 for unpacked in self.data.iter_unpacked(include_comp=include_comp):
4798 assert unpacked.offset is not None
4799 (sha, crc32) = ofs_to_entries[unpacked.offset]
4800 unpacked._sha = sha
4801 unpacked.crc32 = crc32
4802 yield unpacked
4804 def keep(self, msg: bytes | None = None) -> str:
4805 """Add a .keep file for the pack, preventing git from garbage collecting it.
4807 Args:
4808 msg: A message written inside the .keep file; can be used later
4809 to determine whether or not a .keep file is obsolete.
4810 Returns: The path of the .keep file, as a string.
4811 """
4812 keepfile_name = f"{self._basename}.keep"
4813 with GitFile(keepfile_name, "wb") as keepfile:
4814 if msg:
4815 keepfile.write(msg)
4816 keepfile.write(b"\n")
4817 return keepfile_name
4819 def unkeep(self) -> bool:
4820 """Remove the .keep file for the pack, allowing git to garbage collect it.
4822 This is the counterpart of :meth:`keep`. It is not an error to call
4823 this on a pack that has no .keep file.
4825 Returns: True if a .keep file was removed, False if there was none.
4826 """
4827 try:
4828 os.unlink(f"{self._basename}.keep")
4829 except FileNotFoundError:
4830 return False
4831 return True
4833 def get_ref(
4834 self, sha: RawObjectID | ObjectID
4835 ) -> tuple[int | None, int, OldUnpackedObject]:
4836 """Get the object for a ref SHA, only looking in this pack."""
4837 # TODO: cache these results
4838 try:
4839 offset = self.index.object_offset(sha)
4840 except KeyError:
4841 offset = None
4842 if offset:
4843 type, obj = self.data.get_object_at(offset)
4844 elif self.resolve_ext_ref:
4845 type, obj = self.resolve_ext_ref(sha)
4846 else:
4847 raise KeyError(sha)
4848 return offset, type, obj
4850 def resolve_object(
4851 self,
4852 offset: int,
4853 type: int,
4854 obj: OldUnpackedObject,
4855 get_ref: Callable[
4856 [RawObjectID | ObjectID], tuple[int | None, int, OldUnpackedObject]
4857 ]
4858 | None = None,
4859 ) -> tuple[int, OldUnpackedObject]:
4860 """Resolve an object, possibly resolving deltas when necessary.
4862 Returns: Tuple with object type and contents.
4863 """
4864 # Walk down the delta chain, building a stack of deltas to reach
4865 # the requested object.
4866 base_offset: int | None = offset
4867 base_type = type
4868 base_obj = obj
4869 delta_stack = []
4870 seen_ref_offsets: set[int] = {offset}
4871 while base_type in DELTA_TYPES:
4872 prev_offset = base_offset
4873 if get_ref is None:
4874 get_ref = self.get_ref
4875 assert isinstance(base_obj, tuple), (
4876 f"Expected delta tuple, got {base_obj.__class__.__name__}"
4877 )
4878 if base_type == OFS_DELTA:
4879 (delta_offset, delta) = base_obj
4880 # TODO: clean up asserts and replace with nicer error messages
4881 assert isinstance(delta_offset, int), (
4882 f"Expected int, got {delta_offset.__class__}"
4883 )
4884 assert base_offset is not None
4885 base_offset = base_offset - delta_offset
4886 base_type, base_obj = self.data.get_object_at(base_offset)
4887 assert isinstance(base_type, int)
4888 elif base_type == REF_DELTA:
4889 (basename, delta) = base_obj
4890 assert (
4891 isinstance(basename, bytes)
4892 and len(basename) == self.object_format.oid_length
4893 )
4894 base_offset_temp, base_type, base_obj = get_ref(RawObjectID(basename))
4895 assert isinstance(base_type, int)
4896 # base_offset_temp can be None for thin packs (external references)
4897 base_offset = base_offset_temp
4898 if base_offset == prev_offset: # object is based on itself
4899 raise UnresolvedDeltas([basename])
4900 # A repeated REF_DELTA base offset means the chain cycles;
4901 # without this check the loop would never terminate.
4902 if base_offset is not None:
4903 if base_offset in seen_ref_offsets:
4904 raise DeltaCycle([basename])
4905 seen_ref_offsets.add(base_offset)
4906 else:
4907 raise AssertionError(f"Unexpected delta type: {base_type}")
4908 delta_stack.append((prev_offset, base_type, delta))
4910 # Now grab the base object (mustn't be a delta) and apply the
4911 # deltas all the way up the stack.
4912 chunks = base_obj
4913 for prev_offset, _delta_type, delta in reversed(delta_stack):
4914 # Convert chunks to bytes for apply_delta if needed
4915 if isinstance(chunks, list):
4916 chunks_bytes = b"".join(chunks)
4917 elif isinstance(chunks, tuple):
4918 # For tuple type, second element is the actual data
4919 _, chunk_data = chunks
4920 if isinstance(chunk_data, list):
4921 chunks_bytes = b"".join(chunk_data)
4922 else:
4923 chunks_bytes = chunk_data
4924 else:
4925 chunks_bytes = chunks
4927 # Apply delta and get result as list
4928 chunks = apply_delta(chunks_bytes, delta)
4930 if prev_offset is not None:
4931 self.data._cache_object_at(prev_offset, base_type, chunks)
4932 return base_type, chunks
4934 def entries(
4935 self, progress: Callable[[int, int], None] | None = None
4936 ) -> Iterator[PackIndexEntry]:
4937 """Yield entries summarizing the contents of this pack.
4939 Args:
4940 progress: Progress function, called with current and total
4941 object count.
4942 Returns: iterator of tuples with (sha, offset, crc32)
4943 """
4944 return self.data.iterentries(
4945 progress=progress, resolve_ext_ref=self.resolve_ext_ref
4946 )
4948 def sorted_entries(
4949 self, progress: Callable[[int, int], None] | None = None
4950 ) -> Iterator[PackIndexEntry]:
4951 """Return entries in this pack, sorted by SHA.
4953 Args:
4954 progress: Progress function, called with current and total
4955 object count
4956 Returns: Iterator of tuples with (sha, offset, crc32)
4957 """
4958 return iter(
4959 self.data.sorted_entries(
4960 progress=progress, resolve_ext_ref=self.resolve_ext_ref
4961 )
4962 )
4964 def get_unpacked_object(
4965 self,
4966 sha: ObjectID | RawObjectID,
4967 *,
4968 include_comp: bool = False,
4969 convert_ofs_delta: bool = True,
4970 ) -> UnpackedObject:
4971 """Get the unpacked object for a sha.
4973 Args:
4974 sha: SHA of object to fetch
4975 include_comp: Whether to include compression data in UnpackedObject
4976 convert_ofs_delta: Whether to convert offset deltas to ref deltas
4977 """
4978 offset = self.index.object_offset(sha)
4979 unpacked = self.data.get_unpacked_object_at(offset, include_comp=include_comp)
4980 if unpacked.pack_type_num == OFS_DELTA and convert_ofs_delta:
4981 assert isinstance(unpacked.delta_base, int)
4982 unpacked.delta_base = self.index.object_sha1(offset - unpacked.delta_base)
4983 unpacked.pack_type_num = REF_DELTA
4984 return unpacked
4987def extend_pack(
4988 f: BinaryIO,
4989 object_ids: Set["RawObjectID"],
4990 get_raw: Callable[["RawObjectID | ObjectID"], tuple[int, bytes]],
4991 object_format: "ObjectFormat",
4992 *,
4993 compression_level: int = -1,
4994 progress: Callable[[bytes], None] | None = None,
4995) -> tuple[bytes, list[tuple[RawObjectID, int, int]]]:
4996 """Extend a pack file with more objects.
4998 The caller should make sure that object_ids does not contain any objects
4999 that are already in the pack
5000 """
5001 # Update the header with the new number of objects.
5002 f.seek(0)
5003 _version, num_objects = read_pack_header(f.read)
5005 if object_ids:
5006 f.seek(0)
5007 write_pack_header(f.write, num_objects + len(object_ids))
5009 # Must flush before reading (http://bugs.python.org/issue3207)
5010 f.flush()
5012 # Rescan the rest of the pack, computing the SHA with the new header.
5013 new_sha = compute_file_sha(
5014 f, hash_func=object_format.hash_func, end_ofs=-object_format.oid_length
5015 )
5017 # Must reposition before writing (http://bugs.python.org/issue3207)
5018 f.seek(0, os.SEEK_CUR)
5020 extra_entries = []
5022 # Complete the pack.
5023 for i, object_id in enumerate(object_ids):
5024 if progress is not None:
5025 progress(
5026 (f"writing extra base objects: {i}/{len(object_ids)}\r").encode("ascii")
5027 )
5028 assert len(object_id) == object_format.oid_length
5029 type_num, data = get_raw(object_id)
5030 offset = f.tell()
5031 crc32 = write_pack_object(
5032 f.write,
5033 type_num,
5034 [data], # Convert bytes to list[bytes]
5035 sha=new_sha,
5036 compression_level=compression_level,
5037 object_format=object_format,
5038 )
5039 extra_entries.append((object_id, offset, crc32))
5040 pack_sha = new_sha.digest()
5041 f.write(pack_sha)
5042 return pack_sha, extra_entries
5045try:
5046 from dulwich._pack import ( # type: ignore
5047 apply_delta,
5048 bisect_find_sha,
5049 )
5050except ImportError:
5051 pass
5053# Try to import the Rust version of create_delta
5054try:
5055 from dulwich._pack import create_delta as _create_delta_rs
5056except ImportError:
5057 pass
5058else:
5059 # Wrap the Rust version to match the Python API (returns bytes instead of Iterator)
5060 def _create_delta_rs_wrapper(
5061 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
5062 ) -> Iterator[bytes]:
5063 """Wrapper for Rust create_delta to match Python API."""
5064 if isinstance(base_buf, list):
5065 base_buf = b"".join(base_buf)
5066 if isinstance(target_buf, list):
5067 target_buf = b"".join(target_buf)
5068 yield _create_delta_rs(base_buf, target_buf)
5070 create_delta = _create_delta_rs_wrapper