Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/pack.py: 21%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# pack.py -- For dealing with packed git objects.
2# Copyright (C) 2007 James Westby <jw+debian@jameswestby.net>
3# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
23"""Classes for dealing with packed git objects.
25A pack is a compact representation of a bunch of objects, stored
26using deltas where possible.
28They have two parts, the pack file, which stores the data, and an index
29that tells you where the data is.
31To find an object you look in all of the index files 'til you find a
32match for the object name. You then use the pointer got from this as
33a pointer in to the corresponding packfile.
34"""
36__all__ = [
37 "DEFAULT_DELTA_BASE_CACHE_LIMIT",
38 "DEFAULT_PACK_DELTA_WINDOW_SIZE",
39 "DEFAULT_PACK_INDEX_VERSION",
40 "DELTA_TYPES",
41 "OFS_DELTA",
42 "PACK_SPOOL_FILE_MAX_SIZE",
43 "REF_DELTA",
44 "DeltaChainIterator",
45 "DeltaCycle",
46 "FilePackIndex",
47 "HashWriter",
48 "MemoryPackIndex",
49 "ObjectContainer",
50 "OldUnpackedObject",
51 "Pack",
52 "PackChunkGenerator",
53 "PackData",
54 "PackFileDisappeared",
55 "PackHint",
56 "PackIndex",
57 "PackIndex1",
58 "PackIndex2",
59 "PackIndex3",
60 "PackIndexEntry",
61 "PackIndexer",
62 "PackInflater",
63 "PackStreamCopier",
64 "PackStreamReader",
65 "PackedObjectContainer",
66 "ProgressFn",
67 "ResolveExtRefFn",
68 "SHA1Reader",
69 "SHA1Writer",
70 "UnpackedObject",
71 "UnpackedObjectIterator",
72 "UnpackedObjectStream",
73 "UnresolvedDeltas",
74 "apply_delta",
75 "bisect_find_sha",
76 "chunks_length",
77 "compute_buffer_sha",
78 "compute_file_sha",
79 "create_delta",
80 "deltas_from_sorted_objects",
81 "deltify_pack_objects",
82 "extend_pack",
83 "find_reusable_deltas",
84 "full_unpacked_object",
85 "generate_unpacked_objects",
86 "has_mmap",
87 "iter_sha1",
88 "load_pack_index",
89 "load_pack_index_file",
90 "obj_sha",
91 "pack_header_chunks",
92 "pack_object_chunks",
93 "pack_object_header",
94 "pack_objects_to_data",
95 "read_pack_header",
96 "read_pack_header_at",
97 "read_zlib_chunks",
98 "read_zlib_chunks_at",
99 "sort_objects_for_delta",
100 "take_msb_bytes",
101 "take_msb_bytes_at",
102 "unpack_object",
103 "unpack_object_at",
104 "verify_and_read",
105 "write_pack",
106 "write_pack_data",
107 "write_pack_from_container",
108 "write_pack_header",
109 "write_pack_index",
110 "write_pack_index_v1",
111 "write_pack_index_v2",
112 "write_pack_index_v3",
113 "write_pack_object",
114 "write_pack_objects",
115]
117import binascii
118from collections import defaultdict, deque
119from contextlib import suppress
120from io import BytesIO, UnsupportedOperation
122try:
123 from cdifflib import CSequenceMatcher as SequenceMatcher
124except ModuleNotFoundError:
125 from difflib import SequenceMatcher
127import logging
128import os
129import struct
130import sys
131import threading
132import warnings
133import zlib
134from collections.abc import Callable, Iterable, Iterator, Sequence, Set
135from hashlib import sha1, sha256
136from itertools import chain
137from os import SEEK_END
138from struct import unpack_from
139from types import TracebackType
140from typing import (
141 IO,
142 TYPE_CHECKING,
143 Any,
144 BinaryIO,
145 Generic,
146 Protocol,
147 TypeVar,
148)
150if sys.version_info >= (3, 11):
151 from typing import Self
152else:
153 from typing_extensions import Self
155import mmap
157from .errors import ApplyDeltaError, ChecksumMismatch
158from .file import GitFile, _GitFile
159from .lru_cache import LRUSizeCache
160from .object_format import OBJECT_FORMAT_TYPE_NUMS, SHA1, ObjectFormat
161from .objects import (
162 ObjectID,
163 RawObjectID,
164 ShaFile,
165 hex_to_sha,
166 object_header,
167 sha_to_hex,
168)
170if TYPE_CHECKING:
171 from _hashlib import HASH as HashObject
173 from .bitmap import PackBitmap
174 from .commit_graph import CommitGraph
175 from .object_store import BaseObjectStore
176 from .refs import Ref
178logger = logging.getLogger(__name__)
180# Some platforms (e.g. plan9) don't support mmap properly
181has_mmap = sys.platform != "Plan9"
183OFS_DELTA = 6
184REF_DELTA = 7
186DELTA_TYPES = (OFS_DELTA, REF_DELTA)
189DEFAULT_PACK_DELTA_WINDOW_SIZE = 10
191# Keep pack files under 16Mb in memory, otherwise write them out to disk
192PACK_SPOOL_FILE_MAX_SIZE = 16 * 1024 * 1024
194# Default pack index version to use when none is specified
195DEFAULT_PACK_INDEX_VERSION = 2
198OldUnpackedObject = tuple[bytes | int, list[bytes]] | list[bytes] | bytes
199ResolveExtRefFn = Callable[[RawObjectID | ObjectID], tuple[int, bytes | list[bytes]]]
200ProgressFn = Callable[[int, str], None]
201PackHint = tuple[int, bytes | None]
204def verify_and_read(
205 read_func: Callable[[int], bytes],
206 expected_hash: bytes,
207 hash_algo: str,
208 progress: Callable[[bytes], None] | None = None,
209) -> Iterator[bytes]:
210 """Read from stream, verify hash, then yield verified chunks.
212 This function downloads data to a temporary file (in-memory for small files,
213 on-disk for large ones) while computing its hash. Only after the hash is
214 verified to match expected_hash will it yield any data. This prevents
215 corrupted or malicious data from reaching the caller.
217 Args:
218 read_func: Function to read bytes (like file.read or HTTP response reader)
219 expected_hash: Expected hash as hex string bytes (e.g., b'a3b2c1...')
220 hash_algo: Hash algorithm name ('sha1' or 'sha256')
221 progress: Optional progress callback
223 Yields:
224 Chunks of verified data (only after hash verification succeeds)
226 Raises:
227 ValueError: If hash doesn't match or algorithm unsupported
228 """
229 from tempfile import SpooledTemporaryFile
231 from .object_format import OBJECT_FORMATS
233 # Get the hash function for this algorithm
234 obj_format = OBJECT_FORMATS.get(hash_algo)
235 if obj_format is None:
236 raise ValueError(f"Unsupported hash algorithm: {hash_algo}")
238 hasher = obj_format.new_hash()
240 # Download to temporary file (memory or disk) while computing hash
241 with SpooledTemporaryFile(
242 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="dulwich-verify-"
243 ) as temp_file:
244 # Read data, hash it, and write to temp file
245 while True:
246 chunk = read_func(65536) # Read in 64KB chunks
247 if not chunk:
248 break
249 hasher.update(chunk)
250 temp_file.write(chunk)
252 # Verify hash BEFORE yielding any data
253 computed_hash = hasher.hexdigest().encode("ascii")
254 if computed_hash != expected_hash:
255 raise ValueError(
256 f"hash mismatch: expected {expected_hash.decode('ascii')}, "
257 f"got {computed_hash.decode('ascii')}"
258 )
260 # Hash verified! Now read from temp file and yield chunks
261 if progress:
262 progress(b"Hash verified, processing data\n")
264 temp_file.seek(0)
265 while True:
266 chunk = temp_file.read(65536)
267 if not chunk:
268 break
269 yield chunk
272class UnresolvedDeltas(Exception):
273 """Delta objects could not be resolved."""
275 def __init__(self, shas: list[bytes]) -> None:
276 """Initialize UnresolvedDeltas exception.
278 Args:
279 shas: List of SHA hashes for unresolved delta objects
280 """
281 self.shas = shas
284class DeltaCycle(UnresolvedDeltas):
285 """A pack's delta chain references itself and cannot be resolved."""
288class ObjectContainer(Protocol):
289 """Protocol for objects that can contain git objects."""
291 def add_object(self, obj: ShaFile) -> None:
292 """Add a single object to this object store."""
294 def add_objects(
295 self,
296 objects: Sequence[tuple[ShaFile, str | None]],
297 progress: Callable[..., None] | None = None,
298 ) -> "Pack | None":
299 """Add a set of objects to this object store.
301 Args:
302 objects: Iterable over a list of (object, path) tuples
303 progress: Progress callback for object insertion
304 Returns: Optional Pack object of the objects written.
305 """
307 def __contains__(self, sha1: "ObjectID") -> bool:
308 """Check if a hex sha is present."""
309 ...
311 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
312 """Retrieve an object."""
313 ...
315 def peel(self, sha: "ObjectID | RawObjectID") -> tuple[ShaFile, ShaFile]:
316 """Peel all tags from a SHA.
318 Args:
319 sha: The object SHA to peel.
320 Returns: Tuple of (unpeeled, peeled) objects.
321 """
322 from .object_store import peel_sha
324 return peel_sha(self, sha)
326 def get_commit_graph(self) -> "CommitGraph | None":
327 """Get the commit graph for this object store.
329 Returns:
330 CommitGraph object if available, None otherwise
331 """
332 return None
335class PackedObjectContainer(ObjectContainer):
336 """Container for objects packed in a pack file."""
338 def get_unpacked_object(
339 self, sha1: "ObjectID | RawObjectID", *, include_comp: bool = False
340 ) -> "UnpackedObject":
341 """Get a raw unresolved object.
343 Args:
344 sha1: SHA-1 hash of the object
345 include_comp: Whether to include compressed data
347 Returns:
348 UnpackedObject instance
349 """
350 raise NotImplementedError(self.get_unpacked_object)
352 def iterobjects_subset(
353 self, shas: Iterable["ObjectID"], *, allow_missing: bool = False
354 ) -> Iterator[ShaFile]:
355 """Iterate over a subset of objects.
357 Args:
358 shas: Iterable of object SHAs to retrieve
359 allow_missing: If True, skip missing objects
361 Returns:
362 Iterator of ShaFile objects
363 """
364 raise NotImplementedError(self.iterobjects_subset)
366 def iter_unpacked_subset(
367 self,
368 shas: Iterable["ObjectID | RawObjectID"],
369 *,
370 include_comp: bool = False,
371 allow_missing: bool = False,
372 convert_ofs_delta: bool = True,
373 ) -> Iterator["UnpackedObject"]:
374 """Iterate over unpacked objects from a subset of SHAs.
376 Args:
377 shas: Set of object SHAs to retrieve
378 include_comp: Include compressed data if True
379 allow_missing: If True, skip missing objects
380 convert_ofs_delta: If True, convert offset deltas to ref deltas
382 Returns:
383 Iterator of UnpackedObject instances
384 """
385 raise NotImplementedError(self.iter_unpacked_subset)
388class UnpackedObjectStream:
389 """Abstract base class for a stream of unpacked objects."""
391 def __iter__(self) -> Iterator["UnpackedObject"]:
392 """Iterate over unpacked objects."""
393 raise NotImplementedError(self.__iter__)
395 def __len__(self) -> int:
396 """Return the number of objects in the stream."""
397 raise NotImplementedError(self.__len__)
400def take_msb_bytes(
401 read: Callable[[int], bytes], crc32: int | None = None
402) -> tuple[list[int], int | None]:
403 """Read bytes marked with most significant bit.
405 Args:
406 read: Read function
407 crc32: Optional CRC32 checksum to update
409 Returns:
410 Tuple of (list of bytes read, updated CRC32 or None)
411 """
412 ret: list[int] = []
413 while len(ret) == 0 or ret[-1] & 0x80:
414 b = read(1)
415 if crc32 is not None:
416 crc32 = binascii.crc32(b, crc32)
417 ret.append(ord(b[:1]))
418 return ret, crc32
421def take_msb_bytes_at(
422 contents: "bytes | mmap.mmap", offset: int, crc32: int | None = None
423) -> tuple[list[int], int, int | None]:
424 """Read bytes marked with most significant bit from a buffer at an offset.
426 Args:
427 contents: Buffer to read from
428 offset: Offset in contents to start reading at
429 crc32: Optional CRC32 checksum to update
431 Returns:
432 Tuple of (list of bytes read, offset just past them, updated CRC32 or None)
433 """
434 ret: list[int] = []
435 pos = offset
436 while len(ret) == 0 or ret[-1] & 0x80:
437 b = contents[pos : pos + 1]
438 if not b:
439 raise AssertionError(f"unexpected end of pack data at {pos}")
440 pos += 1
441 if crc32 is not None:
442 crc32 = binascii.crc32(b, crc32)
443 ret.append(ord(b))
444 return ret, pos, crc32
447class PackFileDisappeared(Exception):
448 """Raised when a pack file unexpectedly disappears.
450 This typically happens when a concurrent operation (e.g. ``git repack``
451 or ``git gc --auto``) removes a pack file between the moment dulwich
452 snapshots the pack directory and the moment it actually opens the
453 pack's ``.idx`` or ``.pack`` file.
455 The ``obj`` attribute holds the :class:`Pack` (or :class:`FilePackIndex`)
456 whose backing file vanished, so the caller can evict the stale object
457 from its cache and rescan the pack directory.
458 """
460 obj: "Pack | FilePackIndex"
462 def __init__(self, obj: "Pack | FilePackIndex") -> None:
463 """Initialize PackFileDisappeared exception.
465 Args:
466 obj: The pack or pack index that disappeared.
467 """
468 self.obj = obj
471class UnpackedObject:
472 """Class encapsulating an object unpacked from a pack file.
474 These objects should only be created from within unpack_object. Most
475 members start out as empty and are filled in at various points by
476 read_zlib_chunks, unpack_object, DeltaChainIterator, etc.
478 End users of this object should take care that the function they're getting
479 this object from is guaranteed to set the members they need.
480 """
482 __slots__ = [
483 "_sha", # Cached binary SHA.
484 "comp_chunks", # Compressed object chunks.
485 "crc32", # CRC32.
486 "decomp_chunks", # Decompressed object chunks.
487 "decomp_len", # Decompressed length of this object.
488 "delta_base", # Delta base offset or SHA.
489 "hash_func", # Hash function to use for computing object IDs.
490 "obj_chunks", # Decompressed and delta-resolved chunks.
491 "obj_type_num", # Type of this object.
492 "offset", # Offset in its pack.
493 "pack_type_num", # Type of this object in the pack (may be a delta).
494 ]
496 obj_type_num: int | None
497 obj_chunks: list[bytes] | None
498 delta_base: bytes | int | None
499 decomp_chunks: list[bytes]
500 comp_chunks: list[bytes] | None
501 decomp_len: int | None
502 crc32: int | None
503 offset: int | None
504 pack_type_num: int
505 _sha: bytes | None
506 hash_func: Callable[[], "HashObject"]
508 # TODO(dborowitz): read_zlib_chunks and unpack_object could very well be
509 # methods of this object.
510 def __init__(
511 self,
512 pack_type_num: int,
513 *,
514 delta_base: bytes | int | None = None,
515 decomp_len: int | None = None,
516 crc32: int | None = None,
517 sha: bytes | None = None,
518 decomp_chunks: list[bytes] | None = None,
519 offset: int | None = None,
520 hash_func: Callable[[], "HashObject"] = sha1,
521 ) -> None:
522 """Initialize an UnpackedObject.
524 Args:
525 pack_type_num: Type number of this object in the pack
526 delta_base: Delta base (offset or SHA) if this is a delta object
527 decomp_len: Decompressed length of this object
528 crc32: CRC32 checksum
529 sha: SHA hash of the object
530 decomp_chunks: Decompressed chunks
531 offset: Offset in the pack file
532 hash_func: Hash function to use (defaults to sha1)
533 """
534 self.offset = offset
535 self._sha = sha
536 self.pack_type_num = pack_type_num
537 self.delta_base = delta_base
538 self.comp_chunks = None
539 self.decomp_chunks: list[bytes] = decomp_chunks or []
540 if decomp_chunks is not None and decomp_len is None:
541 self.decomp_len = sum(map(len, decomp_chunks))
542 else:
543 self.decomp_len = decomp_len
544 self.crc32 = crc32
545 self.hash_func = hash_func
547 if pack_type_num in DELTA_TYPES:
548 self.obj_type_num = None
549 self.obj_chunks = None
550 else:
551 self.obj_type_num = pack_type_num
552 self.obj_chunks = self.decomp_chunks
553 self.delta_base = delta_base
555 def sha(self) -> RawObjectID:
556 """Return the binary SHA of this object."""
557 if self._sha is None:
558 assert self.obj_type_num is not None and self.obj_chunks is not None
559 self._sha = obj_sha(self.obj_type_num, self.obj_chunks, self.hash_func)
560 return RawObjectID(self._sha)
562 def sha_file(self) -> ShaFile:
563 """Return a ShaFile from this object."""
564 assert self.obj_type_num is not None and self.obj_chunks is not None
565 return ShaFile.from_raw_chunks(self.obj_type_num, self.obj_chunks)
567 # Only provided for backwards compatibility with code that expects either
568 # chunks or a delta tuple.
569 def _obj(self) -> OldUnpackedObject:
570 """Return the decompressed chunks, or (delta base, delta chunks)."""
571 if self.pack_type_num in DELTA_TYPES:
572 assert isinstance(self.delta_base, bytes | int)
573 return (self.delta_base, self.decomp_chunks)
574 else:
575 return self.decomp_chunks
577 def __eq__(self, other: object) -> bool:
578 """Check equality with another UnpackedObject."""
579 if not isinstance(other, UnpackedObject):
580 return False
581 for slot in self.__slots__:
582 if getattr(self, slot) != getattr(other, slot):
583 return False
584 return True
586 def __ne__(self, other: object) -> bool:
587 """Check inequality with another UnpackedObject."""
588 return not (self == other)
590 def __repr__(self) -> str:
591 """Return string representation of this UnpackedObject."""
592 data = [f"{s}={getattr(self, s)!r}" for s in self.__slots__]
593 return "{}({})".format(self.__class__.__name__, ", ".join(data))
596_ZLIB_BUFSIZE = 65536 # 64KB buffer for better I/O performance
598# Default maximum memory for caching delta base objects (matches Git's default
599# for core.deltaBaseCacheLimit).
600DEFAULT_DELTA_BASE_CACHE_LIMIT = 96 * 1024 * 1024 # 96 MiB
603def read_zlib_chunks(
604 read_some: Callable[[int], bytes],
605 unpacked: UnpackedObject,
606 include_comp: bool = False,
607 buffer_size: int = _ZLIB_BUFSIZE,
608) -> bytes:
609 """Read zlib data from a buffer.
611 This function requires that the buffer have additional data following the
612 compressed data, which is guaranteed to be the case for git pack files.
614 Args:
615 read_some: Read function that returns at least one byte, but may
616 return less than the requested size.
617 unpacked: An UnpackedObject to write result data to. If its crc32
618 attr is not None, the CRC32 of the compressed bytes will be computed
619 using this starting CRC32.
620 After this function, will have the following attrs set:
621 * comp_chunks (if include_comp is True)
622 * decomp_chunks
623 * decomp_len
624 * crc32
625 include_comp: If True, include compressed data in the result.
626 buffer_size: Size of the read buffer.
627 Returns: Leftover unused data from the decompression.
629 Raises:
630 zlib.error: if a decompression error occurred.
631 """
632 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
633 raise ValueError("non-negative zlib data stream size expected")
634 decomp_obj = zlib.decompressobj()
636 comp_chunks = []
637 decomp_chunks = unpacked.decomp_chunks
638 decomp_len = 0
639 crc32 = unpacked.crc32
640 max_decomp = unpacked.decomp_len
642 while True:
643 add = read_some(buffer_size)
644 if not add:
645 raise zlib.error("EOF before end of zlib stream")
646 comp_chunks.append(add)
647 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
648 remaining = max_decomp - decomp_len + 1
649 decomp = decomp_obj.decompress(add, remaining)
650 if decomp_obj.unconsumed_tail:
651 raise zlib.error("decompressed data exceeds expected size")
652 decomp_len += len(decomp)
653 decomp_chunks.append(decomp)
654 unused = decomp_obj.unused_data
655 if unused:
656 left = len(unused)
657 if crc32 is not None:
658 crc32 = binascii.crc32(add[:-left], crc32)
659 if include_comp:
660 comp_chunks[-1] = add[:-left]
661 break
662 elif crc32 is not None:
663 crc32 = binascii.crc32(add, crc32)
664 if crc32 is not None:
665 crc32 &= 0xFFFFFFFF
667 if decomp_len != unpacked.decomp_len:
668 raise zlib.error("decompressed data does not match expected size")
670 unpacked.crc32 = crc32
671 if include_comp:
672 unpacked.comp_chunks = comp_chunks
673 return unused
676def read_zlib_chunks_at(
677 contents: "bytes | mmap.mmap",
678 offset: int,
679 unpacked: UnpackedObject,
680 include_comp: bool = False,
681 buffer_size: int = _ZLIB_BUFSIZE,
682) -> int:
683 """Read zlib data from a buffer at a given offset.
685 Like :func:`read_zlib_chunks`, but indexes the buffer directly instead of
686 consuming a read callable, so concurrent readers do not share a position.
688 The buffer is fed to zlib in ``buffer_size`` slices rather than in one
689 piece. That bounds ``unused_data``, which zlib materialises as a copy of
690 everything it was handed past the end of the stream: passing the whole
691 mapping would copy the entire remainder of the pack for every object read.
693 Slices are taken as memoryviews, so the compressed data is decompressed
694 straight out of the mapping. With ``include_comp`` the chunks are kept on
695 ``unpacked`` and outlive the mapping, so those are copied.
697 Args:
698 contents: Buffer holding the compressed data.
699 offset: Offset in contents at which the zlib stream starts.
700 unpacked: An UnpackedObject to write result data to; see
701 :func:`read_zlib_chunks` for the attributes set on it.
702 include_comp: If True, include compressed data in the result.
703 buffer_size: Number of bytes to feed to zlib at a time.
704 Returns: Offset in contents just past the end of the zlib stream.
706 Raises:
707 zlib.error: if a decompression error occurred.
708 """
709 if unpacked.decomp_len is None or unpacked.decomp_len <= -1:
710 raise ValueError("non-negative zlib data stream size expected")
711 decomp_obj = zlib.decompressobj()
713 comp_chunks = []
714 decomp_chunks = unpacked.decomp_chunks
715 decomp_len = 0
716 crc32 = unpacked.crc32
717 max_decomp = unpacked.decomp_len
718 pos = offset
720 with memoryview(contents) as view:
721 while True:
722 add = view[pos : pos + buffer_size]
723 if not add:
724 raise zlib.error("EOF before end of zlib stream")
725 pos += len(add)
726 # +1 so overrun surfaces as unconsumed_tail rather than being truncated.
727 remaining = max_decomp - decomp_len + 1
728 decomp = decomp_obj.decompress(add, remaining)
729 if decomp_obj.unconsumed_tail:
730 raise zlib.error("decompressed data exceeds expected size")
731 decomp_len += len(decomp)
732 decomp_chunks.append(decomp)
733 unused = decomp_obj.unused_data
734 if unused:
735 left = len(unused)
736 pos -= left
737 add = add[:-left]
738 if crc32 is not None:
739 crc32 = binascii.crc32(add, crc32)
740 if include_comp:
741 comp_chunks.append(bytes(add))
742 if unused:
743 break
744 if crc32 is not None:
745 crc32 &= 0xFFFFFFFF
747 if decomp_len != unpacked.decomp_len:
748 raise zlib.error("decompressed data does not match expected size")
750 unpacked.crc32 = crc32
751 if include_comp:
752 unpacked.comp_chunks = comp_chunks
753 return pos
756def iter_sha1(iter: Iterable[bytes]) -> bytes:
757 """Return the hexdigest of the SHA1 over a set of names.
759 Args:
760 iter: Iterator over string objects
761 Returns: 40-byte hex sha1 digest
762 """
763 sha = sha1()
764 for name in iter:
765 sha.update(name)
766 return sha.hexdigest().encode("ascii")
769def load_pack_index(
770 path: str | os.PathLike[str], object_format: ObjectFormat
771) -> "PackIndex":
772 """Load an index file by path.
774 Args:
775 path: Path to the index file
776 object_format: Hash algorithm used by the repository
777 Returns: A PackIndex loaded from the given path
778 """
779 # Ownership of the file is transferred to the returned index, which mmaps
780 # it and closes it in PackIndex.close(). It must not be closed here: on
781 # Windows an mmap keeps the file locked, so closing the handle out from
782 # under a live mapping leaves the .idx undeletable until the index is GCed.
783 f = GitFile(path, "rb")
784 try:
785 return load_pack_index_file(path, f, object_format)
786 except BaseException:
787 f.close()
788 raise
791def _load_file_contents(
792 f: IO[bytes] | _GitFile, size: int | None = None
793) -> tuple[bytes | Any, int]:
794 """Load contents from a file, preferring mmap when possible.
796 Args:
797 f: File-like object to load
798 size: Expected size, or None to determine from file
799 Returns: Tuple of (contents, size)
800 """
801 # Avoid rolling a SpooledTemporaryFile to disk just to get a descriptor.
802 if getattr(f, "_rolled", True) is False:
803 fd = None
804 else:
805 try:
806 fd = f.fileno()
807 except (UnsupportedOperation, AttributeError):
808 fd = None
809 # Attempt to use mmap if possible
810 if fd is not None:
811 if size is None:
812 size = os.fstat(fd).st_size
813 if has_mmap:
814 try:
815 contents = mmap.mmap(fd, size, access=mmap.ACCESS_READ)
816 except (OSError, ValueError):
817 # Can't mmap - perhaps a socket or invalid file descriptor
818 pass
819 else:
820 return contents, size
821 contents_bytes = f.read()
822 size = len(contents_bytes)
823 return contents_bytes, size
826def _close_file_contents(contents: "bytes | mmap.mmap | None") -> None:
827 """Close contents returned by _load_file_contents, if closeable.
829 Callers must close the mapping before the file it maps: on Windows the
830 mapping holds a lock on the file, so the handle cannot be released while
831 it is alive.
832 """
833 close_fn = getattr(contents, "close", None)
834 if close_fn is not None:
835 close_fn()
838def load_pack_index_file(
839 path: str | os.PathLike[str],
840 f: IO[bytes] | _GitFile,
841 object_format: ObjectFormat,
842) -> "PackIndex":
843 """Load an index file from a file-like object.
845 Args:
846 path: Path for the index file
847 f: File-like object
848 object_format: Hash algorithm used by the repository
849 Returns: A PackIndex loaded from the given file
850 """
851 contents, size = _load_file_contents(f)
852 if contents[:4] == b"\377tOc":
853 version = struct.unpack(b">L", contents[4:8])[0]
854 if version == 2:
855 return PackIndex2(
856 path,
857 object_format,
858 file=f,
859 contents=contents,
860 size=size,
861 )
862 elif version == 3:
863 return PackIndex3(path, object_format, file=f, contents=contents, size=size)
864 else:
865 raise KeyError(f"Unknown pack index format {version}")
866 else:
867 return PackIndex1(path, object_format, file=f, contents=contents, size=size)
870def bisect_find_sha(
871 start: int, end: int, sha: bytes, unpack_name: Callable[[int], bytes]
872) -> int | None:
873 """Find a SHA in a data blob with sorted SHAs.
875 Args:
876 start: Start index of range to search
877 end: End index of range to search
878 sha: Sha to find
879 unpack_name: Callback to retrieve SHA by index
880 Returns: Index of the SHA, or None if it wasn't found
881 """
882 assert start <= end
883 while start <= end:
884 i = (start + end) // 2
885 file_sha = unpack_name(i)
886 if file_sha < sha:
887 start = i + 1
888 elif file_sha > sha:
889 end = i - 1
890 else:
891 return i
892 return None
895PackIndexEntry = tuple[RawObjectID, int, int | None]
898class PackIndex:
899 """An index in to a packfile.
901 Given a sha id of an object a pack index can tell you the location in the
902 packfile of that object if it has it.
903 """
905 object_format: "ObjectFormat"
907 def __eq__(self, other: object) -> bool:
908 """Check equality with another PackIndex."""
909 if not isinstance(other, PackIndex):
910 return False
912 for (name1, _, _), (name2, _, _) in zip(
913 self.iterentries(), other.iterentries()
914 ):
915 if name1 != name2:
916 return False
917 return True
919 def __ne__(self, other: object) -> bool:
920 """Check if this pack index is not equal to another."""
921 return not self.__eq__(other)
923 def __len__(self) -> int:
924 """Return the number of entries in this pack index."""
925 raise NotImplementedError(self.__len__)
927 def __iter__(self) -> Iterator[ObjectID]:
928 """Iterate over the SHAs in this pack."""
929 return map(lambda sha: sha_to_hex(RawObjectID(sha)), self._itersha())
931 def iterentries(self) -> Iterator[PackIndexEntry]:
932 """Iterate over the entries in this pack index.
934 Returns: iterator over tuples with object name, offset in packfile and
935 crc32 checksum.
936 """
937 raise NotImplementedError(self.iterentries)
939 def get_pack_checksum(self) -> bytes | None:
940 """Return the SHA1 checksum stored for the corresponding packfile.
942 Returns: 20-byte binary digest, or None if not available
943 """
944 raise NotImplementedError(self.get_pack_checksum)
946 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
947 """Return the offset in to the corresponding packfile for the object.
949 Given the name of an object it will return the offset that object
950 lives at within the corresponding pack file. If the pack file doesn't
951 have the object then None will be returned.
952 """
953 raise NotImplementedError(self.object_offset)
955 def object_sha1(self, index: int) -> bytes:
956 """Return the SHA1 corresponding to the index in the pack file."""
957 for name, offset, _crc32 in self.iterentries():
958 if offset == index:
959 return name
960 else:
961 raise KeyError(index)
963 def object_sha_at_position(self, pos: int) -> RawObjectID:
964 """Return the name of the object at the given position in the index.
966 Positions count objects in the sorted order of the index, starting at
967 zero; this is how pack bitmaps refer to objects.
969 Args:
970 pos: Position of the object in the sorted index
971 Returns: Binary object name
972 Raises:
973 IndexError: If pos is out of range
974 """
975 raise NotImplementedError(self.object_sha_at_position)
977 def _object_offset(self, sha: bytes) -> int:
978 """See object_offset.
980 Args:
981 sha: A *binary* SHA string. (20 characters long)_
982 """
983 raise NotImplementedError(self._object_offset)
985 def objects_sha1(self) -> bytes:
986 """Return the hex SHA1 over all the shas of all objects in this pack.
988 Note: This is used for the filename of the pack.
989 """
990 return iter_sha1(self._itersha())
992 def _itersha(self) -> Iterator[bytes]:
993 """Yield all the SHA1's of the objects in the index, sorted."""
994 raise NotImplementedError(self._itersha)
996 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
997 """Iterate over all SHA1s with the given prefix.
999 Args:
1000 prefix: Binary prefix to match
1001 Returns: Iterator of matching SHA1s
1002 """
1003 # Default implementation for PackIndex classes that don't override
1004 for sha, _, _ in self.iterentries():
1005 if sha.startswith(prefix):
1006 yield RawObjectID(sha)
1008 def close(self) -> None:
1009 """Close any open files."""
1011 def check(self) -> None:
1012 """Check the consistency of this pack index."""
1015class MemoryPackIndex(PackIndex):
1016 """Pack index that is stored entirely in memory."""
1018 def __init__(
1019 self,
1020 entries: list[PackIndexEntry],
1021 object_format: ObjectFormat,
1022 pack_checksum: bytes | None = None,
1023 ) -> None:
1024 """Create a new MemoryPackIndex.
1026 Args:
1027 entries: Sequence of name, idx, crc32 (sorted)
1028 object_format: Object format used by this index
1029 pack_checksum: Optional pack checksum
1030 """
1031 self._by_sha = {}
1032 self._by_offset = {}
1033 for name, offset, _crc32 in entries:
1034 self._by_sha[name] = offset
1035 self._by_offset[offset] = name
1036 self._entries = entries
1037 self._pack_checksum = pack_checksum
1038 self.object_format = object_format
1040 def get_pack_checksum(self) -> bytes | None:
1041 """Return the SHA checksum stored for the corresponding packfile."""
1042 return self._pack_checksum
1044 def __len__(self) -> int:
1045 """Return the number of entries in this pack index."""
1046 return len(self._entries)
1048 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1049 """Return the offset for the given SHA.
1051 Args:
1052 sha: SHA to look up (binary or hex)
1053 Returns: Offset in the pack file
1054 """
1055 lookup_sha: RawObjectID
1056 if len(sha) == self.object_format.hex_length:
1057 lookup_sha = hex_to_sha(ObjectID(sha))
1058 else:
1059 lookup_sha = RawObjectID(sha)
1060 return self._by_sha[lookup_sha]
1062 def object_sha1(self, index: int) -> bytes:
1063 """Return the SHA1 for the object at the given offset."""
1064 return self._by_offset[index]
1066 def object_sha_at_position(self, pos: int) -> RawObjectID:
1067 """Return the name of the object at the given position in the index."""
1068 if not 0 <= pos < len(self._entries):
1069 raise IndexError(pos)
1070 return RawObjectID(self._entries[pos][0])
1072 def _itersha(self) -> Iterator[bytes]:
1073 """Iterate over all SHA1s in the index."""
1074 return iter(self._by_sha)
1076 def iterentries(self) -> Iterator[PackIndexEntry]:
1077 """Iterate over all index entries."""
1078 return iter(self._entries)
1080 @classmethod
1081 def for_pack(cls, pack_data: "PackData") -> "MemoryPackIndex":
1082 """Create a MemoryPackIndex from a PackData object."""
1083 return MemoryPackIndex(
1084 list(pack_data.sorted_entries()),
1085 pack_checksum=pack_data.get_stored_checksum(),
1086 object_format=pack_data.object_format,
1087 )
1089 @classmethod
1090 def clone(cls, other_index: "PackIndex") -> "MemoryPackIndex":
1091 """Create a copy of another PackIndex in memory."""
1092 return cls(
1093 list(other_index.iterentries()),
1094 other_index.object_format,
1095 other_index.get_pack_checksum(),
1096 )
1099class FilePackIndex(PackIndex):
1100 """Pack index that is based on a file.
1102 To do the loop it opens the file, and indexes first 256 4 byte groups
1103 with the first byte of the sha id. The value in the four byte group indexed
1104 is the end of the group that shares the same starting byte. Subtract one
1105 from the starting byte and index again to find the start of the group.
1106 The values are sorted by sha id within the group, so do the math to find
1107 the start and end offset and then bisect in to find if the value is
1108 present.
1109 """
1111 _fan_out_table: list[int]
1112 _file: IO[bytes] | _GitFile
1114 def __init__(
1115 self,
1116 filename: str | os.PathLike[str],
1117 file: IO[bytes] | _GitFile | None = None,
1118 contents: "bytes | mmap.mmap | None" = None,
1119 size: int | None = None,
1120 ) -> None:
1121 """Create a pack index object.
1123 Provide it with the name of the index file to consider, and it will map
1124 it whenever required.
1125 """
1126 self._filename = filename
1127 # Take the size now, so it can be checked each time we map the file to
1128 # ensure that it hasn't changed.
1129 if file is None:
1130 self._file = GitFile(filename, "rb")
1131 else:
1132 self._file = file
1133 if contents is None:
1134 self._contents, self._size = _load_file_contents(self._file, size)
1135 else:
1136 self._contents = contents
1137 self._size = size if size is not None else len(contents)
1139 @property
1140 def path(self) -> str:
1141 """Return the path to this index file."""
1142 return os.fspath(self._filename)
1144 def __eq__(self, other: object) -> bool:
1145 """Check equality with another FilePackIndex."""
1146 # Quick optimization:
1147 if (
1148 isinstance(other, FilePackIndex)
1149 and self._fan_out_table != other._fan_out_table
1150 ):
1151 return False
1153 return super().__eq__(other)
1155 def close(self) -> None:
1156 """Close the underlying file and any mmap."""
1157 _close_file_contents(self._contents)
1158 self._file.close()
1160 def __del__(self) -> None:
1161 """Ensure the file and mmap are closed when GCed."""
1162 if not getattr(self._file, "closed", True):
1163 import warnings
1165 warnings.warn(
1166 f"unclosed pack index {self!r}",
1167 ResourceWarning,
1168 stacklevel=2,
1169 source=self,
1170 )
1171 try:
1172 self.close()
1173 except Exception:
1174 # Ignore errors during cleanup
1175 pass
1177 def __enter__(self) -> Self:
1178 """Enter context manager."""
1179 return self
1181 def __exit__(
1182 self,
1183 type: type | None,
1184 value: BaseException | None,
1185 traceback: TracebackType | None,
1186 ) -> None:
1187 """Exit context manager."""
1188 self.close()
1190 def __len__(self) -> int:
1191 """Return the number of entries in this pack index."""
1192 return self._fan_out_table[-1]
1194 def _unpack_entry(self, i: int) -> PackIndexEntry:
1195 """Unpack the i-th entry in the index file.
1197 Returns: Tuple with object name (SHA), offset in pack file and CRC32
1198 checksum (if known).
1199 """
1200 raise NotImplementedError(self._unpack_entry)
1202 def _unpack_name(self, i: int) -> bytes:
1203 """Unpack the i-th name from the index file."""
1204 raise NotImplementedError(self._unpack_name)
1206 def _unpack_offset(self, i: int) -> int:
1207 """Unpack the i-th object offset from the index file."""
1208 raise NotImplementedError(self._unpack_offset)
1210 def _unpack_crc32_checksum(self, i: int) -> int | None:
1211 """Unpack the crc32 checksum for the i-th object from the index file."""
1212 raise NotImplementedError(self._unpack_crc32_checksum)
1214 def _itersha(self) -> Iterator[bytes]:
1215 """Iterate over all SHA1s in the index."""
1216 for i in range(len(self)):
1217 yield self._unpack_name(i)
1219 def iterentries(self) -> Iterator[PackIndexEntry]:
1220 """Iterate over the entries in this pack index.
1222 Returns: iterator over tuples with object name, offset in packfile and
1223 crc32 checksum.
1224 """
1225 for i in range(len(self)):
1226 yield self._unpack_entry(i)
1228 def object_sha_at_position(self, pos: int) -> RawObjectID:
1229 """Return the name of the object at the given position in the index."""
1230 if not 0 <= pos < len(self):
1231 raise IndexError(pos)
1232 return RawObjectID(self._unpack_name(pos))
1234 def _read_fan_out_table(self, start_offset: int) -> list[int]:
1235 """Read the fan-out table from the index.
1237 The fan-out table contains 256 entries mapping first byte values
1238 to the number of objects with SHA1s less than or equal to that byte.
1240 Args:
1241 start_offset: Offset in the file where the fan-out table starts
1242 Returns: List of 256 integers
1243 """
1244 ret = []
1245 for i in range(0x100):
1246 fanout_entry = self._contents[
1247 start_offset + i * 4 : start_offset + (i + 1) * 4
1248 ]
1249 ret.append(struct.unpack(">L", fanout_entry)[0])
1250 return ret
1252 def check(self) -> None:
1253 """Check that the stored checksum matches the actual checksum."""
1254 actual = self.calculate_checksum()
1255 stored = self.get_stored_checksum()
1256 if actual != stored:
1257 raise ChecksumMismatch(stored, actual)
1259 def calculate_checksum(self) -> bytes:
1260 """Calculate the SHA1 checksum over this pack index.
1262 Returns: This is a 20-byte binary digest
1263 """
1264 return sha1(self._contents[:-20]).digest()
1266 def get_pack_checksum(self) -> bytes:
1267 """Return the SHA1 checksum stored for the corresponding packfile.
1269 Returns: 20-byte binary digest
1270 """
1271 return bytes(self._contents[-40:-20])
1273 def get_stored_checksum(self) -> bytes:
1274 """Return the SHA1 checksum stored for this index.
1276 Returns: 20-byte binary digest
1277 """
1278 return bytes(self._contents[-20:])
1280 def object_offset(self, sha: ObjectID | RawObjectID) -> int:
1281 """Return the offset in to the corresponding packfile for the object.
1283 Given the name of an object it will return the offset that object
1284 lives at within the corresponding pack file. If the pack file doesn't
1285 have the object then None will be returned.
1286 """
1287 lookup_sha: RawObjectID
1288 if len(sha) == self.object_format.hex_length: # hex string
1289 lookup_sha = hex_to_sha(ObjectID(sha))
1290 else:
1291 lookup_sha = RawObjectID(sha)
1292 try:
1293 return self._object_offset(lookup_sha)
1294 except ValueError as exc:
1295 closed = getattr(self._contents, "closed", None)
1296 if closed in (None, True):
1297 raise PackFileDisappeared(self) from exc
1298 raise
1300 def _object_offset(self, sha: bytes) -> int:
1301 """See object_offset.
1303 Args:
1304 sha: A *binary* SHA string. (20 characters long)_
1305 """
1306 hash_size = getattr(self, "hash_size", 20) # Default to SHA1 for v1
1307 assert len(sha) == hash_size
1308 idx = ord(sha[:1])
1309 if idx == 0:
1310 start = 0
1311 else:
1312 start = self._fan_out_table[idx - 1]
1313 end = self._fan_out_table[idx]
1314 i = bisect_find_sha(start, end, sha, self._unpack_name)
1315 if i is None:
1316 raise KeyError(sha)
1317 return self._unpack_offset(i)
1319 def iter_prefix(self, prefix: bytes) -> Iterator[RawObjectID]:
1320 """Iterate over all SHA1s with the given prefix."""
1321 start = ord(prefix[:1])
1322 if start == 0:
1323 start = 0
1324 else:
1325 start = self._fan_out_table[start - 1]
1326 end = ord(prefix[:1]) + 1
1327 if end == 0x100:
1328 end = len(self)
1329 else:
1330 end = self._fan_out_table[end]
1331 assert start <= end
1332 started = False
1333 for i in range(start, end):
1334 name: bytes = self._unpack_name(i)
1335 if name.startswith(prefix):
1336 yield RawObjectID(name)
1337 started = True
1338 elif started:
1339 break
1342class PackIndex1(FilePackIndex):
1343 """Version 1 Pack Index file."""
1345 object_format = SHA1
1347 def __init__(
1348 self,
1349 filename: str | os.PathLike[str],
1350 object_format: ObjectFormat,
1351 file: IO[bytes] | _GitFile | None = None,
1352 contents: bytes | None = None,
1353 size: int | None = None,
1354 ) -> None:
1355 """Initialize a version 1 pack index.
1357 Args:
1358 filename: Path to the index file
1359 object_format: Object format used by the repository
1360 file: Optional file object
1361 contents: Optional mmap'd contents
1362 size: Optional size of the index
1363 """
1364 super().__init__(filename, file, contents, size)
1366 # PackIndex1 only supports SHA1
1367 if object_format != SHA1:
1368 raise AssertionError(
1369 f"PackIndex1 only supports SHA1, not {object_format.name}"
1370 )
1372 self.object_format = object_format
1373 self.version = 1
1374 self._fan_out_table = self._read_fan_out_table(0)
1375 self.hash_size = self.object_format.oid_length
1376 self._entry_size = 4 + self.hash_size
1378 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, None]:
1379 base_offset = (0x100 * 4) + (i * self._entry_size)
1380 offset = unpack_from(">L", self._contents, base_offset)[0]
1381 name = self._contents[base_offset + 4 : base_offset + 4 + self.hash_size]
1382 return (RawObjectID(name), offset, None)
1384 def _unpack_name(self, i: int) -> bytes:
1385 offset = (0x100 * 4) + (i * self._entry_size) + 4
1386 return self._contents[offset : offset + self.hash_size]
1388 def _unpack_offset(self, i: int) -> int:
1389 offset = (0x100 * 4) + (i * self._entry_size)
1390 return int(unpack_from(">L", self._contents, offset)[0])
1392 def _unpack_crc32_checksum(self, i: int) -> None:
1393 # Not stored in v1 index files
1394 return None
1397class PackIndex2(FilePackIndex):
1398 """Version 2 Pack Index file."""
1400 object_format = SHA1
1402 def __init__(
1403 self,
1404 filename: str | os.PathLike[str],
1405 object_format: ObjectFormat,
1406 file: IO[bytes] | _GitFile | None = None,
1407 contents: bytes | None = None,
1408 size: int | None = None,
1409 ) -> None:
1410 """Initialize a version 2 pack index.
1412 Args:
1413 filename: Path to the index file
1414 object_format: Object format used by the repository
1415 file: Optional file object
1416 contents: Optional mmap'd contents
1417 size: Optional size of the index
1418 """
1419 super().__init__(filename, file, contents, size)
1420 self.object_format = object_format
1421 if self._contents[:4] != b"\377tOc":
1422 raise AssertionError("Not a v2 pack index file")
1423 (self.version,) = unpack_from(b">L", self._contents, 4)
1424 if self.version != 2:
1425 raise AssertionError(f"Version was {self.version}")
1426 self._fan_out_table = self._read_fan_out_table(8)
1427 self.hash_size = self.object_format.oid_length
1428 self._name_table_offset = 8 + 0x100 * 4
1429 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1430 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1431 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1432 self
1433 )
1435 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1436 return (
1437 RawObjectID(self._unpack_name(i)),
1438 self._unpack_offset(i),
1439 self._unpack_crc32_checksum(i),
1440 )
1442 def _unpack_name(self, i: int) -> bytes:
1443 offset = self._name_table_offset + i * self.hash_size
1444 return self._contents[offset : offset + self.hash_size]
1446 def _unpack_offset(self, i: int) -> int:
1447 offset = self._pack_offset_table_offset + i * 4
1448 offset_val = int(unpack_from(">L", self._contents, offset)[0])
1449 if offset_val & (2**31):
1450 offset = (
1451 self._pack_offset_largetable_offset + (offset_val & (2**31 - 1)) * 8
1452 )
1453 offset_val = int(unpack_from(">Q", self._contents, offset)[0])
1454 return offset_val
1456 def _unpack_crc32_checksum(self, i: int) -> int:
1457 return int(
1458 unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1459 )
1461 def get_pack_checksum(self) -> bytes:
1462 """Return the checksum stored for the corresponding packfile.
1464 Returns: binary digest (size depends on hash algorithm)
1465 """
1466 # Index ends with: pack_checksum + index_checksum
1467 # Each checksum is hash_size bytes
1468 checksum_size = self.hash_size
1469 return bytes(self._contents[-2 * checksum_size : -checksum_size])
1471 def get_stored_checksum(self) -> bytes:
1472 """Return the checksum stored for this index.
1474 Returns: binary digest (size depends on hash algorithm)
1475 """
1476 checksum_size = self.hash_size
1477 return bytes(self._contents[-checksum_size:])
1479 def calculate_checksum(self) -> bytes:
1480 """Calculate the checksum over this pack index.
1482 Returns: binary digest (size depends on hash algorithm)
1483 """
1484 # Determine hash function based on hash_size
1485 if self.hash_size == 20:
1486 hash_func = sha1
1487 elif self.hash_size == 32:
1488 hash_func = sha256
1489 else:
1490 raise ValueError(f"Unsupported hash size: {self.hash_size}")
1492 return hash_func(self._contents[: -self.hash_size]).digest()
1495class PackIndex3(FilePackIndex):
1496 """Version 3 Pack Index file.
1498 Supports variable hash sizes for SHA-1 (20 bytes) and SHA-256 (32 bytes).
1499 """
1501 def __init__(
1502 self,
1503 filename: str | os.PathLike[str],
1504 object_format: ObjectFormat,
1505 file: IO[bytes] | _GitFile | None = None,
1506 contents: bytes | None = None,
1507 size: int | None = None,
1508 ) -> None:
1509 """Initialize a version 3 pack index.
1511 Args:
1512 filename: Path to the index file
1513 object_format: Object format used by the repository
1514 file: Optional file object
1515 contents: Optional mmap'd contents
1516 size: Optional size of the index
1517 """
1518 super().__init__(filename, file, contents, size)
1519 if self._contents[:4] != b"\377tOc":
1520 raise AssertionError("Not a v3 pack index file")
1521 (self.version,) = unpack_from(b">L", self._contents, 4)
1522 if self.version != 3:
1523 raise AssertionError(f"Version was {self.version}")
1525 # Read hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
1526 (self.hash_format,) = unpack_from(b">L", self._contents, 8)
1527 file_object_format = OBJECT_FORMAT_TYPE_NUMS[self.hash_format]
1529 # Verify provided object_format matches what's in the file
1530 if object_format != file_object_format:
1531 raise AssertionError(
1532 f"Object format mismatch: provided {object_format.name}, "
1533 f"but file contains {file_object_format.name}"
1534 )
1536 self.object_format = object_format
1537 self.hash_size = self.object_format.oid_length
1539 # Read length of shortened object names
1540 (self.shortened_oid_len,) = unpack_from(b">L", self._contents, 12)
1542 # Calculate offsets based on variable hash size
1543 self._fan_out_table = self._read_fan_out_table(
1544 16
1545 ) # After header (4 + 4 + 4 + 4)
1546 self._name_table_offset = 16 + 0x100 * 4
1547 self._crc32_table_offset = self._name_table_offset + self.hash_size * len(self)
1548 self._pack_offset_table_offset = self._crc32_table_offset + 4 * len(self)
1549 self._pack_offset_largetable_offset = self._pack_offset_table_offset + 4 * len(
1550 self
1551 )
1553 def _unpack_entry(self, i: int) -> tuple[RawObjectID, int, int]:
1554 return (
1555 RawObjectID(self._unpack_name(i)),
1556 self._unpack_offset(i),
1557 self._unpack_crc32_checksum(i),
1558 )
1560 def _unpack_name(self, i: int) -> bytes:
1561 offset = self._name_table_offset + i * self.hash_size
1562 return self._contents[offset : offset + self.hash_size]
1564 def _unpack_offset(self, i: int) -> int:
1565 offset_pos = self._pack_offset_table_offset + i * 4
1566 offset = unpack_from(">L", self._contents, offset_pos)[0]
1567 assert isinstance(offset, int)
1568 if offset & (2**31):
1569 large_offset_pos = (
1570 self._pack_offset_largetable_offset + (offset & (2**31 - 1)) * 8
1571 )
1572 offset = unpack_from(">Q", self._contents, large_offset_pos)[0]
1573 assert isinstance(offset, int)
1574 return offset
1576 def _unpack_crc32_checksum(self, i: int) -> int:
1577 result = unpack_from(">L", self._contents, self._crc32_table_offset + i * 4)[0]
1578 assert isinstance(result, int)
1579 return result
1582def read_pack_header_at(
1583 contents: "bytes | mmap.mmap", offset: int = 0
1584) -> tuple[int, int]:
1585 """Read the header of a pack file from a buffer.
1587 Args:
1588 contents: Buffer holding the pack
1589 offset: Offset in contents at which the header starts
1590 Returns: Tuple of (pack version, number of objects).
1591 """
1592 header = contents[offset : offset + 12]
1593 if not header:
1594 raise AssertionError("file too short to contain pack")
1595 if header[:4] != b"PACK":
1596 raise AssertionError(f"Invalid pack header {bytes(header)!r}")
1597 (version,) = unpack_from(b">L", header, 4)
1598 if version not in (2, 3):
1599 raise AssertionError(f"Version was {version}")
1600 (num_objects,) = unpack_from(b">L", header, 8)
1601 return (version, num_objects)
1604def read_pack_header(read: Callable[[int], bytes]) -> tuple[int, int]:
1605 """Read the header of a pack file.
1607 Args:
1608 read: Read function
1609 Returns: Tuple of (pack version, number of objects).
1610 """
1611 return read_pack_header_at(read(12))
1614def chunks_length(chunks: bytes | Iterable[bytes]) -> int:
1615 """Get the total length of a sequence of chunks.
1617 Args:
1618 chunks: Either a single bytes object or an iterable of bytes
1619 Returns: Total length in bytes
1620 """
1621 if isinstance(chunks, bytes):
1622 return len(chunks)
1623 else:
1624 return sum(map(len, chunks))
1627def _decode_object_header(raw: list[int]) -> tuple[int, int]:
1628 """Decode an object type and size from a pack object header."""
1629 type_num = (raw[0] >> 4) & 0x07
1630 size = raw[0] & 0x0F
1631 for i, byte in enumerate(raw[1:]):
1632 size += (byte & 0x7F) << ((i * 7) + 4)
1633 return type_num, size
1636def _decode_delta_base_offset(raw: list[int]) -> int:
1637 """Decode an OFS_DELTA base offset from its variable-length encoding."""
1638 if raw[-1] & 0x80:
1639 raise AssertionError
1640 delta_base_offset = raw[0] & 0x7F
1641 for byte in raw[1:]:
1642 delta_base_offset += 1
1643 delta_base_offset <<= 7
1644 delta_base_offset += byte & 0x7F
1645 if delta_base_offset == 0:
1646 # A zero offset makes the delta reference itself, which would
1647 # loop forever in resolve_object. git's C client rejects this
1648 # with "delta offset == 0 is invalid".
1649 raise ApplyDeltaError("OFS_DELTA has delta_base_offset of 0")
1650 return delta_base_offset
1653def unpack_object(
1654 read_all: Callable[[int], bytes],
1655 hash_func: Callable[[], "HashObject"],
1656 read_some: Callable[[int], bytes] | None = None,
1657 compute_crc32: bool = False,
1658 include_comp: bool = False,
1659 zlib_bufsize: int = _ZLIB_BUFSIZE,
1660) -> tuple[UnpackedObject, bytes]:
1661 """Unpack a Git object.
1663 Args:
1664 read_all: Read function that blocks until the number of requested
1665 bytes are read.
1666 hash_func: Hash function to use for computing object IDs.
1667 read_some: Read function that returns at least one byte, but may not
1668 return the number of bytes requested.
1669 compute_crc32: If True, compute the CRC32 of the compressed data. If
1670 False, the returned CRC32 will be None.
1671 include_comp: If True, include compressed data in the result.
1672 zlib_bufsize: An optional buffer size for zlib operations.
1673 Returns: A tuple of (unpacked, unused), where unused is the unused data
1674 leftover from decompression, and unpacked in an UnpackedObject with
1675 the following attrs set:
1677 * obj_chunks (for non-delta types)
1678 * pack_type_num
1679 * delta_base (for delta types)
1680 * comp_chunks (if include_comp is True)
1681 * decomp_chunks
1682 * decomp_len
1683 * crc32 (if compute_crc32 is True)
1684 """
1685 if read_some is None:
1686 read_some = read_all
1687 if compute_crc32:
1688 crc32 = 0
1689 else:
1690 crc32 = None
1692 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1693 type_num, size = _decode_object_header(raw)
1695 delta_base: int | bytes | None
1696 raw_base = len(raw)
1697 if type_num == OFS_DELTA:
1698 raw, crc32 = take_msb_bytes(read_all, crc32=crc32)
1699 raw_base += len(raw)
1700 delta_base = _decode_delta_base_offset(raw)
1701 elif type_num == REF_DELTA:
1702 # Determine hash size from hash_func
1703 hash_size = len(hash_func().digest())
1704 delta_base_obj = read_all(hash_size)
1705 if crc32 is not None:
1706 crc32 = binascii.crc32(delta_base_obj, crc32)
1707 delta_base = delta_base_obj
1708 raw_base += hash_size
1709 else:
1710 delta_base = None
1712 unpacked = UnpackedObject(
1713 type_num,
1714 delta_base=delta_base,
1715 decomp_len=size,
1716 crc32=crc32,
1717 hash_func=hash_func,
1718 )
1719 unused = read_zlib_chunks(
1720 read_some,
1721 unpacked,
1722 buffer_size=zlib_bufsize,
1723 include_comp=include_comp,
1724 )
1725 return unpacked, unused
1728def unpack_object_at(
1729 contents: "bytes | mmap.mmap",
1730 offset: int,
1731 hash_func: Callable[[], "HashObject"],
1732 compute_crc32: bool = False,
1733 include_comp: bool = False,
1734 zlib_bufsize: int = _ZLIB_BUFSIZE,
1735) -> tuple[UnpackedObject, int]:
1736 """Unpack a Git object from a buffer at a given offset.
1738 Like :func:`unpack_object`, but indexes the buffer directly rather than
1739 consuming a read callable, so any number of readers can work on the same
1740 buffer concurrently.
1742 Args:
1743 contents: Buffer holding the pack.
1744 offset: Offset in contents at which the object starts.
1745 hash_func: Hash function to use for computing object IDs.
1746 compute_crc32: If True, compute the CRC32 of the compressed data.
1747 include_comp: If True, include compressed data in the result.
1748 zlib_bufsize: An optional buffer size for zlib operations.
1749 Returns: A tuple of (unpacked, end), where end is the offset just past
1750 the object and unpacked is an UnpackedObject with its ``offset`` set;
1751 see :func:`unpack_object` for the other attributes.
1752 """
1753 crc32: int | None = 0 if compute_crc32 else None
1755 raw, pos, crc32 = take_msb_bytes_at(contents, offset, crc32=crc32)
1756 type_num, size = _decode_object_header(raw)
1758 delta_base: int | bytes | None
1759 if type_num == OFS_DELTA:
1760 raw, pos, crc32 = take_msb_bytes_at(contents, pos, crc32=crc32)
1761 delta_base = _decode_delta_base_offset(raw)
1762 elif type_num == REF_DELTA:
1763 hash_size = len(hash_func().digest())
1764 delta_base_obj = bytes(contents[pos : pos + hash_size])
1765 if len(delta_base_obj) != hash_size:
1766 raise AssertionError(f"unexpected end of pack data at {pos}")
1767 pos += hash_size
1768 if crc32 is not None:
1769 crc32 = binascii.crc32(delta_base_obj, crc32)
1770 delta_base = delta_base_obj
1771 else:
1772 delta_base = None
1774 unpacked = UnpackedObject(
1775 type_num,
1776 delta_base=delta_base,
1777 decomp_len=size,
1778 crc32=crc32,
1779 hash_func=hash_func,
1780 )
1781 unpacked.offset = offset
1782 end = read_zlib_chunks_at(
1783 contents,
1784 pos,
1785 unpacked,
1786 buffer_size=zlib_bufsize,
1787 include_comp=include_comp,
1788 )
1789 return unpacked, end
1792def _compute_object_size(value: tuple[int, Any]) -> int:
1793 """Compute the size of a unresolved object for use with LRUSizeCache."""
1794 (num, obj) = value
1795 if num in DELTA_TYPES:
1796 return chunks_length(obj[1])
1797 return chunks_length(obj)
1800class PackStreamReader:
1801 """Class to read a pack stream.
1803 The pack is read from a ReceivableProtocol using read() or recv() as
1804 appropriate.
1805 """
1807 def __init__(
1808 self,
1809 hash_func: Callable[[], "HashObject"],
1810 read_all: Callable[[int], bytes],
1811 read_some: Callable[[int], bytes] | None = None,
1812 zlib_bufsize: int = _ZLIB_BUFSIZE,
1813 ) -> None:
1814 """Initialize pack stream reader.
1816 Args:
1817 hash_func: Hash function to use for computing object IDs
1818 read_all: Function to read all requested bytes
1819 read_some: Function to read some bytes (optional)
1820 zlib_bufsize: Buffer size for zlib decompression
1821 """
1822 self.read_all = read_all
1823 if read_some is None:
1824 self.read_some = read_all
1825 else:
1826 self.read_some = read_some
1827 self.hash_func = hash_func
1828 self.sha = hash_func()
1829 self._hash_size = len(hash_func().digest())
1830 self._offset = 0
1831 self._rbuf = BytesIO()
1832 # trailer is a deque to avoid memory allocation on small reads
1833 self._trailer: deque[int] = deque()
1834 self._zlib_bufsize = zlib_bufsize
1836 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1837 """Read up to size bytes using the given callback.
1839 As a side effect, update the verifier's hash (excluding the last
1840 hash_size bytes read, which is the pack checksum).
1842 Args:
1843 read: The read callback to read from.
1844 size: The maximum number of bytes to read; the particular
1845 behavior is callback-specific.
1846 Returns: Bytes read
1847 """
1848 data = read(size)
1850 # maintain a trailer of the last hash_size bytes we've read
1851 n = len(data)
1852 self._offset += n
1853 tn = len(self._trailer)
1854 if n >= self._hash_size:
1855 to_pop = tn
1856 to_add = self._hash_size
1857 else:
1858 to_pop = max(n + tn - self._hash_size, 0)
1859 to_add = n
1860 self.sha.update(
1861 bytes(bytearray([self._trailer.popleft() for _ in range(to_pop)]))
1862 )
1863 self._trailer.extend(data[-to_add:])
1865 # hash everything but the trailer
1866 self.sha.update(data[:-to_add])
1867 return data
1869 def _buf_len(self) -> int:
1870 buf = self._rbuf
1871 start = buf.tell()
1872 buf.seek(0, SEEK_END)
1873 end = buf.tell()
1874 buf.seek(start)
1875 return end - start
1877 @property
1878 def offset(self) -> int:
1879 """Return current offset in the stream."""
1880 return self._offset - self._buf_len()
1882 def read(self, size: int) -> bytes:
1883 """Read, blocking until size bytes are read."""
1884 buf_len = self._buf_len()
1885 if buf_len >= size:
1886 return self._rbuf.read(size)
1887 buf_data = self._rbuf.read()
1888 self._rbuf = BytesIO()
1889 return buf_data + self._read(self.read_all, size - buf_len)
1891 def recv(self, size: int) -> bytes:
1892 """Read up to size bytes, blocking until one byte is read."""
1893 buf_len = self._buf_len()
1894 if buf_len:
1895 data = self._rbuf.read(size)
1896 if size >= buf_len:
1897 self._rbuf = BytesIO()
1898 return data
1899 return self._read(self.read_some, size)
1901 def __len__(self) -> int:
1902 """Return the number of objects in this pack."""
1903 return self._num_objects
1905 def read_objects(self, compute_crc32: bool = False) -> Iterator[UnpackedObject]:
1906 """Read the objects in this pack file.
1908 Args:
1909 compute_crc32: If True, compute the CRC32 of the compressed
1910 data. If False, the returned CRC32 will be None.
1911 Returns: Iterator over UnpackedObjects with the following members set:
1912 offset
1913 obj_type_num
1914 obj_chunks (for non-delta types)
1915 delta_base (for delta types)
1916 decomp_chunks
1917 decomp_len
1918 crc32 (if compute_crc32 is True)
1920 Raises:
1921 ChecksumMismatch: if the checksum of the pack contents does not
1922 match the checksum in the pack trailer.
1923 zlib.error: if an error occurred during zlib decompression.
1924 IOError: if an error occurred writing to the output file.
1925 """
1926 _pack_version, self._num_objects = read_pack_header(self.read)
1928 for _ in range(self._num_objects):
1929 offset = self.offset
1930 unpacked, unused = unpack_object(
1931 self.read,
1932 self.hash_func,
1933 read_some=self.recv,
1934 compute_crc32=compute_crc32,
1935 zlib_bufsize=self._zlib_bufsize,
1936 )
1937 unpacked.offset = offset
1939 # prepend any unused data to current read buffer
1940 buf = BytesIO()
1941 buf.write(unused)
1942 buf.write(self._rbuf.read())
1943 buf.seek(0)
1944 self._rbuf = buf
1946 yield unpacked
1948 if self._buf_len() < self._hash_size:
1949 # If the read buffer is full, then the last read() got the whole
1950 # trailer off the wire. If not, it means there is still some of the
1951 # trailer to read. We need to read() all hash_size bytes; N come from the
1952 # read buffer and (hash_size - N) come from the wire.
1953 self.read(self._hash_size)
1955 pack_sha = bytearray(self._trailer)
1956 if pack_sha != self.sha.digest():
1957 raise ChecksumMismatch(
1958 sha_to_hex(RawObjectID(bytes(pack_sha))), self.sha.hexdigest()
1959 )
1962class PackStreamCopier(PackStreamReader):
1963 """Class to verify a pack stream as it is being read.
1965 The pack is read from a ReceivableProtocol using read() or recv() as
1966 appropriate and written out to the given file-like object.
1967 """
1969 def __init__(
1970 self,
1971 hash_func: Callable[[], "HashObject"],
1972 read_all: Callable[[int], bytes],
1973 read_some: Callable[[int], bytes] | None,
1974 outfile: IO[bytes],
1975 delta_iter: "DeltaChainIterator[UnpackedObject] | None" = None,
1976 ) -> None:
1977 """Initialize the copier.
1979 Args:
1980 hash_func: Hash function to use for computing object IDs
1981 read_all: Read function that blocks until the number of
1982 requested bytes are read.
1983 read_some: Read function that returns at least one byte, but may
1984 not return the number of bytes requested.
1985 outfile: File-like object to write output through.
1986 delta_iter: Optional DeltaChainIterator to record deltas as we
1987 read them.
1988 """
1989 super().__init__(hash_func, read_all, read_some=read_some)
1990 self.outfile = outfile
1991 self._delta_iter = delta_iter
1993 def _read(self, read: Callable[[int], bytes], size: int) -> bytes:
1994 """Read data from the read callback and write it to the file."""
1995 data = super()._read(read, size)
1996 self.outfile.write(data)
1997 return data
1999 def verify(self, progress: Callable[..., None] | None = None) -> None:
2000 """Verify a pack stream and write it to the output file.
2002 See PackStreamReader.iterobjects for a list of exceptions this may
2003 throw.
2004 """
2005 i = 0 # default count of entries if read_objects() is empty
2006 for i, unpacked in enumerate(self.read_objects()):
2007 if self._delta_iter:
2008 self._delta_iter.record(unpacked)
2009 if progress is not None:
2010 progress(f"copying pack entries: {i}/{len(self)}\r".encode("ascii"))
2011 if progress is not None:
2012 progress(f"copied {i} pack entries\n".encode("ascii"))
2015def obj_sha(
2016 type: int,
2017 chunks: bytes | Iterable[bytes],
2018 hash_func: Callable[[], "HashObject"] = sha1,
2019) -> bytes:
2020 """Compute the SHA for a numeric type and object chunks.
2022 Args:
2023 type: Object type number
2024 chunks: Object data chunks
2025 hash_func: Hash function to use (defaults to sha1)
2027 Returns:
2028 Binary hash digest
2029 """
2030 sha = hash_func()
2031 sha.update(object_header(type, chunks_length(chunks)))
2032 if isinstance(chunks, bytes):
2033 sha.update(chunks)
2034 else:
2035 for chunk in chunks:
2036 sha.update(chunk)
2037 return sha.digest()
2040def compute_file_sha(
2041 f: IO[bytes],
2042 hash_func: Callable[[], "HashObject"],
2043 start_ofs: int = 0,
2044 end_ofs: int = 0,
2045 buffer_size: int = 1 << 16,
2046) -> "HashObject":
2047 """Hash a portion of a file into a new SHA.
2049 Args:
2050 f: A file-like object to read from that supports seek().
2051 hash_func: A callable that returns a new HashObject.
2052 start_ofs: The offset in the file to start reading at.
2053 end_ofs: The offset in the file to end reading at, relative to the
2054 end of the file.
2055 buffer_size: A buffer size for reading.
2056 Returns: A new SHA object updated with data read from the file.
2057 """
2058 sha = hash_func()
2059 f.seek(0, SEEK_END)
2060 length = f.tell()
2061 if start_ofs < 0:
2062 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
2063 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
2064 raise AssertionError(
2065 f"Attempt to read beyond file length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, file length: {length}"
2066 )
2067 todo = length + end_ofs - start_ofs
2068 f.seek(start_ofs)
2069 while todo:
2070 data = f.read(min(todo, buffer_size))
2071 sha.update(data)
2072 todo -= len(data)
2073 return sha
2076def compute_buffer_sha(
2077 contents: "bytes | mmap.mmap",
2078 hash_func: Callable[[], "HashObject"],
2079 start_ofs: int = 0,
2080 end_ofs: int = 0,
2081) -> "HashObject":
2082 """Hash a portion of a buffer into a new SHA.
2084 The region is hashed in one pass through a memoryview, so a mapped pack
2085 is never copied. The view is released before returning rather than left
2086 to the garbage collector, since ``mmap.close()`` raises BufferError while
2087 an export is alive.
2089 Args:
2090 contents: Buffer to hash.
2091 hash_func: A callable that returns a new HashObject.
2092 start_ofs: The offset in the buffer to start hashing at.
2093 end_ofs: The offset to end hashing at, relative to the end of the
2094 buffer.
2095 Returns: A new SHA object updated with data read from the buffer.
2096 """
2097 sha = hash_func()
2098 length = len(contents)
2099 if start_ofs < 0:
2100 raise AssertionError(f"start_ofs cannot be negative: {start_ofs}")
2101 if (end_ofs < 0 and length + end_ofs < start_ofs) or end_ofs > length:
2102 raise AssertionError(
2103 f"Attempt to read beyond buffer length. start_ofs: {start_ofs}, end_ofs: {end_ofs}, buffer length: {length}"
2104 )
2105 with memoryview(contents) as view:
2106 sha.update(view[start_ofs : length + end_ofs])
2107 return sha
2110class PackData:
2111 """The data contained in a packfile.
2113 Pack files can be accessed both sequentially for exploding a pack, and
2114 directly with the help of an index to retrieve a specific object.
2116 The objects within are either complete or a delta against another.
2118 The header is variable length. If the MSB of each byte is set then it
2119 indicates that the subsequent byte is still part of the header.
2120 For the first byte the next MS bits are the type, which tells you the type
2121 of object, and whether it is a delta. The LS byte is the lowest bits of the
2122 size. For each subsequent byte the LS 7 bits are the next MS bits of the
2123 size, i.e. the last byte of the header contains the MS bits of the size.
2125 For the complete objects the data is stored as zlib deflated data.
2126 The size in the header is the uncompressed object size, so to uncompress
2127 you need to just keep feeding data to zlib until you get an object back,
2128 or it errors on bad data. This is done here by reading from the mapped
2129 pack contents starting at the deflated object.
2131 Currently there are no integrity checks done. Also no attempt is made to
2132 try and detect the delta case, or a request for an object at the wrong
2133 position. It will all just throw a zlib or KeyError.
2134 """
2136 def __init__(
2137 self,
2138 filename: str | os.PathLike[str],
2139 object_format: ObjectFormat,
2140 file: IO[bytes] | None = None,
2141 size: int | None = None,
2142 *,
2143 delta_window_size: int | None = None,
2144 window_memory: int | None = None,
2145 delta_cache_size: int | None = None,
2146 depth: int | None = None,
2147 threads: int | None = None,
2148 big_file_threshold: int | None = None,
2149 delta_base_cache_limit: int | None = None,
2150 ) -> None:
2151 """Create a PackData object representing the pack in the given filename.
2153 The file must exist and stay readable until the object is disposed of.
2154 It must also stay the same size. It is mapped into memory on open, so
2155 reads index the mapping directly rather than sharing a file position.
2157 The size argument is not trusted for checksum offsets, which are
2158 derived from the mapped length instead. When given it is checked
2159 against that length, so a caller passing a stale size gets an error
2160 rather than silently wrong offsets.
2161 """
2162 self._filename = filename
2163 self.object_format = object_format
2164 self._header_size = 12
2165 self.delta_window_size = delta_window_size
2166 self.window_memory = window_memory
2167 self.delta_cache_size = delta_cache_size
2168 self.depth = depth
2169 self.threads = threads
2170 self.big_file_threshold = big_file_threshold
2171 self.delta_base_cache_limit = delta_base_cache_limit
2172 self._file: IO[bytes]
2173 self._contents: bytes | mmap.mmap | None = None
2175 if file is None:
2176 self._file = GitFile(self._filename, "rb")
2177 self._close_file = True
2178 else:
2179 # A caller-supplied file stays the caller's to close; it may well
2180 # keep writing to it after we are done reading.
2181 self._file = file
2182 self._close_file = False
2183 try:
2184 # Map the pack once; every read indexes this buffer at an explicit
2185 # offset, so concurrent reads never contend on a file position.
2186 self._contents, self._size = _load_file_contents(self._file)
2187 if size is not None and size != self._size:
2188 raise AssertionError(
2189 f"{self._filename} is {self._size} bytes, but caller said {size}"
2190 )
2191 minimum_size = self._header_size + self.object_format.oid_length
2192 if self._size < minimum_size:
2193 raise AssertionError(
2194 f"{self._filename} is too small for a packfile ({self._size} < {minimum_size})"
2195 )
2196 (_version, self._num_objects) = read_pack_header_at(self._contents)
2198 # Use delta_base_cache_limit, then delta_cache_size, then default
2199 cache_size = (
2200 delta_base_cache_limit
2201 or delta_cache_size
2202 or DEFAULT_DELTA_BASE_CACHE_LIMIT
2203 )
2204 self._init_offset_cache(cache_size)
2205 except BaseException:
2206 self.close()
2207 raise
2209 def _init_offset_cache(self, max_size: int) -> None:
2210 """Initialize the resolved object cache."""
2211 self._offset_cache = LRUSizeCache[int, tuple[int, OldUnpackedObject]](
2212 max_size, compute_size=_compute_object_size
2213 )
2214 # Cache hits update the LRU linked list, so reads need locking too.
2215 self._offset_cache_lock = threading.Lock()
2217 @property
2218 def filename(self) -> str:
2219 """Get the filename of the pack file.
2221 Returns:
2222 Base filename without directory path
2223 """
2224 return os.path.basename(self._filename)
2226 @property
2227 def path(self) -> str | os.PathLike[str]:
2228 """Get the full path of the pack file.
2230 Returns:
2231 Full path to the pack file
2232 """
2233 return self._filename
2235 @classmethod
2236 def from_file(
2237 cls,
2238 file: IO[bytes],
2239 object_format: ObjectFormat,
2240 size: int | None = None,
2241 ) -> "PackData":
2242 """Create a PackData object from an open file.
2244 Args:
2245 file: Open file object
2246 object_format: Object format
2247 size: Optional expected file size, checked against the mapped length
2249 Returns:
2250 PackData instance
2251 """
2252 return cls(str(file), object_format, file=file, size=size)
2254 @classmethod
2255 def from_path(
2256 cls,
2257 path: str | os.PathLike[str],
2258 object_format: ObjectFormat,
2259 ) -> "PackData":
2260 """Create a PackData object from a file path.
2262 Args:
2263 path: Path to the pack file
2264 object_format: Object format
2266 Returns:
2267 PackData instance
2268 """
2269 return cls(filename=path, object_format=object_format)
2271 def _buffer(self) -> "bytes | mmap.mmap":
2272 """Return the mapped pack contents."""
2273 contents = self._contents
2274 if contents is None:
2275 raise ValueError(f"read from closed PackData: {self._filename}")
2276 return contents
2278 def close(self) -> None:
2279 """Release the mapping, and the pack file if we opened it.
2281 Callers must drop the mapping before writing to or renaming the pack:
2282 on Windows a live mapping locks the file.
2283 """
2284 contents = self._contents
2285 self._contents = None
2286 _close_file_contents(contents)
2287 if self._file is not None:
2288 if self._close_file:
2289 self._file.close()
2290 self._file = None # type: ignore
2292 def __del__(self) -> None:
2293 """Ensure pack file is closed when PackData is garbage collected."""
2294 if getattr(self, "_file", None) is not None:
2295 import warnings
2297 warnings.warn(
2298 f"unclosed PackData {self!r}",
2299 ResourceWarning,
2300 stacklevel=2,
2301 source=self,
2302 )
2303 try:
2304 self.close()
2305 except Exception:
2306 # Ignore errors during cleanup
2307 pass
2309 def __enter__(self) -> Self:
2310 """Enter context manager."""
2311 return self
2313 def __exit__(
2314 self,
2315 type: type | None,
2316 value: BaseException | None,
2317 traceback: TracebackType | None,
2318 ) -> None:
2319 """Exit context manager."""
2320 self.close()
2322 def __eq__(self, other: object) -> bool:
2323 """Check equality with another object."""
2324 if isinstance(other, PackData):
2325 return self.get_stored_checksum() == other.get_stored_checksum()
2326 return False
2328 def __len__(self) -> int:
2329 """Returns the number of objects in this pack."""
2330 return self._num_objects
2332 def calculate_checksum(self) -> bytes:
2333 """Calculate the checksum for this pack.
2335 Returns: Binary digest (size depends on hash algorithm)
2336 """
2337 return compute_buffer_sha(
2338 self._buffer(),
2339 hash_func=self.object_format.hash_func,
2340 end_ofs=-self.object_format.oid_length,
2341 ).digest()
2343 def iter_unpacked(self, *, include_comp: bool = False) -> Iterator[UnpackedObject]:
2344 """Iterate over unpacked objects in the pack."""
2345 if self._num_objects is None:
2346 return
2348 contents = self._buffer()
2349 offset = self._header_size
2350 for _ in range(self._num_objects):
2351 unpacked, offset = unpack_object_at(
2352 contents,
2353 offset,
2354 self.object_format.hash_func,
2355 compute_crc32=False,
2356 include_comp=include_comp,
2357 )
2358 yield unpacked
2360 def iterentries(
2361 self,
2362 progress: Callable[[int, int], None] | None = None,
2363 resolve_ext_ref: ResolveExtRefFn | None = None,
2364 ) -> Iterator[PackIndexEntry]:
2365 """Yield entries summarizing the contents of this pack.
2367 Args:
2368 progress: Progress function, called with current and total
2369 object count.
2370 resolve_ext_ref: Optional function to resolve external references
2371 Returns: iterator of tuples with (sha, offset, crc32)
2372 """
2373 num_objects = self._num_objects
2374 indexer = PackIndexer.for_pack_data(self, resolve_ext_ref=resolve_ext_ref)
2375 for i, result in enumerate(indexer):
2376 if progress is not None:
2377 progress(i, num_objects)
2378 yield result
2380 def sorted_entries(
2381 self,
2382 progress: Callable[[int, int], None] | None = None,
2383 resolve_ext_ref: ResolveExtRefFn | None = None,
2384 ) -> list[tuple[RawObjectID, int, int]]:
2385 """Return entries in this pack, sorted by SHA.
2387 Args:
2388 progress: Progress function, called with current and total
2389 object count
2390 resolve_ext_ref: Optional function to resolve external references
2391 Returns: Iterator of tuples with (sha, offset, crc32)
2392 """
2393 return sorted(
2394 self.iterentries(progress=progress, resolve_ext_ref=resolve_ext_ref) # type: ignore
2395 )
2397 def create_index_v1(
2398 self,
2399 filename: str,
2400 progress: Callable[..., None] | None = None,
2401 resolve_ext_ref: ResolveExtRefFn | None = None,
2402 ) -> bytes:
2403 """Create a version 1 file for this data file.
2405 Args:
2406 filename: Index filename.
2407 progress: Progress report function
2408 resolve_ext_ref: Optional function to resolve external references
2409 Returns: Checksum of index file
2410 """
2411 entries = self.sorted_entries(
2412 progress=progress, resolve_ext_ref=resolve_ext_ref
2413 )
2414 checksum = self.calculate_checksum()
2415 with GitFile(filename, "wb") as f:
2416 write_pack_index_v1(
2417 f,
2418 entries,
2419 checksum,
2420 )
2421 return checksum
2423 def create_index_v2(
2424 self,
2425 filename: str,
2426 progress: Callable[..., None] | None = None,
2427 resolve_ext_ref: ResolveExtRefFn | None = None,
2428 ) -> bytes:
2429 """Create a version 2 index file for this data file.
2431 Args:
2432 filename: Index filename.
2433 progress: Progress report function
2434 resolve_ext_ref: Optional function to resolve external references
2435 Returns: Checksum of index file
2436 """
2437 entries = self.sorted_entries(
2438 progress=progress, resolve_ext_ref=resolve_ext_ref
2439 )
2440 with GitFile(filename, "wb") as f:
2441 return write_pack_index_v2(f, entries, self.calculate_checksum())
2443 def create_index_v3(
2444 self,
2445 filename: str,
2446 progress: Callable[..., None] | None = None,
2447 resolve_ext_ref: ResolveExtRefFn | None = None,
2448 hash_format: int | None = None,
2449 ) -> bytes:
2450 """Create a version 3 index file for this data file.
2452 Args:
2453 filename: Index filename.
2454 progress: Progress report function
2455 resolve_ext_ref: Function to resolve external references
2456 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
2457 Returns: Checksum of index file
2458 """
2459 entries = self.sorted_entries(
2460 progress=progress, resolve_ext_ref=resolve_ext_ref
2461 )
2462 with GitFile(filename, "wb") as f:
2463 if hash_format is None:
2464 hash_format = 1 # Default to SHA-1
2465 return write_pack_index_v3(
2466 f, entries, self.calculate_checksum(), hash_format=hash_format
2467 )
2469 def create_index(
2470 self,
2471 filename: str,
2472 progress: Callable[..., None] | None = None,
2473 version: int = 2,
2474 resolve_ext_ref: ResolveExtRefFn | None = None,
2475 hash_format: int | None = None,
2476 ) -> bytes:
2477 """Create an index file for this data file.
2479 Args:
2480 filename: Index filename.
2481 progress: Progress report function
2482 version: Index version (1, 2, or 3)
2483 resolve_ext_ref: Function to resolve external references
2484 hash_format: Hash algorithm identifier for v3 (1 = SHA-1, 2 = SHA-256)
2485 Returns: Checksum of index file
2486 """
2487 if version == 1:
2488 return self.create_index_v1(
2489 filename, progress, resolve_ext_ref=resolve_ext_ref
2490 )
2491 elif version == 2:
2492 return self.create_index_v2(
2493 filename, progress, resolve_ext_ref=resolve_ext_ref
2494 )
2495 elif version == 3:
2496 return self.create_index_v3(
2497 filename,
2498 progress,
2499 resolve_ext_ref=resolve_ext_ref,
2500 hash_format=hash_format,
2501 )
2502 else:
2503 raise ValueError(f"unknown index format {version}")
2505 def get_stored_checksum(self) -> bytes:
2506 """Return the expected checksum stored in this pack."""
2507 checksum_size = self.object_format.oid_length
2508 return bytes(self._buffer()[self._size - checksum_size :])
2510 def check(self) -> None:
2511 """Check the consistency of this pack."""
2512 actual = self.calculate_checksum()
2513 stored = self.get_stored_checksum()
2514 if actual != stored:
2515 raise ChecksumMismatch(stored, actual)
2517 def get_unpacked_object_at(
2518 self, offset: int, *, include_comp: bool = False
2519 ) -> UnpackedObject:
2520 """Given offset in the packfile return a UnpackedObject."""
2521 assert offset >= self._header_size
2522 unpacked, _ = unpack_object_at(
2523 self._buffer(),
2524 offset,
2525 self.object_format.hash_func,
2526 include_comp=include_comp,
2527 )
2528 return unpacked
2530 def _get_cached_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2531 """Return the cached object at offset, or raise KeyError."""
2532 # Hot path: acquire/release directly rather than using the context
2533 # manager, which measurably speeds up cache hits on get_object_at.
2534 self._offset_cache_lock.acquire()
2535 try:
2536 return self._offset_cache[offset]
2537 finally:
2538 self._offset_cache_lock.release()
2540 def _cache_object_at(
2541 self, offset: int, type_num: int, chunks: OldUnpackedObject
2542 ) -> None:
2543 """Cache a resolved object at offset."""
2544 with self._offset_cache_lock:
2545 self._offset_cache[offset] = (type_num, chunks)
2547 def get_object_at(self, offset: int) -> tuple[int, OldUnpackedObject]:
2548 """Given an offset in to the packfile return the object that is there.
2550 Using the associated index the location of an object can be looked up,
2551 and then the packfile can be asked directly for that object using this
2552 function.
2553 """
2554 try:
2555 return self._get_cached_object_at(offset)
2556 except KeyError:
2557 pass
2558 unpacked = self.get_unpacked_object_at(offset, include_comp=False)
2559 return (unpacked.pack_type_num, unpacked._obj())
2562T = TypeVar("T")
2565class DeltaChainIterator(Generic[T]):
2566 """Abstract iterator over pack data based on delta chains.
2568 Each object in the pack is guaranteed to be inflated exactly once,
2569 regardless of how many objects reference it as a delta base. As a result,
2570 memory usage is proportional to the length of the longest delta chain.
2572 Subclasses can override _result to define the result type of the iterator.
2573 By default, results are UnpackedObjects with the following members set:
2575 * offset
2576 * obj_type_num
2577 * obj_chunks
2578 * pack_type_num
2579 * delta_base (for delta types)
2580 * comp_chunks (if _include_comp is True)
2581 * decomp_chunks
2582 * decomp_len
2583 * crc32 (if _compute_crc32 is True)
2584 """
2586 _compute_crc32 = False
2587 _include_comp = False
2589 def __init__(
2590 self,
2591 file_obj: IO[bytes] | None,
2592 hash_func: Callable[[], "HashObject"],
2593 *,
2594 resolve_ext_ref: ResolveExtRefFn | None = None,
2595 object_format: "ObjectFormat | None" = None,
2596 ) -> None:
2597 """Initialize DeltaChainIterator.
2599 Args:
2600 file_obj: File object to read pack data from
2601 hash_func: Hash function to use for computing object IDs
2602 resolve_ext_ref: Optional function to resolve external references
2603 object_format: Optional object format. Required by subclasses
2604 that materialise objects (e.g. PackInflater) when iterating
2605 packs in a non-default hash algorithm such as SHA-256.
2606 """
2607 self._file = file_obj
2608 self._contents: bytes | mmap.mmap | None = None
2609 self.hash_func = hash_func
2610 self._object_format = object_format
2611 self._resolve_ext_ref = resolve_ext_ref
2612 self._pending_ofs: dict[int, list[int]] = defaultdict(list)
2613 self._pending_ref: dict[bytes, list[int]] = defaultdict(list)
2614 self._full_ofs: list[tuple[int, int]] = []
2615 self._ext_refs: list[RawObjectID] = []
2617 @classmethod
2618 def for_pack_data(
2619 cls, pack_data: PackData, resolve_ext_ref: ResolveExtRefFn | None = None
2620 ) -> "DeltaChainIterator[T]":
2621 """Create a DeltaChainIterator from pack data.
2623 Args:
2624 pack_data: PackData object to iterate
2625 resolve_ext_ref: Optional function to resolve external refs
2627 Returns:
2628 DeltaChainIterator instance
2629 """
2630 walker = cls(
2631 None,
2632 pack_data.object_format.hash_func,
2633 resolve_ext_ref=resolve_ext_ref,
2634 object_format=pack_data.object_format,
2635 )
2636 walker.set_pack_data(pack_data)
2637 for unpacked in pack_data.iter_unpacked(include_comp=False):
2638 walker.record(unpacked)
2639 return walker
2641 @classmethod
2642 def for_pack_subset(
2643 cls,
2644 pack: "Pack",
2645 shas: Iterable[ObjectID | RawObjectID],
2646 *,
2647 allow_missing: bool = False,
2648 resolve_ext_ref: ResolveExtRefFn | None = None,
2649 ) -> "DeltaChainIterator[T]":
2650 """Create a DeltaChainIterator for a subset of objects.
2652 Args:
2653 pack: Pack object containing the data
2654 shas: Iterable of object SHAs to include
2655 allow_missing: If True, skip missing objects
2656 resolve_ext_ref: Optional function to resolve external refs
2658 Returns:
2659 DeltaChainIterator instance
2660 """
2661 walker = cls(
2662 None,
2663 pack.object_format.hash_func,
2664 resolve_ext_ref=resolve_ext_ref,
2665 object_format=pack.object_format,
2666 )
2667 walker.set_pack_data(pack.data)
2668 todo = set()
2669 for sha in shas:
2670 try:
2671 off = pack.index.object_offset(sha)
2672 except KeyError:
2673 if not allow_missing:
2674 raise
2675 else:
2676 todo.add(off)
2677 done = set()
2678 while todo:
2679 off = todo.pop()
2680 unpacked = pack.data.get_unpacked_object_at(off)
2681 walker.record(unpacked)
2682 done.add(off)
2683 base_ofs = None
2684 if unpacked.pack_type_num == OFS_DELTA:
2685 assert unpacked.offset is not None
2686 assert unpacked.delta_base is not None
2687 assert isinstance(unpacked.delta_base, int)
2688 base_ofs = unpacked.offset - unpacked.delta_base
2689 elif unpacked.pack_type_num == REF_DELTA:
2690 with suppress(KeyError):
2691 assert isinstance(unpacked.delta_base, bytes)
2692 base_ofs = pack.index.object_offset(
2693 RawObjectID(unpacked.delta_base)
2694 )
2695 if base_ofs is not None and base_ofs not in done:
2696 todo.add(base_ofs)
2697 return walker
2699 def record(self, unpacked: UnpackedObject) -> None:
2700 """Record an unpacked object for later processing.
2702 Args:
2703 unpacked: UnpackedObject to record
2704 """
2705 type_num = unpacked.pack_type_num
2706 offset = unpacked.offset
2707 assert offset is not None
2708 if type_num == OFS_DELTA:
2709 assert unpacked.delta_base is not None
2710 assert isinstance(unpacked.delta_base, int)
2711 base_offset = offset - unpacked.delta_base
2712 self._pending_ofs[base_offset].append(offset)
2713 elif type_num == REF_DELTA:
2714 assert isinstance(unpacked.delta_base, bytes)
2715 self._pending_ref[unpacked.delta_base].append(offset)
2716 else:
2717 self._full_ofs.append((offset, type_num))
2719 def set_pack_data(self, pack_data: PackData) -> None:
2720 """Set the pack data for iteration.
2722 Args:
2723 pack_data: PackData object to use
2724 """
2725 self._file = None
2726 self._contents = pack_data._buffer()
2728 def _walk_all_chains(self) -> Iterator[T]:
2729 for offset, type_num in self._full_ofs:
2730 yield from self._follow_chain(offset, type_num, None)
2731 yield from self._walk_ref_chains()
2732 assert not self._pending_ofs, repr(self._pending_ofs)
2734 def _ensure_no_pending(self) -> None:
2735 if self._pending_ref:
2736 raise UnresolvedDeltas(
2737 [sha_to_hex(RawObjectID(s)) for s in self._pending_ref]
2738 )
2740 def _walk_ref_chains(self) -> Iterator[T]:
2741 if not self._resolve_ext_ref:
2742 self._ensure_no_pending()
2743 return
2745 for base_sha, pending in sorted(self._pending_ref.items()):
2746 if base_sha not in self._pending_ref:
2747 continue
2748 try:
2749 type_num, chunks = self._resolve_ext_ref(RawObjectID(base_sha))
2750 except KeyError:
2751 # Not an external ref, but may depend on one. Either it will
2752 # get popped via a _follow_chain call, or we will raise an
2753 # error below.
2754 continue
2755 self._ext_refs.append(RawObjectID(base_sha))
2756 self._pending_ref.pop(base_sha)
2757 for new_offset in pending:
2758 yield from self._follow_chain(new_offset, type_num, chunks)
2760 self._ensure_no_pending()
2762 def _result(self, unpacked: UnpackedObject) -> T:
2763 raise NotImplementedError
2765 def _resolve_object(
2766 self,
2767 offset: int,
2768 obj_type_num: int,
2769 base_chunks: bytes | list[bytes] | None,
2770 ) -> UnpackedObject:
2771 if self._contents is not None:
2772 unpacked, _ = unpack_object_at(
2773 self._contents,
2774 offset,
2775 self.hash_func,
2776 compute_crc32=self._compute_crc32,
2777 include_comp=self._include_comp,
2778 )
2779 else:
2780 # add_thin_pack may still be writing to this file, so it cannot be
2781 # mapped up front; read through the file position instead.
2782 assert self._file is not None
2783 self._file.seek(offset)
2784 unpacked, _ = unpack_object(
2785 self._file.read,
2786 self.hash_func,
2787 read_some=None,
2788 compute_crc32=self._compute_crc32,
2789 include_comp=self._include_comp,
2790 )
2791 unpacked.offset = offset
2792 if base_chunks is None:
2793 assert unpacked.pack_type_num == obj_type_num
2794 else:
2795 assert unpacked.pack_type_num in DELTA_TYPES
2796 unpacked.obj_type_num = obj_type_num
2797 unpacked.obj_chunks = apply_delta(base_chunks, unpacked.decomp_chunks)
2798 # A delta that resolves to a zero-byte payload for a
2799 # commit/tree/tag is malformed: ``_parse_message`` /
2800 # ``parse_tree`` accept the empty input silently, so without
2801 # this guard a too-short delta could materialise an
2802 # otherwise-valid SHA pointing at an empty commit object
2803 # (which ``git fsck`` rejects). Only blobs may legitimately
2804 # be empty, and an empty blob would never be stored as a
2805 # delta in practice.
2806 # Blob.type_num == 3 (avoid the import cycle).
2807 if obj_type_num != 3 and chunks_length(unpacked.obj_chunks) == 0:
2808 raise ApplyDeltaError(
2809 f"delta resolved to empty payload for type {obj_type_num}"
2810 )
2811 return unpacked
2813 def _follow_chain(
2814 self,
2815 offset: int,
2816 obj_type_num: int,
2817 base_chunks: bytes | list[bytes] | None,
2818 ) -> Iterator[T]:
2819 # Unlike PackData.get_object_at, there is no need to cache offsets as
2820 # this approach by design inflates each object exactly once.
2821 todo = [(offset, obj_type_num, base_chunks)]
2822 while todo:
2823 (offset, obj_type_num, base_chunks) = todo.pop()
2824 unpacked = self._resolve_object(offset, obj_type_num, base_chunks)
2825 yield self._result(unpacked)
2827 assert unpacked.offset is not None
2828 unblocked = chain(
2829 self._pending_ofs.pop(unpacked.offset, []),
2830 self._pending_ref.pop(unpacked.sha(), []),
2831 )
2832 todo.extend(
2833 (new_offset, unpacked.obj_type_num, unpacked.obj_chunks) # type: ignore
2834 for new_offset in unblocked
2835 )
2837 def __iter__(self) -> Iterator[T]:
2838 """Iterate over objects in the pack."""
2839 return self._walk_all_chains()
2841 def ext_refs(self) -> list[RawObjectID]:
2842 """Return external references."""
2843 return self._ext_refs
2846class UnpackedObjectIterator(DeltaChainIterator[UnpackedObject]):
2847 """Delta chain iterator that yield unpacked objects."""
2849 def _result(self, unpacked: UnpackedObject) -> UnpackedObject:
2850 """Return the unpacked object.
2852 Args:
2853 unpacked: The unpacked object
2855 Returns:
2856 The unpacked object unchanged
2857 """
2858 return unpacked
2861class PackIndexer(DeltaChainIterator[PackIndexEntry]):
2862 """Delta chain iterator that yields index entries."""
2864 _compute_crc32 = True
2866 def _result(self, unpacked: UnpackedObject) -> PackIndexEntry:
2867 """Convert unpacked object to pack index entry.
2869 Args:
2870 unpacked: The unpacked object
2872 Returns:
2873 Tuple of (sha, offset, crc32) for index entry
2874 """
2875 assert unpacked.offset is not None
2876 return unpacked.sha(), unpacked.offset, unpacked.crc32
2879class PackInflater(DeltaChainIterator[ShaFile]):
2880 """Delta chain iterator that yields ShaFile objects."""
2882 def _result(self, unpacked: UnpackedObject) -> ShaFile:
2883 """Convert unpacked object to ShaFile.
2885 Args:
2886 unpacked: The unpacked object
2888 Returns:
2889 ShaFile object from the unpacked data
2890 """
2891 assert unpacked.obj_type_num is not None and unpacked.obj_chunks is not None
2892 return ShaFile.from_raw_chunks(
2893 unpacked.obj_type_num,
2894 unpacked.obj_chunks,
2895 object_format=self._object_format,
2896 )
2899class SHA1Reader(BinaryIO):
2900 """Wrapper for file-like object that remembers the SHA1 of its data."""
2902 def __init__(self, f: IO[bytes]) -> None:
2903 """Initialize SHA1Reader.
2905 Args:
2906 f: File-like object to wrap
2907 """
2908 self.f = f
2909 self.sha1 = sha1(b"")
2911 def read(self, size: int = -1) -> bytes:
2912 """Read bytes and update SHA1.
2914 Args:
2915 size: Number of bytes to read, -1 for all
2917 Returns:
2918 Bytes read from file
2919 """
2920 data = self.f.read(size)
2921 self.sha1.update(data)
2922 return data
2924 def check_sha(self, allow_empty: bool = False) -> None:
2925 """Check if the SHA1 matches the expected value.
2927 Args:
2928 allow_empty: Allow empty SHA1 hash
2930 Raises:
2931 ChecksumMismatch: If SHA1 doesn't match
2932 """
2933 stored = self.f.read(20)
2934 # If git option index.skipHash is set the index will be empty
2935 if stored != self.sha1.digest() and (
2936 not allow_empty
2937 or (
2938 len(stored) == 20
2939 and sha_to_hex(RawObjectID(stored))
2940 != b"0000000000000000000000000000000000000000"
2941 )
2942 ):
2943 raise ChecksumMismatch(
2944 self.sha1.hexdigest(),
2945 sha_to_hex(RawObjectID(stored)) if stored else b"",
2946 )
2948 def close(self) -> None:
2949 """Close the underlying file."""
2950 return self.f.close()
2952 def tell(self) -> int:
2953 """Return current file position."""
2954 return self.f.tell()
2956 # BinaryIO abstract methods
2957 def readable(self) -> bool:
2958 """Check if file is readable."""
2959 return True
2961 def writable(self) -> bool:
2962 """Check if file is writable."""
2963 return False
2965 def seekable(self) -> bool:
2966 """Check if file is seekable."""
2967 return getattr(self.f, "seekable", lambda: False)()
2969 def seek(self, offset: int, whence: int = 0) -> int:
2970 """Seek to position in file.
2972 Args:
2973 offset: Position offset
2974 whence: Reference point (0=start, 1=current, 2=end)
2976 Returns:
2977 New file position
2978 """
2979 return self.f.seek(offset, whence)
2981 def flush(self) -> None:
2982 """Flush the file buffer."""
2983 if hasattr(self.f, "flush"):
2984 self.f.flush()
2986 def readline(self, size: int = -1) -> bytes:
2987 """Read a line from the file.
2989 Args:
2990 size: Maximum bytes to read
2992 Returns:
2993 Line read from file
2994 """
2995 return self.f.readline(size)
2997 def readlines(self, hint: int = -1) -> list[bytes]:
2998 """Read all lines from the file.
3000 Args:
3001 hint: Approximate number of bytes to read
3003 Returns:
3004 List of lines
3005 """
3006 return self.f.readlines(hint)
3008 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3009 """Write multiple lines to the file (not supported)."""
3010 raise UnsupportedOperation("writelines")
3012 def write(self, data: bytes, /) -> int: # type: ignore[override]
3013 """Write data to the file (not supported)."""
3014 raise UnsupportedOperation("write")
3016 def __enter__(self) -> Self:
3017 """Enter context manager."""
3018 return self
3020 def __exit__(
3021 self,
3022 type: type | None,
3023 value: BaseException | None,
3024 traceback: TracebackType | None,
3025 ) -> None:
3026 """Exit context manager and close file."""
3027 self.close()
3029 def __iter__(self) -> "SHA1Reader":
3030 """Return iterator for reading file lines."""
3031 return self
3033 def __next__(self) -> bytes:
3034 """Get next line from file.
3036 Returns:
3037 Next line
3039 Raises:
3040 StopIteration: When no more lines
3041 """
3042 line = self.readline()
3043 if not line:
3044 raise StopIteration
3045 return line
3047 def fileno(self) -> int:
3048 """Return file descriptor number."""
3049 return self.f.fileno()
3051 def isatty(self) -> bool:
3052 """Check if file is a terminal."""
3053 return getattr(self.f, "isatty", lambda: False)()
3055 def truncate(self, size: int | None = None) -> int:
3056 """Not supported for read-only file.
3058 Raises:
3059 UnsupportedOperation: Always raised
3060 """
3061 raise UnsupportedOperation("truncate")
3064class SHA1Writer(BinaryIO):
3065 """Wrapper for file-like object that remembers the SHA1 of its data."""
3067 def __init__(self, f: BinaryIO | IO[bytes]) -> None:
3068 """Initialize SHA1Writer.
3070 Args:
3071 f: File-like object to wrap
3072 """
3073 self.f = f
3074 self.length = 0
3075 self.sha1 = sha1(b"")
3076 self.digest: bytes | None = None
3078 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
3079 """Write data and update SHA1.
3081 Args:
3082 data: Data to write
3084 Returns:
3085 Number of bytes written
3086 """
3087 self.sha1.update(data)
3088 written = self.f.write(data)
3089 self.length += written
3090 return written
3092 def write_sha(self) -> bytes:
3093 """Write the SHA1 digest to the file.
3095 Returns:
3096 The SHA1 digest bytes
3097 """
3098 sha = self.sha1.digest()
3099 assert len(sha) == 20
3100 self.f.write(sha)
3101 self.length += len(sha)
3102 return sha
3104 def close(self) -> None:
3105 """Close the pack file and finalize the SHA."""
3106 self.digest = self.write_sha()
3107 self.f.close()
3109 def offset(self) -> int:
3110 """Get the total number of bytes written.
3112 Returns:
3113 Total bytes written
3114 """
3115 return self.length
3117 def tell(self) -> int:
3118 """Return current file position."""
3119 return self.f.tell()
3121 # BinaryIO abstract methods
3122 def readable(self) -> bool:
3123 """Check if file is readable."""
3124 return False
3126 def writable(self) -> bool:
3127 """Check if file is writable."""
3128 return True
3130 def seekable(self) -> bool:
3131 """Check if file is seekable."""
3132 return getattr(self.f, "seekable", lambda: False)()
3134 def seek(self, offset: int, whence: int = 0) -> int:
3135 """Seek to position in file.
3137 Args:
3138 offset: Position offset
3139 whence: Reference point (0=start, 1=current, 2=end)
3141 Returns:
3142 New file position
3143 """
3144 return self.f.seek(offset, whence)
3146 def flush(self) -> None:
3147 """Flush the file buffer."""
3148 if hasattr(self.f, "flush"):
3149 self.f.flush()
3151 def readline(self, size: int = -1) -> bytes:
3152 """Not supported for write-only file.
3154 Raises:
3155 UnsupportedOperation: Always raised
3156 """
3157 raise UnsupportedOperation("readline")
3159 def readlines(self, hint: int = -1) -> list[bytes]:
3160 """Not supported for write-only file.
3162 Raises:
3163 UnsupportedOperation: Always raised
3164 """
3165 raise UnsupportedOperation("readlines")
3167 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3168 """Write multiple lines to the file.
3170 Args:
3171 lines: Iterable of lines to write
3172 """
3173 for line in lines:
3174 self.write(line)
3176 def read(self, size: int = -1) -> bytes:
3177 """Not supported for write-only file.
3179 Raises:
3180 UnsupportedOperation: Always raised
3181 """
3182 raise UnsupportedOperation("read")
3184 def __enter__(self) -> Self:
3185 """Enter context manager."""
3186 return self
3188 def __exit__(
3189 self,
3190 type: type | None,
3191 value: BaseException | None,
3192 traceback: TracebackType | None,
3193 ) -> None:
3194 """Exit context manager and close file."""
3195 self.f.close()
3197 def __iter__(self) -> "SHA1Writer":
3198 """Return iterator."""
3199 return self
3201 def __next__(self) -> bytes:
3202 """Not supported for write-only file.
3204 Raises:
3205 UnsupportedOperation: Always raised
3206 """
3207 raise UnsupportedOperation("__next__")
3209 def fileno(self) -> int:
3210 """Return file descriptor number."""
3211 return self.f.fileno()
3213 def isatty(self) -> bool:
3214 """Check if file is a terminal."""
3215 return getattr(self.f, "isatty", lambda: False)()
3217 def truncate(self, size: int | None = None) -> int:
3218 """Not supported for write-only file.
3220 Raises:
3221 UnsupportedOperation: Always raised
3222 """
3223 raise UnsupportedOperation("truncate")
3226class HashWriter(BinaryIO):
3227 """Wrapper for file-like object that computes hash of its data.
3229 This is a generic version that works with any hash algorithm.
3230 """
3232 def __init__(
3233 self, f: BinaryIO | IO[bytes], hash_func: Callable[[], "HashObject"]
3234 ) -> None:
3235 """Initialize HashWriter.
3237 Args:
3238 f: File-like object to wrap
3239 hash_func: Hash function (e.g., sha1, sha256)
3240 """
3241 self.f = f
3242 self.length = 0
3243 self.hash_obj = hash_func()
3244 self.digest: bytes | None = None
3246 def write(self, data: bytes | bytearray | memoryview, /) -> int: # type: ignore[override]
3247 """Write data and update hash.
3249 Args:
3250 data: Data to write
3252 Returns:
3253 Number of bytes written
3254 """
3255 self.hash_obj.update(data)
3256 written = self.f.write(data)
3257 self.length += written
3258 return written
3260 def write_hash(self) -> bytes:
3261 """Write the hash digest to the file.
3263 Returns:
3264 The hash digest bytes
3265 """
3266 digest = self.hash_obj.digest()
3267 self.f.write(digest)
3268 self.length += len(digest)
3269 return digest
3271 def close(self) -> None:
3272 """Close the pack file and finalize the hash."""
3273 self.digest = self.write_hash()
3274 self.f.close()
3276 def offset(self) -> int:
3277 """Get the total number of bytes written.
3279 Returns:
3280 Total bytes written
3281 """
3282 return self.length
3284 def tell(self) -> int:
3285 """Return current file position."""
3286 return self.f.tell()
3288 # BinaryIO abstract methods
3289 def readable(self) -> bool:
3290 """Check if file is readable."""
3291 return False
3293 def writable(self) -> bool:
3294 """Check if file is writable."""
3295 return True
3297 def seekable(self) -> bool:
3298 """Check if file is seekable."""
3299 return getattr(self.f, "seekable", lambda: False)()
3301 def seek(self, offset: int, whence: int = 0) -> int:
3302 """Seek to position in file.
3304 Args:
3305 offset: Position offset
3306 whence: Reference point (0=start, 1=current, 2=end)
3308 Returns:
3309 New file position
3310 """
3311 return self.f.seek(offset, whence)
3313 def flush(self) -> None:
3314 """Flush the file buffer."""
3315 if hasattr(self.f, "flush"):
3316 self.f.flush()
3318 def readline(self, size: int = -1) -> bytes:
3319 """Not supported for write-only file.
3321 Raises:
3322 UnsupportedOperation: Always raised
3323 """
3324 raise UnsupportedOperation("readline")
3326 def readlines(self, hint: int = -1) -> list[bytes]:
3327 """Not supported for write-only file.
3329 Raises:
3330 UnsupportedOperation: Always raised
3331 """
3332 raise UnsupportedOperation("readlines")
3334 def writelines(self, lines: Iterable[bytes], /) -> None: # type: ignore[override]
3335 """Write multiple lines to the file.
3337 Args:
3338 lines: Iterable of lines to write
3339 """
3340 for line in lines:
3341 self.write(line)
3343 def read(self, size: int = -1) -> bytes:
3344 """Not supported for write-only file.
3346 Raises:
3347 UnsupportedOperation: Always raised
3348 """
3349 raise UnsupportedOperation("read")
3351 def __enter__(self) -> Self:
3352 """Enter context manager."""
3353 return self
3355 def __exit__(
3356 self,
3357 type: type | None,
3358 value: BaseException | None,
3359 traceback: TracebackType | None,
3360 ) -> None:
3361 """Exit context manager and close file."""
3362 self.close()
3364 def __iter__(self) -> "HashWriter":
3365 """Return iterator."""
3366 return self
3368 def __next__(self) -> bytes:
3369 """Not supported for write-only file.
3371 Raises:
3372 UnsupportedOperation: Always raised
3373 """
3374 raise UnsupportedOperation("__next__")
3376 def fileno(self) -> int:
3377 """Return file descriptor number."""
3378 return self.f.fileno()
3380 def isatty(self) -> bool:
3381 """Check if file is a terminal."""
3382 return getattr(self.f, "isatty", lambda: False)()
3384 def truncate(self, size: int | None = None) -> int:
3385 """Not supported for write-only file.
3387 Raises:
3388 UnsupportedOperation: Always raised
3389 """
3390 raise UnsupportedOperation("truncate")
3393def pack_object_header(
3394 type_num: int,
3395 delta_base: bytes | int | None,
3396 size: int,
3397 object_format: "ObjectFormat",
3398) -> bytearray:
3399 """Create a pack object header for the given object info.
3401 Args:
3402 type_num: Numeric type of the object.
3403 delta_base: Delta base offset or ref, or None for whole objects.
3404 size: Uncompressed object size.
3405 object_format: Object format (hash algorithm) to use.
3406 Returns: A header for a packed object.
3407 """
3408 header = []
3409 c = (type_num << 4) | (size & 15)
3410 size >>= 4
3411 while size:
3412 header.append(c | 0x80)
3413 c = size & 0x7F
3414 size >>= 7
3415 header.append(c)
3416 if type_num == OFS_DELTA:
3417 assert isinstance(delta_base, int)
3418 ret = [delta_base & 0x7F]
3419 delta_base >>= 7
3420 while delta_base:
3421 delta_base -= 1
3422 ret.insert(0, 0x80 | (delta_base & 0x7F))
3423 delta_base >>= 7
3424 header.extend(ret)
3425 elif type_num == REF_DELTA:
3426 assert isinstance(delta_base, bytes)
3427 assert len(delta_base) == object_format.oid_length
3428 header += delta_base
3429 return bytearray(header)
3432def pack_object_chunks(
3433 type: int,
3434 object: list[bytes] | tuple[bytes | int, list[bytes]],
3435 object_format: "ObjectFormat",
3436 *,
3437 compression_level: int = -1,
3438) -> Iterator[bytes]:
3439 """Generate chunks for a pack object.
3441 Args:
3442 type: Numeric type of the object
3443 object: Object to write
3444 object_format: Object format (hash algorithm) to use
3445 compression_level: the zlib compression level
3446 Returns: Chunks
3447 """
3448 if type in DELTA_TYPES:
3449 if isinstance(object, tuple):
3450 delta_base, object = object
3451 else:
3452 raise TypeError("Delta types require a tuple of (delta_base, object)")
3453 else:
3454 delta_base = None
3456 # Convert object to list of bytes chunks
3457 if isinstance(object, bytes):
3458 chunks = [object]
3459 elif isinstance(object, list):
3460 chunks = object
3461 elif isinstance(object, ShaFile):
3462 chunks = object.as_raw_chunks()
3463 else:
3464 # Shouldn't reach here with proper typing
3465 raise TypeError(f"Unexpected object type: {object.__class__.__name__}")
3467 yield bytes(
3468 pack_object_header(
3469 type, delta_base, sum(map(len, chunks)), object_format=object_format
3470 )
3471 )
3472 compressor = zlib.compressobj(level=compression_level)
3473 for data in chunks:
3474 yield compressor.compress(data)
3475 yield compressor.flush()
3478def write_pack_object(
3479 write: Callable[[bytes], int],
3480 type: int,
3481 object: list[bytes] | tuple[bytes | int, list[bytes]],
3482 object_format: "ObjectFormat",
3483 *,
3484 sha: "HashObject | None" = None,
3485 compression_level: int = -1,
3486) -> int:
3487 """Write pack object to a file.
3489 Args:
3490 write: Write function to use
3491 type: Numeric type of the object
3492 object: Object to write
3493 object_format: Object format (hash algorithm) to use
3494 sha: Optional SHA-1 hasher to update
3495 compression_level: the zlib compression level
3496 Returns: CRC32 checksum of the written object
3497 """
3498 crc32 = 0
3499 for chunk in pack_object_chunks(
3500 type, object, compression_level=compression_level, object_format=object_format
3501 ):
3502 write(chunk)
3503 if sha is not None:
3504 sha.update(chunk)
3505 crc32 = binascii.crc32(chunk, crc32)
3506 return crc32 & 0xFFFFFFFF
3509def write_pack(
3510 filename: str,
3511 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3512 object_format: "ObjectFormat",
3513 *,
3514 deltify: bool | None = None,
3515 delta_window_size: int | None = None,
3516 compression_level: int = -1,
3517) -> tuple[bytes, bytes]:
3518 """Write a new pack data file.
3520 Args:
3521 filename: Path to the new pack file (without .pack extension)
3522 objects: Objects to write to the pack
3523 object_format: Object format
3524 delta_window_size: Delta window size
3525 deltify: Whether to deltify pack objects
3526 compression_level: the zlib compression level
3527 Returns: Tuple with checksum of pack file and index file
3528 """
3529 with GitFile(filename + ".pack", "wb") as f:
3530 entries, data_sum = write_pack_objects(
3531 f,
3532 objects,
3533 delta_window_size=delta_window_size,
3534 deltify=deltify,
3535 compression_level=compression_level,
3536 object_format=object_format,
3537 )
3538 entries_list = sorted([(k, v[0], v[1]) for (k, v) in entries.items()])
3539 with GitFile(filename + ".idx", "wb") as f:
3540 idx_sha = write_pack_index(f, entries_list, data_sum)
3541 return data_sum, idx_sha
3544def pack_header_chunks(num_objects: int) -> Iterator[bytes]:
3545 """Yield chunks for a pack header."""
3546 yield b"PACK" # Pack header
3547 yield struct.pack(b">L", 2) # Pack version
3548 yield struct.pack(b">L", num_objects) # Number of objects in pack
3551def write_pack_header(
3552 write: Callable[[bytes], int] | IO[bytes], num_objects: int
3553) -> None:
3554 """Write a pack header for the given number of objects."""
3555 if not callable(write):
3556 write_fn: Callable[[bytes], int] = write.write
3557 warnings.warn(
3558 "write_pack_header() now takes a write rather than file argument",
3559 DeprecationWarning,
3560 stacklevel=2,
3561 )
3562 else:
3563 write_fn = write
3564 for chunk in pack_header_chunks(num_objects):
3565 write_fn(chunk)
3568def find_reusable_deltas(
3569 container: PackedObjectContainer,
3570 object_ids: Set[ObjectID],
3571 *,
3572 other_haves: Set[ObjectID] | None = None,
3573 progress: Callable[..., None] | None = None,
3574) -> Iterator[UnpackedObject]:
3575 """Find deltas in a pack that can be reused.
3577 Args:
3578 container: Pack container to search for deltas
3579 object_ids: Set of object IDs to find deltas for
3580 other_haves: Set of other object IDs we have
3581 progress: Optional progress reporting callback
3583 Returns:
3584 Iterator of UnpackedObject entries that can be reused
3585 """
3586 if other_haves is None:
3587 other_haves = set()
3588 reused = 0
3589 for i, unpacked in enumerate(
3590 container.iter_unpacked_subset(
3591 object_ids, allow_missing=True, convert_ofs_delta=True
3592 )
3593 ):
3594 if progress is not None and i % 1000 == 0:
3595 progress(f"checking for reusable deltas: {i}/{len(object_ids)}\r".encode())
3596 if unpacked.pack_type_num == REF_DELTA:
3597 hexsha = sha_to_hex(unpacked.delta_base) # type: ignore
3598 if hexsha in object_ids or hexsha in other_haves:
3599 yield unpacked
3600 reused += 1
3601 if progress is not None:
3602 progress((f"found {reused} deltas to reuse\n").encode())
3605def deltify_pack_objects(
3606 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, bytes | None]],
3607 *,
3608 window_size: int | None = None,
3609 progress: Callable[..., None] | None = None,
3610) -> Iterator[UnpackedObject]:
3611 """Generate deltas for pack objects.
3613 Args:
3614 objects: An iterable of (object, path) tuples to deltify.
3615 window_size: Window size; None for default
3616 progress: Optional progress reporting callback
3617 Returns: Iterator over type_num, object id, delta_base, content
3618 delta_base is None for full text entries
3619 """
3621 def objects_with_hints() -> Iterator[tuple[ShaFile, tuple[int, bytes | None]]]:
3622 for e in objects:
3623 if isinstance(e, ShaFile):
3624 yield (e, (e.type_num, None))
3625 else:
3626 yield (e[0], (e[0].type_num, e[1]))
3628 sorted_objs = sort_objects_for_delta(objects_with_hints())
3629 yield from deltas_from_sorted_objects(
3630 sorted_objs,
3631 window_size=window_size,
3632 progress=progress,
3633 )
3636def sort_objects_for_delta(
3637 objects: Iterator[ShaFile] | Iterator[tuple[ShaFile, PackHint | None]],
3638) -> Iterator[tuple[ShaFile, bytes | None]]:
3639 """Sort objects for optimal delta compression.
3641 Args:
3642 objects: Iterator of objects or (object, hint) tuples
3644 Returns:
3645 Iterator of sorted (ShaFile, path) tuples
3646 """
3647 magic = []
3648 for entry in objects:
3649 if isinstance(entry, tuple):
3650 obj, hint = entry
3651 if hint is None:
3652 type_num = None
3653 path = None
3654 else:
3655 (type_num, path) = hint
3656 else:
3657 obj = entry
3658 type_num = None
3659 path = None
3660 magic.append((type_num, path, -obj.raw_length(), obj))
3661 # Build a list of objects ordered by the magic Linus heuristic
3662 # This helps us find good objects to diff against us
3663 magic.sort()
3664 return ((x[3], x[1]) for x in magic)
3667def deltas_from_sorted_objects(
3668 objects: Iterator[tuple[ShaFile, bytes | None]],
3669 window_size: int | None = None,
3670 progress: Callable[..., None] | None = None,
3671) -> Iterator[UnpackedObject]:
3672 """Create deltas from sorted objects.
3674 Args:
3675 objects: Iterator of sorted objects to deltify
3676 window_size: Delta window size; None for default
3677 progress: Optional progress reporting callback
3679 Returns:
3680 Iterator of UnpackedObject entries
3681 """
3682 # TODO(jelmer): Use threads
3683 if window_size is None:
3684 window_size = DEFAULT_PACK_DELTA_WINDOW_SIZE
3686 possible_bases: deque[tuple[bytes, int, bytes]] = deque()
3687 for i, (o, path) in enumerate(objects):
3688 if progress is not None and i % 1000 == 0:
3689 progress((f"generating deltas: {i}\r").encode())
3690 raw = o.as_raw_chunks()
3691 raw_bytes = b"".join(raw) # Join once for efficiency
3692 winner = raw
3693 winner_len = sum(map(len, winner))
3694 winner_base = None
3695 for base_id, base_type_num, base_bytes in possible_bases:
3696 if base_type_num != o.type_num:
3697 continue
3698 delta_len = 0
3699 delta = []
3700 for chunk in create_delta(base_bytes, raw_bytes):
3701 delta_len += len(chunk)
3702 if delta_len >= winner_len:
3703 break
3704 delta.append(chunk)
3705 else:
3706 winner_base = base_id
3707 winner = delta
3708 winner_len = sum(map(len, winner))
3709 yield UnpackedObject(
3710 o.type_num,
3711 sha=o.sha().digest(),
3712 delta_base=winner_base,
3713 decomp_len=winner_len,
3714 decomp_chunks=winner,
3715 )
3716 possible_bases.appendleft((o.sha().digest(), o.type_num, raw_bytes))
3717 while len(possible_bases) > window_size:
3718 possible_bases.pop()
3721def pack_objects_to_data(
3722 objects: Sequence[ShaFile]
3723 | Sequence[tuple[ShaFile, bytes | None]]
3724 | Sequence[tuple[ShaFile, PackHint | None]],
3725 *,
3726 deltify: bool | None = None,
3727 delta_window_size: int | None = None,
3728 ofs_delta: bool = True,
3729 progress: Callable[..., None] | None = None,
3730) -> tuple[int, Iterator[UnpackedObject]]:
3731 """Create pack data from objects.
3733 Args:
3734 objects: Pack objects
3735 deltify: Whether to deltify pack objects
3736 delta_window_size: Delta window size
3737 ofs_delta: Whether to use offset deltas
3738 progress: Optional progress reporting callback
3739 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3740 """
3741 count = len(objects)
3742 if deltify is None:
3743 # PERFORMANCE/TODO(jelmer): This should be enabled but the python
3744 # implementation is *much* too slow at the moment.
3745 # Maybe consider enabling it just if the rust extension is available?
3746 deltify = False
3747 if deltify:
3748 return (
3749 count,
3750 deltify_pack_objects(
3751 iter(objects), # type: ignore
3752 window_size=delta_window_size,
3753 progress=progress,
3754 ),
3755 )
3756 else:
3758 def iter_without_path() -> Iterator[UnpackedObject]:
3759 for o in objects:
3760 if isinstance(o, tuple):
3761 yield full_unpacked_object(o[0])
3762 else:
3763 yield full_unpacked_object(o)
3765 return (count, iter_without_path())
3768def generate_unpacked_objects(
3769 container: PackedObjectContainer,
3770 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3771 delta_window_size: int | None = None,
3772 deltify: bool | None = None,
3773 reuse_deltas: bool = True,
3774 ofs_delta: bool = True,
3775 other_haves: set[ObjectID] | None = None,
3776 progress: Callable[..., None] | None = None,
3777) -> Iterator[UnpackedObject]:
3778 """Create pack data from objects.
3780 Returns: Tuples with (type_num, hexdigest, delta base, object chunks)
3781 """
3782 todo = dict(object_ids)
3783 if reuse_deltas:
3784 for unpack in find_reusable_deltas(
3785 container, set(todo), other_haves=other_haves, progress=progress
3786 ):
3787 del todo[sha_to_hex(RawObjectID(unpack.sha()))]
3788 yield unpack
3789 if deltify is None:
3790 # PERFORMANCE/TODO(jelmer): This should be enabled but is *much* too
3791 # slow at the moment.
3792 deltify = False
3793 if deltify:
3794 objects_to_delta = container.iterobjects_subset(
3795 todo.keys(), allow_missing=False
3796 )
3797 sorted_objs = sort_objects_for_delta((o, todo[o.id]) for o in objects_to_delta)
3798 yield from deltas_from_sorted_objects(
3799 sorted_objs,
3800 window_size=delta_window_size,
3801 progress=progress,
3802 )
3803 else:
3804 for oid in todo:
3805 yield full_unpacked_object(container[oid])
3808def full_unpacked_object(o: ShaFile) -> UnpackedObject:
3809 """Create an UnpackedObject from a ShaFile.
3811 Args:
3812 o: ShaFile object to convert
3814 Returns:
3815 UnpackedObject with full object data
3816 """
3817 return UnpackedObject(
3818 o.type_num,
3819 delta_base=None,
3820 crc32=None,
3821 decomp_chunks=o.as_raw_chunks(),
3822 sha=o.sha().digest(),
3823 )
3826def write_pack_from_container(
3827 write: Callable[[bytes], None]
3828 | Callable[[bytes | bytearray | memoryview], int]
3829 | IO[bytes],
3830 container: PackedObjectContainer,
3831 object_ids: Sequence[tuple[ObjectID, PackHint | None]],
3832 object_format: "ObjectFormat",
3833 *,
3834 delta_window_size: int | None = None,
3835 deltify: bool | None = None,
3836 reuse_deltas: bool = True,
3837 compression_level: int = -1,
3838 other_haves: set[ObjectID] | None = None,
3839) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3840 """Write a new pack data file.
3842 Args:
3843 write: write function to use
3844 container: PackedObjectContainer
3845 object_ids: Sequence of (object_id, hint) tuples to write
3846 object_format: Object format (hash algorithm) to use
3847 delta_window_size: Sliding window size for searching for deltas;
3848 Set to None for default window size.
3849 deltify: Whether to deltify objects
3850 reuse_deltas: Whether to reuse existing deltas
3851 compression_level: the zlib compression level to use
3852 other_haves: Set of additional object IDs the receiver has
3853 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3854 """
3855 pack_contents_count = len(object_ids)
3856 pack_contents = generate_unpacked_objects(
3857 container,
3858 object_ids,
3859 delta_window_size=delta_window_size,
3860 deltify=deltify,
3861 reuse_deltas=reuse_deltas,
3862 other_haves=other_haves,
3863 )
3865 return write_pack_data(
3866 write,
3867 pack_contents,
3868 num_records=pack_contents_count,
3869 compression_level=compression_level,
3870 object_format=object_format,
3871 )
3874def write_pack_objects(
3875 write: Callable[[bytes], None] | IO[bytes],
3876 objects: Sequence[ShaFile] | Sequence[tuple[ShaFile, bytes | None]],
3877 object_format: "ObjectFormat",
3878 *,
3879 delta_window_size: int | None = None,
3880 deltify: bool | None = None,
3881 compression_level: int = -1,
3882) -> tuple[dict[bytes, tuple[int, int]], bytes]:
3883 """Write a new pack data file.
3885 Args:
3886 write: write function to use
3887 objects: Sequence of (object, path) tuples to write
3888 object_format: Object format (hash algorithm) to use
3889 delta_window_size: Sliding window size for searching for deltas;
3890 Set to None for default window size.
3891 deltify: Whether to deltify objects
3892 compression_level: the zlib compression level to use
3893 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3894 """
3895 pack_contents_count, pack_contents = pack_objects_to_data(objects, deltify=deltify)
3897 return write_pack_data(
3898 write,
3899 pack_contents,
3900 num_records=pack_contents_count,
3901 compression_level=compression_level,
3902 object_format=object_format,
3903 )
3906class PackChunkGenerator:
3907 """Generator for pack data chunks."""
3909 def __init__(
3910 self,
3911 object_format: "ObjectFormat",
3912 num_records: int | None = None,
3913 records: Iterator[UnpackedObject] | None = None,
3914 progress: Callable[..., None] | None = None,
3915 compression_level: int = -1,
3916 reuse_compressed: bool = True,
3917 ) -> None:
3918 """Initialize PackChunkGenerator.
3920 Args:
3921 num_records: Expected number of records
3922 records: Iterator of pack records
3923 progress: Optional progress callback
3924 compression_level: Compression level (-1 for default)
3925 reuse_compressed: Whether to reuse compressed chunks
3926 object_format: Object format (hash algorithm) to use
3927 """
3928 self.object_format = object_format
3929 self.cs = object_format.new_hash()
3930 self.entries: dict[bytes, tuple[int, int]] = {}
3931 if records is None:
3932 records = iter([]) # Empty iterator if None
3933 self._it = self._pack_data_chunks(
3934 records=records,
3935 num_records=num_records,
3936 progress=progress,
3937 compression_level=compression_level,
3938 reuse_compressed=reuse_compressed,
3939 )
3941 def sha1digest(self) -> bytes:
3942 """Return the SHA1 digest of the pack data."""
3943 return self.cs.digest()
3945 def __iter__(self) -> Iterator[bytes]:
3946 """Iterate over pack data chunks."""
3947 return self._it
3949 def _pack_data_chunks(
3950 self,
3951 records: Iterator[UnpackedObject],
3952 *,
3953 num_records: int | None = None,
3954 progress: Callable[..., None] | None = None,
3955 compression_level: int = -1,
3956 reuse_compressed: bool = True,
3957 ) -> Iterator[bytes]:
3958 """Iterate pack data file chunks.
3960 Args:
3961 records: Iterator over UnpackedObject
3962 num_records: Number of records (defaults to len(records) if not specified)
3963 progress: Function to report progress to
3964 compression_level: the zlib compression level
3965 reuse_compressed: Whether to reuse compressed chunks
3966 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
3967 """
3968 # Write the pack
3969 if num_records is None:
3970 num_records = len(records) # type: ignore
3971 offset = 0
3972 for chunk in pack_header_chunks(num_records):
3973 yield chunk
3974 self.cs.update(chunk)
3975 offset += len(chunk)
3976 actual_num_records = 0
3977 for i, unpacked in enumerate(records):
3978 type_num = unpacked.pack_type_num
3979 if progress is not None and i % 1000 == 0:
3980 progress((f"writing pack data: {i}/{num_records}\r").encode("ascii"))
3981 raw: list[bytes] | tuple[int, list[bytes]] | tuple[bytes, list[bytes]]
3982 if unpacked.delta_base is not None:
3983 assert isinstance(unpacked.delta_base, bytes), (
3984 f"Expected bytes, got {type(unpacked.delta_base)}"
3985 )
3986 try:
3987 base_offset, _base_crc32 = self.entries[unpacked.delta_base]
3988 except KeyError:
3989 type_num = REF_DELTA
3990 assert isinstance(unpacked.delta_base, bytes)
3991 raw = (unpacked.delta_base, unpacked.decomp_chunks)
3992 else:
3993 type_num = OFS_DELTA
3994 raw = (offset - base_offset, unpacked.decomp_chunks)
3995 else:
3996 raw = unpacked.decomp_chunks
3997 chunks: list[bytes] | Iterator[bytes]
3998 if unpacked.comp_chunks is not None and reuse_compressed:
3999 chunks = unpacked.comp_chunks
4000 else:
4001 chunks = pack_object_chunks(
4002 type_num,
4003 raw,
4004 compression_level=compression_level,
4005 object_format=self.object_format,
4006 )
4007 crc32 = 0
4008 object_size = 0
4009 for chunk in chunks:
4010 yield chunk
4011 crc32 = binascii.crc32(chunk, crc32)
4012 self.cs.update(chunk)
4013 object_size += len(chunk)
4014 actual_num_records += 1
4015 self.entries[unpacked.sha()] = (offset, crc32)
4016 offset += object_size
4017 if actual_num_records != num_records:
4018 raise AssertionError(
4019 f"actual records written differs: {actual_num_records} != {num_records}"
4020 )
4022 yield self.cs.digest()
4025def write_pack_data(
4026 write: Callable[[bytes], None]
4027 | Callable[[bytes | bytearray | memoryview], int]
4028 | IO[bytes],
4029 records: Iterator[UnpackedObject],
4030 object_format: "ObjectFormat",
4031 *,
4032 num_records: int | None = None,
4033 progress: Callable[..., None] | None = None,
4034 compression_level: int = -1,
4035) -> tuple[dict[bytes, tuple[int, int]], bytes]:
4036 """Write a new pack data file.
4038 Args:
4039 write: Write function to use
4040 num_records: Number of records (defaults to len(records) if None)
4041 records: Iterator over type_num, object_id, delta_base, raw
4042 object_format: Object format (hash algorithm) to use
4043 progress: Function to report progress to
4044 compression_level: the zlib compression level
4045 Returns: Dict mapping id -> (offset, crc32 checksum), pack checksum
4046 """
4047 chunk_generator = PackChunkGenerator(
4048 num_records=num_records,
4049 records=records,
4050 progress=progress,
4051 compression_level=compression_level,
4052 object_format=object_format,
4053 )
4054 for chunk in chunk_generator:
4055 if callable(write):
4056 write(chunk)
4057 else:
4058 write.write(chunk)
4059 return chunk_generator.entries, chunk_generator.sha1digest()
4062def write_pack_index_v1(
4063 f: IO[bytes],
4064 entries: Iterable[tuple[bytes, int, int | None]],
4065 pack_checksum: bytes,
4066) -> bytes:
4067 """Write a new pack index file.
4069 Args:
4070 f: A file-like object to write to
4071 entries: List of tuples with object name (sha), offset_in_pack,
4072 and crc32_checksum.
4073 pack_checksum: Checksum of the pack file.
4074 Returns: The SHA of the written index file
4075 """
4076 f = SHA1Writer(f)
4077 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4078 for name, _offset, _entry_checksum in entries:
4079 fan_out_table[ord(name[:1])] += 1
4080 # Fan-out table
4081 for i in range(0x100):
4082 f.write(struct.pack(">L", fan_out_table[i]))
4083 fan_out_table[i + 1] += fan_out_table[i]
4084 for name, offset, _entry_checksum in entries:
4085 if len(name) != 20:
4086 raise TypeError("pack index v1 only supports SHA-1 names")
4087 if not (offset <= 0xFFFFFFFF):
4088 raise TypeError("pack format 1 only supports offsets < 2Gb")
4089 f.write(struct.pack(">L20s", offset, name))
4090 assert len(pack_checksum) == 20
4091 f.write(pack_checksum)
4092 return f.write_sha()
4095def _delta_encode_size(size: int) -> bytes:
4096 ret = bytearray()
4097 c = size & 0x7F
4098 size >>= 7
4099 while size:
4100 ret.append(c | 0x80)
4101 c = size & 0x7F
4102 size >>= 7
4103 ret.append(c)
4104 return bytes(ret)
4107# The length of delta compression copy operations in version 2 packs is limited
4108# to 64K. To copy more, we use several copy operations. Version 3 packs allow
4109# 24-bit lengths in copy operations, but we always make version 2 packs.
4110_MAX_COPY_LEN = 0xFFFF
4113def _encode_copy_operation(start: int, length: int) -> bytes:
4114 scratch = bytearray([0x80])
4115 for i in range(4):
4116 if start & 0xFF << i * 8:
4117 scratch.append((start >> i * 8) & 0xFF)
4118 scratch[0] |= 1 << i
4119 for i in range(2):
4120 if length & 0xFF << i * 8:
4121 scratch.append((length >> i * 8) & 0xFF)
4122 scratch[0] |= 1 << (4 + i)
4123 return bytes(scratch)
4126def _create_delta_py(
4127 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
4128) -> Iterator[bytes]:
4129 """Use python difflib to work out how to transform base_buf to target_buf.
4131 Args:
4132 base_buf: Base buffer
4133 target_buf: Target buffer
4134 """
4135 if isinstance(base_buf, list):
4136 base_buf = b"".join(base_buf)
4137 if isinstance(target_buf, list):
4138 target_buf = b"".join(target_buf)
4139 # write delta header
4140 yield _delta_encode_size(len(base_buf))
4141 yield _delta_encode_size(len(target_buf))
4142 # write out delta opcodes
4143 seq = SequenceMatcher(isjunk=None, a=base_buf, b=target_buf)
4144 for opcode, i1, i2, j1, j2 in seq.get_opcodes():
4145 # Git patch opcodes don't care about deletes!
4146 # if opcode == 'replace' or opcode == 'delete':
4147 # pass
4148 if opcode == "equal":
4149 # If they are equal, unpacker will use data from base_buf
4150 # Write out an opcode that says what range to use
4151 copy_start = i1
4152 copy_len = i2 - i1
4153 while copy_len > 0:
4154 to_copy = min(copy_len, _MAX_COPY_LEN)
4155 yield _encode_copy_operation(copy_start, to_copy)
4156 copy_start += to_copy
4157 copy_len -= to_copy
4158 if opcode == "replace" or opcode == "insert":
4159 # If we are replacing a range or adding one, then we just
4160 # output it to the stream (prefixed by its size)
4161 s = j2 - j1
4162 o = j1
4163 while s > 127:
4164 yield bytes([127])
4165 yield bytes(memoryview(target_buf)[o : o + 127])
4166 s -= 127
4167 o += 127
4168 yield bytes([s])
4169 yield bytes(memoryview(target_buf)[o : o + s])
4172# Default to pure Python implementation
4173create_delta = _create_delta_py
4176def apply_delta(
4177 src_buf: bytes | list[bytes], delta: bytes | list[bytes]
4178) -> list[bytes]:
4179 """Based on the similar function in git's patch-delta.c.
4181 Args:
4182 src_buf: Source buffer
4183 delta: Delta instructions
4184 """
4185 if not isinstance(src_buf, bytes):
4186 src_buf = b"".join(src_buf)
4187 if not isinstance(delta, bytes):
4188 delta = b"".join(delta)
4189 out = []
4190 index = 0
4191 delta_length = len(delta)
4193 def get_delta_header_size(delta: bytes, index: int) -> tuple[int, int]:
4194 size = 0
4195 i = 0
4196 while True:
4197 # Bound-check explicitly: ``delta[index:index+1]`` silently
4198 # returns b"" past the end, which would crash with TypeError
4199 # in ``ord`` and leave the caller unable to distinguish a
4200 # truncated delta from a programming bug.
4201 if index >= delta_length:
4202 raise ApplyDeltaError("delta truncated in size header")
4203 cmd = ord(delta[index : index + 1])
4204 index += 1
4205 size |= (cmd & ~0x80) << i
4206 i += 7
4207 if not cmd & 0x80:
4208 break
4209 return size, index
4211 def read_byte(delta: bytes) -> int:
4212 nonlocal index
4213 # Bound-check explicitly: ``delta[index:index+1]`` silently returns
4214 # b"" past the end, which would crash with TypeError in ``ord`` and
4215 # leave the caller unable to distinguish a truncated delta from a
4216 # programming bug.
4217 if index >= delta_length:
4218 raise ApplyDeltaError("delta truncated in copy op")
4219 index += 1
4220 return ord(delta[index - 1 : index])
4222 src_size, index = get_delta_header_size(delta, index)
4223 dest_size, index = get_delta_header_size(delta, index)
4224 if src_size != len(src_buf):
4225 raise ApplyDeltaError(
4226 f"Unexpected source buffer size: {src_size} vs {len(src_buf)}"
4227 )
4228 while index < delta_length:
4229 cmd = ord(delta[index : index + 1])
4230 index += 1
4231 if cmd & 0x80:
4232 cp_off = 0
4233 for i in range(4):
4234 if cmd & (1 << i):
4235 x = read_byte(delta)
4236 cp_off |= x << (i * 8)
4237 cp_size = 0
4238 # Version 3 packs can contain copy sizes larger than 64K.
4239 for i in range(3):
4240 if cmd & (1 << (4 + i)):
4241 x = read_byte(delta)
4242 cp_size |= x << (i * 8)
4243 if cp_size == 0:
4244 cp_size = 0x10000
4245 if (
4246 cp_off + cp_size < cp_size
4247 or cp_off + cp_size > src_size
4248 or cp_size > dest_size
4249 ):
4250 break
4251 out.append(src_buf[cp_off : cp_off + cp_size])
4252 elif cmd != 0:
4253 if index + cmd > delta_length:
4254 raise ApplyDeltaError("delta truncated in insert op")
4255 out.append(delta[index : index + cmd])
4256 index += cmd
4257 else:
4258 raise ApplyDeltaError("Invalid opcode 0")
4260 if index != delta_length:
4261 raise ApplyDeltaError(f"delta not empty: {delta[index:]!r}")
4263 if dest_size != chunks_length(out):
4264 raise ApplyDeltaError("dest size incorrect")
4266 return out
4269def write_pack_index_v2(
4270 f: IO[bytes],
4271 entries: Iterable[tuple[bytes, int, int | None]],
4272 pack_checksum: bytes,
4273) -> bytes:
4274 """Write a new pack index file.
4276 Args:
4277 f: File-like object to write to
4278 entries: List of tuples with object name (sha), offset_in_pack, and
4279 crc32_checksum.
4280 pack_checksum: Checksum of the pack file.
4281 Returns: The checksum of the index file written
4282 """
4283 # Determine hash algorithm from pack_checksum length
4284 if len(pack_checksum) == 20:
4285 hash_func = sha1
4286 elif len(pack_checksum) == 32:
4287 hash_func = sha256
4288 else:
4289 raise ValueError(f"Unsupported pack checksum length: {len(pack_checksum)}")
4291 f_writer = HashWriter(f, hash_func)
4292 f_writer.write(b"\377tOc") # Magic!
4293 f_writer.write(struct.pack(">L", 2))
4295 # Convert to list to allow multiple iterations
4296 entries_list = list(entries)
4298 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4299 for name, offset, entry_checksum in entries_list:
4300 fan_out_table[ord(name[:1])] += 1
4302 if entries_list:
4303 hash_size = len(entries_list[0][0])
4304 else:
4305 hash_size = len(pack_checksum) # Use pack_checksum length as hash size
4307 # Fan-out table
4308 largetable: list[int] = []
4309 for i in range(0x100):
4310 f_writer.write(struct.pack(b">L", fan_out_table[i]))
4311 fan_out_table[i + 1] += fan_out_table[i]
4312 for name, offset, entry_checksum in entries_list:
4313 if len(name) != hash_size:
4314 raise TypeError(
4315 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4316 )
4317 f_writer.write(name)
4318 for name, offset, entry_checksum in entries_list:
4319 f_writer.write(struct.pack(b">L", entry_checksum))
4320 for name, offset, entry_checksum in entries_list:
4321 if offset < 2**31:
4322 f_writer.write(struct.pack(b">L", offset))
4323 else:
4324 f_writer.write(struct.pack(b">L", 2**31 + len(largetable)))
4325 largetable.append(offset)
4326 for offset in largetable:
4327 f_writer.write(struct.pack(b">Q", offset))
4328 f_writer.write(pack_checksum)
4329 return f_writer.write_hash()
4332def write_pack_index_v3(
4333 f: IO[bytes],
4334 entries: Iterable[tuple[bytes, int, int | None]],
4335 pack_checksum: bytes,
4336 hash_format: int = 1,
4337) -> bytes:
4338 """Write a new pack index file in v3 format.
4340 Args:
4341 f: File-like object to write to
4342 entries: List of tuples with object name (sha), offset_in_pack, and
4343 crc32_checksum.
4344 pack_checksum: Checksum of the pack file.
4345 hash_format: Hash algorithm identifier (1 = SHA-1, 2 = SHA-256)
4346 Returns: The SHA of the index file written
4347 """
4348 if hash_format == 1:
4349 hash_size = 20 # SHA-1
4350 writer_cls = SHA1Writer
4351 elif hash_format == 2:
4352 hash_size = 32 # SHA-256
4353 # TODO: Add SHA256Writer when SHA-256 support is implemented
4354 raise NotImplementedError("SHA-256 support not yet implemented")
4355 else:
4356 raise ValueError(f"Unknown hash algorithm {hash_format}")
4358 # Convert entries to list to allow multiple iterations
4359 entries_list = list(entries)
4361 # Calculate shortest unambiguous prefix length for object names
4362 # For now, use full hash size (this could be optimized)
4363 shortened_oid_len = hash_size
4365 f = writer_cls(f)
4366 f.write(b"\377tOc") # Magic!
4367 f.write(struct.pack(">L", 3)) # Version 3
4368 f.write(struct.pack(">L", hash_format)) # Hash algorithm
4369 f.write(struct.pack(">L", shortened_oid_len)) # Shortened OID length
4371 fan_out_table: dict[int, int] = defaultdict(lambda: 0)
4372 for name, offset, entry_checksum in entries_list:
4373 if len(name) != hash_size:
4374 raise ValueError(
4375 f"Object name has wrong length: expected {hash_size}, got {len(name)}"
4376 )
4377 fan_out_table[ord(name[:1])] += 1
4379 # Fan-out table
4380 largetable: list[int] = []
4381 for i in range(0x100):
4382 f.write(struct.pack(b">L", fan_out_table[i]))
4383 fan_out_table[i + 1] += fan_out_table[i]
4385 # Object names table
4386 for name, offset, entry_checksum in entries_list:
4387 f.write(name)
4389 # CRC32 checksums table
4390 for name, offset, entry_checksum in entries_list:
4391 f.write(struct.pack(b">L", entry_checksum))
4393 # Offset table
4394 for name, offset, entry_checksum in entries_list:
4395 if offset < 2**31:
4396 f.write(struct.pack(b">L", offset))
4397 else:
4398 f.write(struct.pack(b">L", 2**31 + len(largetable)))
4399 largetable.append(offset)
4401 # Large offset table
4402 for offset in largetable:
4403 f.write(struct.pack(b">Q", offset))
4405 assert len(pack_checksum) == hash_size, (
4406 f"Pack checksum has wrong length: expected {hash_size}, got {len(pack_checksum)}"
4407 )
4408 f.write(pack_checksum)
4409 return f.write_sha()
4412def write_pack_index(
4413 f: IO[bytes],
4414 entries: Iterable[tuple[bytes, int, int | None]],
4415 pack_checksum: bytes,
4416 progress: Callable[..., None] | None = None,
4417 version: int | None = None,
4418) -> bytes:
4419 """Write a pack index file.
4421 Args:
4422 f: File-like object to write to.
4423 entries: List of (checksum, offset, crc32) tuples
4424 pack_checksum: Checksum of the pack file.
4425 progress: Progress function (not currently used)
4426 version: Pack index version to use (1, 2, or 3). If None, defaults to DEFAULT_PACK_INDEX_VERSION.
4428 Returns:
4429 SHA of the written index file
4431 Raises:
4432 ValueError: If an unsupported version is specified
4433 """
4434 if version is None:
4435 version = DEFAULT_PACK_INDEX_VERSION
4437 if version == 1:
4438 return write_pack_index_v1(f, entries, pack_checksum)
4439 elif version == 2:
4440 return write_pack_index_v2(f, entries, pack_checksum)
4441 elif version == 3:
4442 return write_pack_index_v3(f, entries, pack_checksum)
4443 else:
4444 raise ValueError(f"Unsupported pack index version: {version}")
4447class Pack:
4448 """A Git pack object."""
4450 _data_load: Callable[[], PackData] | None
4451 _idx_load: Callable[[], PackIndex] | None
4453 _data: PackData | None
4454 _idx: PackIndex | None
4455 _bitmap: "PackBitmap | None"
4457 def __init__(
4458 self,
4459 basename: str,
4460 *,
4461 object_format: ObjectFormat,
4462 resolve_ext_ref: ResolveExtRefFn | None = None,
4463 delta_window_size: int | None = None,
4464 window_memory: int | None = None,
4465 delta_cache_size: int | None = None,
4466 depth: int | None = None,
4467 threads: int | None = None,
4468 big_file_threshold: int | None = None,
4469 delta_base_cache_limit: int | None = None,
4470 ) -> None:
4471 """Initialize a Pack object.
4473 Args:
4474 basename: Base path for pack files (without .pack/.idx extension)
4475 object_format: Hash algorithm used by the repository
4476 resolve_ext_ref: Optional function to resolve external references
4477 delta_window_size: Size of the delta compression window
4478 window_memory: Memory limit for delta compression window
4479 delta_cache_size: Size of the delta cache
4480 depth: Maximum depth for delta chains
4481 threads: Number of threads to use for operations
4482 big_file_threshold: Size threshold for big file handling
4483 delta_base_cache_limit: Maximum bytes for delta base object cache
4484 """
4485 self._basename = basename
4486 self.object_format = object_format
4487 self._data = None
4488 self._idx = None
4489 self._bitmap = None
4490 self._idx_path = self._basename + ".idx"
4491 self._data_path = self._basename + ".pack"
4492 self._bitmap_path = self._basename + ".bitmap"
4493 self.delta_window_size = delta_window_size
4494 self.window_memory = window_memory
4495 self.delta_cache_size = delta_cache_size
4496 self.depth = depth
4497 self.threads = threads
4498 self.big_file_threshold = big_file_threshold
4499 self.delta_base_cache_limit = delta_base_cache_limit
4500 self._idx_load = lambda: load_pack_index(self._idx_path, object_format)
4501 self._data_load = lambda: PackData(
4502 self._data_path,
4503 delta_window_size=delta_window_size,
4504 window_memory=window_memory,
4505 delta_cache_size=delta_cache_size,
4506 depth=depth,
4507 threads=threads,
4508 big_file_threshold=big_file_threshold,
4509 delta_base_cache_limit=delta_base_cache_limit,
4510 object_format=object_format,
4511 )
4512 self.resolve_ext_ref = resolve_ext_ref
4514 @classmethod
4515 def from_lazy_objects(
4516 cls,
4517 data_fn: Callable[[], PackData],
4518 idx_fn: Callable[[], PackIndex],
4519 ) -> "Pack":
4520 """Create a new pack object from callables to load pack data and index objects."""
4521 # Load index to get object format
4522 idx = idx_fn()
4523 ret = cls("", object_format=idx.object_format)
4524 ret._data_load = data_fn
4525 ret._idx = idx
4526 ret._idx_load = None
4527 return ret
4529 @classmethod
4530 def from_objects(cls, data: PackData, idx: PackIndex) -> "Pack":
4531 """Create a new pack object from pack data and index objects."""
4532 ret = cls("", object_format=idx.object_format)
4533 ret._data = data
4534 ret._data_load = None
4535 ret._idx = idx
4536 ret._idx_load = None
4537 ret.check_length_and_checksum()
4538 return ret
4540 def name(self) -> bytes:
4541 """The SHA over the SHAs of the objects in this pack."""
4542 return self.index.objects_sha1()
4544 @property
4545 def data(self) -> PackData:
4546 """The pack data object being used."""
4547 if self._data is None:
4548 assert self._data_load
4549 try:
4550 self._data = self._data_load()
4551 except FileNotFoundError as exc:
4552 raise PackFileDisappeared(self) from exc
4553 self.check_length_and_checksum()
4554 return self._data
4556 @property
4557 def index(self) -> PackIndex:
4558 """The index being used.
4560 Note: This may be an in-memory index
4561 """
4562 if self._idx is None:
4563 assert self._idx_load
4564 try:
4565 self._idx = self._idx_load()
4566 except FileNotFoundError as exc:
4567 raise PackFileDisappeared(self) from exc
4568 return self._idx
4570 @property
4571 def bitmap(self) -> "PackBitmap | None":
4572 """The bitmap being used, if available.
4574 Returns:
4575 PackBitmap instance, or None if no bitmap exists or the bitmap
4576 was built for a different pack
4578 Raises:
4579 ValueError: If bitmap file is invalid or corrupt
4580 """
4581 if self._bitmap is None:
4582 from .bitmap import read_bitmap
4584 try:
4585 self._bitmap = read_bitmap(
4586 self._bitmap_path,
4587 pack_index=self.index,
4588 pack_checksum=self.get_stored_checksum(),
4589 )
4590 except ChecksumMismatch:
4591 # The bitmap records the checksum of the pack it was built for.
4592 # A mismatch means it is stale or was swapped in from another
4593 # pack, so its positions no longer describe this pack's objects.
4594 # Ignore it and let callers fall back to graph traversal, the
4595 # same as git.
4596 logger.warning(
4597 "Ignoring bitmap %s: checksum does not match pack",
4598 self._bitmap_path,
4599 )
4600 return None
4601 return self._bitmap
4603 def ensure_bitmap(
4604 self,
4605 object_store: "BaseObjectStore",
4606 refs: dict["Ref", "ObjectID"],
4607 commit_interval: int | None = None,
4608 progress: Callable[[str], None] | None = None,
4609 ) -> "PackBitmap":
4610 """Ensure a bitmap exists for this pack, generating one if needed.
4612 Args:
4613 object_store: Object store to read objects from
4614 refs: Dictionary of ref names to commit SHAs
4615 commit_interval: Include every Nth commit in bitmap index
4616 progress: Optional progress reporting callback
4618 Returns:
4619 PackBitmap instance (either existing or newly generated)
4620 """
4621 from .bitmap import generate_bitmap, write_bitmap
4623 # Check if bitmap already exists
4624 try:
4625 existing = self.bitmap
4626 if existing is not None:
4627 return existing
4628 except FileNotFoundError:
4629 pass # No bitmap, we'll generate one
4631 # Generate new bitmap
4632 if progress:
4633 progress(f"Generating bitmap for {self.name().decode('utf-8')}...\n")
4635 pack_bitmap = generate_bitmap(
4636 self.index,
4637 object_store,
4638 refs,
4639 self.get_stored_checksum(),
4640 commit_interval=commit_interval,
4641 progress=progress,
4642 )
4644 # Write bitmap file
4645 write_bitmap(self._bitmap_path, pack_bitmap)
4647 if progress:
4648 progress(f"Wrote {self._bitmap_path}\n")
4650 # Update cached bitmap
4651 self._bitmap = pack_bitmap
4653 return pack_bitmap
4655 @property
4656 def mmap_size(self) -> int:
4657 """Return the total mmapped memory usage of this pack.
4659 This includes the pack data file and index file sizes,
4660 but only for components that have been loaded (and thus mmapped).
4661 """
4662 total = 0
4663 if self._data is not None:
4664 total += self._data._size
4665 if self._idx is not None and isinstance(self._idx, FilePackIndex):
4666 total += self._idx._size
4667 return total
4669 def close(self) -> None:
4670 """Close the pack file and index."""
4671 if self._data is not None:
4672 self._data.close()
4673 self._data = None
4674 if self._idx is not None:
4675 self._idx.close()
4676 self._idx = None
4678 def __del__(self) -> None:
4679 """Ensure pack file is closed when Pack is garbage collected."""
4680 if self._data is not None or self._idx is not None:
4681 import warnings
4683 warnings.warn(
4684 f"unclosed Pack {self!r}", ResourceWarning, stacklevel=2, source=self
4685 )
4686 try:
4687 self.close()
4688 except Exception:
4689 # Ignore errors during cleanup
4690 pass
4692 def __enter__(self) -> Self:
4693 """Enter context manager."""
4694 return self
4696 def __exit__(
4697 self,
4698 type: type | None,
4699 value: BaseException | None,
4700 traceback: TracebackType | None,
4701 ) -> None:
4702 """Exit context manager."""
4703 self.close()
4705 def __eq__(self, other: object) -> bool:
4706 """Check equality with another pack."""
4707 if not isinstance(other, Pack):
4708 return False
4709 return self.index == other.index
4711 def __len__(self) -> int:
4712 """Number of entries in this pack."""
4713 return len(self.index)
4715 def __repr__(self) -> str:
4716 """Return string representation of this pack."""
4717 return f"{self.__class__.__name__}({self._basename!r})"
4719 def __iter__(self) -> Iterator[ObjectID]:
4720 """Iterate over all the sha1s of the objects in this pack."""
4721 return iter(self.index)
4723 def check_length_and_checksum(self) -> None:
4724 """Sanity check the length and checksum of the pack index and data."""
4725 assert len(self.index) == len(self.data), (
4726 f"Length mismatch: {len(self.index)} (index) != {len(self.data)} (data)"
4727 )
4728 idx_stored_checksum = self.index.get_pack_checksum()
4729 data_stored_checksum = self.data.get_stored_checksum()
4730 if (
4731 idx_stored_checksum is not None
4732 and idx_stored_checksum != data_stored_checksum
4733 ):
4734 raise ChecksumMismatch(
4735 sha_to_hex(RawObjectID(idx_stored_checksum)),
4736 sha_to_hex(RawObjectID(data_stored_checksum)),
4737 )
4739 def check(self) -> None:
4740 """Check the integrity of this pack.
4742 Raises:
4743 ChecksumMismatch: if a checksum for the index or data is wrong
4744 """
4745 self.index.check()
4746 self.data.check()
4747 for obj in self.iterobjects():
4748 obj.check()
4749 # TODO: object connectivity checks
4751 def get_stored_checksum(self) -> bytes:
4752 """Return the stored checksum of the pack data."""
4753 return self.data.get_stored_checksum()
4755 def pack_tuples(self) -> list[tuple[ShaFile, None]]:
4756 """Return pack tuples for all objects in pack."""
4757 return [(o, None) for o in self.iterobjects()]
4759 def __contains__(self, sha1: ObjectID | RawObjectID) -> bool:
4760 """Check whether this pack contains a particular SHA1."""
4761 try:
4762 self.index.object_offset(sha1)
4763 return True
4764 except KeyError:
4765 return False
4767 def get_raw(self, sha1: RawObjectID | ObjectID) -> tuple[int, bytes]:
4768 """Get raw object data by SHA1."""
4769 offset = self.index.object_offset(sha1)
4770 obj_type, obj = self.data.get_object_at(offset)
4771 type_num, chunks = self.resolve_object(offset, obj_type, obj)
4772 return type_num, b"".join(chunks) # type: ignore[arg-type]
4774 def __getitem__(self, sha1: "ObjectID | RawObjectID") -> ShaFile:
4775 """Retrieve the specified SHA1."""
4776 type, uncomp = self.get_raw(sha1)
4777 return ShaFile.from_raw_string(type, uncomp, sha=sha1)
4779 def iterobjects(self) -> Iterator[ShaFile]:
4780 """Iterate over the objects in this pack."""
4781 return iter(
4782 PackInflater.for_pack_data(self.data, resolve_ext_ref=self.resolve_ext_ref)
4783 )
4785 def iterobjects_subset(
4786 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
4787 ) -> Iterator[ShaFile]:
4788 """Iterate over a subset of objects in this pack."""
4789 return (
4790 uo
4791 for uo in PackInflater.for_pack_subset(
4792 self,
4793 shas,
4794 allow_missing=allow_missing,
4795 resolve_ext_ref=self.resolve_ext_ref,
4796 )
4797 if uo.id in shas
4798 )
4800 def iter_unpacked_subset(
4801 self,
4802 shas: Iterable[ObjectID | RawObjectID],
4803 *,
4804 include_comp: bool = False,
4805 allow_missing: bool = False,
4806 convert_ofs_delta: bool = False,
4807 ) -> Iterator[UnpackedObject]:
4808 """Iterate over unpacked objects in subset."""
4809 ofs_pending: dict[int, list[UnpackedObject]] = defaultdict(list)
4810 ofs: dict[int, bytes] = {}
4811 todo: set[ObjectID | RawObjectID] = set(shas)
4812 for unpacked in self.iter_unpacked(include_comp=include_comp):
4813 sha = unpacked.sha()
4814 if unpacked.offset is not None:
4815 ofs[unpacked.offset] = sha
4816 hexsha = sha_to_hex(RawObjectID(sha))
4817 if hexsha in todo:
4818 if unpacked.pack_type_num == OFS_DELTA:
4819 assert isinstance(unpacked.delta_base, int)
4820 assert unpacked.offset is not None
4821 base_offset = unpacked.offset - unpacked.delta_base
4822 try:
4823 unpacked.delta_base = ofs[base_offset]
4824 except KeyError:
4825 ofs_pending[base_offset].append(unpacked)
4826 continue
4827 else:
4828 unpacked.pack_type_num = REF_DELTA
4829 yield unpacked
4830 todo.remove(hexsha)
4831 if unpacked.offset is not None:
4832 for child in ofs_pending.pop(unpacked.offset, []):
4833 child.pack_type_num = REF_DELTA
4834 child.delta_base = sha
4835 yield child
4836 assert not ofs_pending
4837 if not allow_missing and todo:
4838 raise UnresolvedDeltas(list(todo))
4840 def iter_unpacked(self, include_comp: bool = False) -> Iterator[UnpackedObject]:
4841 """Iterate over all unpacked objects in this pack."""
4842 ofs_to_entries = {
4843 ofs: (sha, crc32) for (sha, ofs, crc32) in self.index.iterentries()
4844 }
4845 for unpacked in self.data.iter_unpacked(include_comp=include_comp):
4846 assert unpacked.offset is not None
4847 (sha, crc32) = ofs_to_entries[unpacked.offset]
4848 unpacked._sha = sha
4849 unpacked.crc32 = crc32
4850 yield unpacked
4852 def keep(self, msg: bytes | None = None) -> str:
4853 """Add a .keep file for the pack, preventing git from garbage collecting it.
4855 Args:
4856 msg: A message written inside the .keep file; can be used later
4857 to determine whether or not a .keep file is obsolete.
4858 Returns: The path of the .keep file, as a string.
4859 """
4860 keepfile_name = f"{self._basename}.keep"
4861 with GitFile(keepfile_name, "wb") as keepfile:
4862 if msg:
4863 keepfile.write(msg)
4864 keepfile.write(b"\n")
4865 return keepfile_name
4867 def unkeep(self) -> bool:
4868 """Remove the .keep file for the pack, allowing git to garbage collect it.
4870 This is the counterpart of :meth:`keep`. It is not an error to call
4871 this on a pack that has no .keep file.
4873 Returns: True if a .keep file was removed, False if there was none.
4874 """
4875 try:
4876 os.unlink(f"{self._basename}.keep")
4877 except FileNotFoundError:
4878 return False
4879 return True
4881 def get_ref(
4882 self, sha: RawObjectID | ObjectID
4883 ) -> tuple[int | None, int, OldUnpackedObject]:
4884 """Get the object for a ref SHA, only looking in this pack."""
4885 # TODO: cache these results
4886 try:
4887 offset = self.index.object_offset(sha)
4888 except KeyError:
4889 offset = None
4890 if offset:
4891 type, obj = self.data.get_object_at(offset)
4892 elif self.resolve_ext_ref:
4893 type, obj = self.resolve_ext_ref(sha)
4894 else:
4895 raise KeyError(sha)
4896 return offset, type, obj
4898 def resolve_object(
4899 self,
4900 offset: int,
4901 type: int,
4902 obj: OldUnpackedObject,
4903 get_ref: Callable[
4904 [RawObjectID | ObjectID], tuple[int | None, int, OldUnpackedObject]
4905 ]
4906 | None = None,
4907 ) -> tuple[int, OldUnpackedObject]:
4908 """Resolve an object, possibly resolving deltas when necessary.
4910 Returns: Tuple with object type and contents.
4911 """
4912 # Walk down the delta chain, building a stack of deltas to reach
4913 # the requested object.
4914 base_offset: int | None = offset
4915 base_type = type
4916 base_obj = obj
4917 delta_stack = []
4918 seen_ref_offsets: set[int] = {offset}
4919 while base_type in DELTA_TYPES:
4920 prev_offset = base_offset
4921 if get_ref is None:
4922 get_ref = self.get_ref
4923 assert isinstance(base_obj, tuple), (
4924 f"Expected delta tuple, got {base_obj.__class__.__name__}"
4925 )
4926 if base_type == OFS_DELTA:
4927 (delta_offset, delta) = base_obj
4928 # TODO: clean up asserts and replace with nicer error messages
4929 assert isinstance(delta_offset, int), (
4930 f"Expected int, got {delta_offset.__class__}"
4931 )
4932 assert base_offset is not None
4933 base_offset = base_offset - delta_offset
4934 base_type, base_obj = self.data.get_object_at(base_offset)
4935 assert isinstance(base_type, int)
4936 elif base_type == REF_DELTA:
4937 (basename, delta) = base_obj
4938 assert (
4939 isinstance(basename, bytes)
4940 and len(basename) == self.object_format.oid_length
4941 )
4942 base_offset_temp, base_type, base_obj = get_ref(RawObjectID(basename))
4943 assert isinstance(base_type, int)
4944 # base_offset_temp can be None for thin packs (external references)
4945 base_offset = base_offset_temp
4946 if base_offset == prev_offset: # object is based on itself
4947 raise UnresolvedDeltas([basename])
4948 # A repeated REF_DELTA base offset means the chain cycles;
4949 # without this check the loop would never terminate.
4950 if base_offset is not None:
4951 if base_offset in seen_ref_offsets:
4952 raise DeltaCycle([basename])
4953 seen_ref_offsets.add(base_offset)
4954 else:
4955 raise AssertionError(f"Unexpected delta type: {base_type}")
4956 delta_stack.append((prev_offset, base_type, delta))
4958 # Now grab the base object (mustn't be a delta) and apply the
4959 # deltas all the way up the stack.
4960 chunks = base_obj
4961 for prev_offset, _delta_type, delta in reversed(delta_stack):
4962 # Convert chunks to bytes for apply_delta if needed
4963 if isinstance(chunks, list):
4964 chunks_bytes = b"".join(chunks)
4965 elif isinstance(chunks, tuple):
4966 # For tuple type, second element is the actual data
4967 _, chunk_data = chunks
4968 if isinstance(chunk_data, list):
4969 chunks_bytes = b"".join(chunk_data)
4970 else:
4971 chunks_bytes = chunk_data
4972 else:
4973 chunks_bytes = chunks
4975 # Apply delta and get result as list
4976 chunks = apply_delta(chunks_bytes, delta)
4978 if prev_offset is not None:
4979 self.data._cache_object_at(prev_offset, base_type, chunks)
4980 return base_type, chunks
4982 def entries(
4983 self, progress: Callable[[int, int], None] | None = None
4984 ) -> Iterator[PackIndexEntry]:
4985 """Yield entries summarizing the contents of this pack.
4987 Args:
4988 progress: Progress function, called with current and total
4989 object count.
4990 Returns: iterator of tuples with (sha, offset, crc32)
4991 """
4992 return self.data.iterentries(
4993 progress=progress, resolve_ext_ref=self.resolve_ext_ref
4994 )
4996 def sorted_entries(
4997 self, progress: Callable[[int, int], None] | None = None
4998 ) -> Iterator[PackIndexEntry]:
4999 """Return entries in this pack, sorted by SHA.
5001 Args:
5002 progress: Progress function, called with current and total
5003 object count
5004 Returns: Iterator of tuples with (sha, offset, crc32)
5005 """
5006 return iter(
5007 self.data.sorted_entries(
5008 progress=progress, resolve_ext_ref=self.resolve_ext_ref
5009 )
5010 )
5012 def get_unpacked_object(
5013 self,
5014 sha: ObjectID | RawObjectID,
5015 *,
5016 include_comp: bool = False,
5017 convert_ofs_delta: bool = True,
5018 ) -> UnpackedObject:
5019 """Get the unpacked object for a sha.
5021 Args:
5022 sha: SHA of object to fetch
5023 include_comp: Whether to include compression data in UnpackedObject
5024 convert_ofs_delta: Whether to convert offset deltas to ref deltas
5025 """
5026 offset = self.index.object_offset(sha)
5027 unpacked = self.data.get_unpacked_object_at(offset, include_comp=include_comp)
5028 if unpacked.pack_type_num == OFS_DELTA and convert_ofs_delta:
5029 assert isinstance(unpacked.delta_base, int)
5030 unpacked.delta_base = self.index.object_sha1(offset - unpacked.delta_base)
5031 unpacked.pack_type_num = REF_DELTA
5032 return unpacked
5035def extend_pack(
5036 f: BinaryIO,
5037 object_ids: Set["RawObjectID"],
5038 get_raw: Callable[["RawObjectID | ObjectID"], tuple[int, bytes]],
5039 object_format: "ObjectFormat",
5040 *,
5041 compression_level: int = -1,
5042 progress: Callable[[bytes], None] | None = None,
5043) -> tuple[bytes, list[tuple[RawObjectID, int, int]]]:
5044 """Extend a pack file with more objects.
5046 The caller should make sure that object_ids does not contain any objects
5047 that are already in the pack
5048 """
5049 # Update the header with the new number of objects.
5050 f.seek(0)
5051 _version, num_objects = read_pack_header(f.read)
5053 if object_ids:
5054 f.seek(0)
5055 write_pack_header(f.write, num_objects + len(object_ids))
5057 # Must flush before reading (http://bugs.python.org/issue3207)
5058 f.flush()
5060 # Rescan the rest of the pack, computing the SHA with the new header.
5061 new_sha = compute_file_sha(
5062 f, hash_func=object_format.hash_func, end_ofs=-object_format.oid_length
5063 )
5065 # Must reposition before writing (http://bugs.python.org/issue3207)
5066 f.seek(0, os.SEEK_CUR)
5068 extra_entries = []
5070 # Complete the pack.
5071 for i, object_id in enumerate(object_ids):
5072 if progress is not None:
5073 progress(
5074 (f"writing extra base objects: {i}/{len(object_ids)}\r").encode("ascii")
5075 )
5076 assert len(object_id) == object_format.oid_length
5077 type_num, data = get_raw(object_id)
5078 offset = f.tell()
5079 crc32 = write_pack_object(
5080 f.write,
5081 type_num,
5082 [data], # Convert bytes to list[bytes]
5083 sha=new_sha,
5084 compression_level=compression_level,
5085 object_format=object_format,
5086 )
5087 extra_entries.append((object_id, offset, crc32))
5088 pack_sha = new_sha.digest()
5089 f.write(pack_sha)
5090 return pack_sha, extra_entries
5093try:
5094 from dulwich._pack import ( # type: ignore
5095 apply_delta,
5096 bisect_find_sha,
5097 )
5098except ImportError:
5099 pass
5101# Try to import the Rust version of create_delta
5102try:
5103 from dulwich._pack import create_delta as _create_delta_rs
5104except ImportError:
5105 pass
5106else:
5107 # Wrap the Rust version to match the Python API (returns bytes instead of Iterator)
5108 def _create_delta_rs_wrapper(
5109 base_buf: bytes | list[bytes], target_buf: bytes | list[bytes]
5110 ) -> Iterator[bytes]:
5111 """Wrapper for Rust create_delta to match Python API."""
5112 if isinstance(base_buf, list):
5113 base_buf = b"".join(base_buf)
5114 if isinstance(target_buf, list):
5115 target_buf = b"".join(target_buf)
5116 yield _create_delta_rs(base_buf, target_buf)
5118 create_delta = _create_delta_rs_wrapper