Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/object_store.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# object_store.py -- Object store for git objects
2# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
3# and others
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
24"""Git object store interfaces and implementation."""
26__all__ = [
27 "DEFAULT_TEMPFILE_GRACE_PERIOD",
28 "INFODIR",
29 "PACKDIR",
30 "PACK_MODE",
31 "BaseObjectStore",
32 "BitmapReachability",
33 "BucketBasedObjectStore",
34 "DiskObjectStore",
35 "GraphTraversalReachability",
36 "GraphWalker",
37 "MemoryObjectStore",
38 "MissingObjectFinder",
39 "ObjectIterator",
40 "ObjectReachabilityProvider",
41 "ObjectStoreGraphWalker",
42 "OverlayObjectStore",
43 "PackBasedObjectStore",
44 "PackCapableObjectStore",
45 "PackContainer",
46 "PackInputTooLarge",
47 "commit_tree_changes",
48 "find_shallow",
49 "get_depth",
50 "iter_commit_contents",
51 "iter_tree_contents",
52 "peel_sha",
53 "read_packs_file",
54 "tree_lookup_path",
55]
57import binascii
58import logging
59import os
60import stat
61import sys
62import time
63import warnings
64from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set
65from contextlib import closing, suppress
66from io import BytesIO
67from pathlib import Path
68from typing import (
69 TYPE_CHECKING,
70 BinaryIO,
71 Protocol,
72 TypeVar,
73 cast,
74)
76if TYPE_CHECKING:
77 from .object_format import ObjectFormat
79from .errors import NotTreeError
80from .file import GitFile, _GitFile
81from .midx import MultiPackIndex, load_midx
82from .objects import (
83 DEFAULT_LOOSE_OBJECT_SIZE_LIMIT,
84 S_ISGITLINK,
85 Blob,
86 Commit,
87 ObjectID,
88 RawObjectID,
89 ShaFile,
90 Tag,
91 Tree,
92 TreeEntry,
93 hex_to_filename,
94 hex_to_sha,
95 object_class,
96 sha_to_hex,
97 valid_hexsha,
98)
99from .pack import (
100 PACK_SPOOL_FILE_MAX_SIZE,
101 ObjectContainer,
102 Pack,
103 PackData,
104 PackedObjectContainer,
105 PackFileDisappeared,
106 PackHint,
107 PackIndexer,
108 PackInflater,
109 PackStreamCopier,
110 UnpackedObject,
111 extend_pack,
112 full_unpacked_object,
113 generate_unpacked_objects,
114 iter_sha1,
115 load_pack_index_file,
116 pack_objects_to_data,
117 write_pack_data,
118 write_pack_index,
119)
120from .protocol import DEPTH_INFINITE, PEELED_TAG_SUFFIX
121from .refs import Ref
123if TYPE_CHECKING:
124 from .bitmap import EWAHBitmap
125 from .commit_graph import CommitGraph
126 from .config import Config
127 from .diff_tree import RenameDetector
128 from .pack import FilePackIndex, Pack
131logger = logging.getLogger(__name__)
133# Maximum number of times to rescan the pack directory after a pack file
134# disappears between snapshot and lazy open (e.g. concurrent repack).
135# Mirrors git's bounded reprepare_packed_git() retry.
136_MAX_PACK_RESCAN_ATTEMPTS = 3
138_T = TypeVar("_T")
141class GraphWalker(Protocol):
142 """Protocol for graph walker objects.
144 Implementations may also expose a ``shallow`` set, an ``unshallow`` set,
145 and an ``update_shallow`` callable for shallow-clone negotiation. These
146 are not part of the minimal protocol and callers must use ``hasattr`` or
147 ``getattr`` to access them.
148 """
150 def __next__(self) -> ObjectID | None:
151 """Return the next object SHA to visit."""
152 ...
154 def ack(self, sha: ObjectID, /) -> None:
155 """Acknowledge that an object has been received."""
156 ...
158 def nak(self) -> None:
159 """Nothing in common was found."""
160 ...
163class ObjectReachabilityProvider(Protocol):
164 """Protocol for computing object reachability queries.
166 This abstraction allows reachability computations to be backed by either
167 naive graph traversal or optimized bitmap indexes, with a consistent interface.
168 """
170 def get_reachable_commits(
171 self,
172 heads: Iterable[ObjectID],
173 exclude: Iterable[ObjectID] | None = None,
174 shallow: Set[ObjectID] | None = None,
175 ) -> set[ObjectID]:
176 """Get all commits reachable from heads, excluding those in exclude.
178 Args:
179 heads: Starting commit SHAs
180 exclude: Commit SHAs to exclude (and their ancestors)
181 shallow: Set of shallow commit boundaries (traversal stops here)
183 Returns:
184 Set of commit SHAs reachable from heads but not from exclude
185 """
186 ...
188 def get_reachable_objects(
189 self,
190 commits: Iterable[ObjectID],
191 exclude_commits: Iterable[ObjectID] | None = None,
192 ) -> set[ObjectID]:
193 """Get all objects (commits + trees + blobs) reachable from commits.
195 Args:
196 commits: Starting commit SHAs
197 exclude_commits: Commits whose objects should be excluded
199 Returns:
200 Set of all object SHAs (commits, trees, blobs, tags)
201 """
202 ...
204 def get_tree_objects(
205 self,
206 tree_shas: Iterable[ObjectID],
207 ) -> set[ObjectID]:
208 """Get all trees and blobs reachable from the given trees.
210 Args:
211 tree_shas: Starting tree SHAs
213 Returns:
214 Set of tree and blob SHAs
215 """
216 ...
219INFODIR = "info"
220PACKDIR = "pack"
222# use permissions consistent with Git; just readable by everyone
223# TODO: should packs also be non-writable on Windows? if so, that
224# would requite some rather significant adjustments to the test suite
225PACK_MODE = 0o444 if sys.platform != "win32" else 0o644
227# Grace period for cleaning up temporary pack files (in seconds)
228# Matches git's default of 2 weeks
229DEFAULT_TEMPFILE_GRACE_PERIOD = 14 * 24 * 60 * 60 # 2 weeks
232def _remove_readonly(path: str) -> None:
233 """Remove a file, clearing the read-only attribute first on Windows.
235 git stores pack files and loose objects read-only. Unix lets you unlink a
236 read-only file in a writable directory, but Windows refuses with
237 PermissionError, so clear the attribute and retry there.
238 """
239 try:
240 os.remove(path)
241 except PermissionError:
242 if sys.platform != "win32":
243 raise
244 os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
245 os.remove(path)
248class PackInputTooLarge(OSError):
249 """Raised when a received pack exceeds the configured input size cap.
251 Mirrors the failure mode of git's ``receive.maxInputSize`` /
252 ``git index-pack --max-input-size``.
253 """
256def _bound_read_callables(
257 read_all: Callable[[int], bytes],
258 read_some: Callable[[int], bytes] | None,
259 max_input_size: int,
260) -> tuple[Callable[[int], bytes], Callable[[int], bytes] | None]:
261 """Wrap pack-stream read callbacks so total bytes are capped.
263 When the cumulative number of bytes returned across ``read_all`` and
264 ``read_some`` exceeds ``max_input_size``, the next read raises
265 ``PackInputTooLarge``. This is the in-process analogue of
266 ``git index-pack --max-input-size``.
267 """
268 bytes_read = [0]
270 def _check(n: int) -> None:
271 bytes_read[0] += n
272 if bytes_read[0] > max_input_size:
273 raise PackInputTooLarge(
274 f"pack exceeds maximum input size of {max_input_size} bytes"
275 )
277 def wrapped_read_all(n: int) -> bytes:
278 data = read_all(n)
279 _check(len(data))
280 return data
282 if read_some is None:
283 return wrapped_read_all, None
285 def wrapped_read_some(n: int) -> bytes:
286 data = read_some(n)
287 _check(len(data))
288 return data
290 return wrapped_read_all, wrapped_read_some
293def find_shallow(
294 store: ObjectContainer, heads: Iterable[ObjectID], depth: int
295) -> tuple[set[ObjectID], set[ObjectID]]:
296 """Find shallow commits according to a given depth.
298 Args:
299 store: An ObjectStore for looking up objects.
300 heads: Iterable of head SHAs to start walking from.
301 depth: The depth of ancestors to include. A depth of one includes
302 only the heads themselves.
303 Returns: A tuple of (shallow, not_shallow), sets of SHAs that should be
304 considered shallow and unshallow according to the arguments. Note that
305 these sets may overlap if a commit is reachable along multiple paths.
306 """
307 parents: dict[ObjectID, list[ObjectID]] = {}
308 commit_graph = store.get_commit_graph()
310 def get_parents(sha: ObjectID) -> list[ObjectID]:
311 result = parents.get(sha, None)
312 if not result:
313 # Try to use commit graph first if available
314 if commit_graph:
315 graph_parents = commit_graph.get_parents(sha)
316 if graph_parents is not None:
317 result = graph_parents
318 parents[sha] = result
319 return result
320 # Fall back to loading the object
321 commit = store[sha]
322 assert isinstance(commit, Commit)
323 result = commit.parents
324 parents[sha] = result
325 return result
327 todo = [] # stack of (sha, depth)
328 for head_sha in heads:
329 obj = store[head_sha]
330 # Peel tags if necessary
331 while isinstance(obj, Tag):
332 _, sha = obj.object
333 obj = store[sha]
334 if isinstance(obj, Commit):
335 todo.append((obj.id, 1))
337 not_shallow = set()
338 shallow = set()
339 while todo:
340 sha, cur_depth = todo.pop()
341 if cur_depth < depth:
342 not_shallow.add(sha)
343 new_depth = cur_depth + 1
344 todo.extend((p, new_depth) for p in get_parents(sha))
345 else:
346 shallow.add(sha)
348 return shallow, not_shallow
351def get_depth(
352 store: ObjectContainer,
353 head: ObjectID,
354 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
355 max_depth: int | None = None,
356) -> int:
357 """Return the current available depth for the given head.
359 For commits with multiple parents, the largest possible depth will be
360 returned.
362 Args:
363 store: Object store to search in
364 head: commit to start from
365 get_parents: optional function for getting the parents of a commit
366 max_depth: maximum depth to search
367 """
368 if head not in store:
369 return 0
370 current_depth = 1
371 queue = [(head, current_depth)]
372 commit_graph = store.get_commit_graph()
374 while queue and (max_depth is None or current_depth < max_depth):
375 e, depth = queue.pop(0)
376 current_depth = max(current_depth, depth)
378 # Try to use commit graph for parent lookup if available
379 parents = None
380 if commit_graph:
381 parents = commit_graph.get_parents(e)
383 if parents is None:
384 # Fall back to loading the object
385 cmt = store[e]
386 if isinstance(cmt, Tag):
387 _cls, sha = cmt.object
388 cmt = store[sha]
389 parents = get_parents(cmt)
391 queue.extend((parent, depth + 1) for parent in parents if parent in store)
392 return current_depth
395class PackContainer(Protocol):
396 """Protocol for containers that can accept pack files."""
398 def add_pack(self) -> tuple[BytesIO, Callable[[], None], Callable[[], None]]:
399 """Add a new pack."""
400 ...
403class BaseObjectStore:
404 """Object store interface."""
406 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
407 """Initialize object store.
409 Args:
410 object_format: Object format to use (defaults to DEFAULT_OBJECT_FORMAT)
411 """
412 from .object_format import DEFAULT_OBJECT_FORMAT
414 self.object_format = object_format if object_format else DEFAULT_OBJECT_FORMAT
416 def determine_wants_all(
417 self, refs: Mapping[Ref, ObjectID], depth: int | None = None
418 ) -> list[ObjectID]:
419 """Determine which objects are wanted based on refs."""
421 def _want_deepen(sha: ObjectID) -> bool:
422 if not depth:
423 return False
424 if depth == DEPTH_INFINITE:
425 return True
426 return depth > self._get_depth(sha)
428 return [
429 sha
430 for (ref, sha) in refs.items()
431 if (sha not in self or _want_deepen(sha))
432 and not ref.endswith(PEELED_TAG_SUFFIX)
433 ]
435 def contains_loose(self, sha: ObjectID) -> bool:
436 """Check if a particular object is present by SHA1 and is loose."""
437 raise NotImplementedError(self.contains_loose)
439 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
440 """Check if a particular object is present by SHA1 and is packed."""
441 return False # Default implementation for stores that don't support packing
443 def __contains__(self, sha1: ObjectID) -> bool:
444 """Check if a particular object is present by SHA1.
446 This method makes no distinction between loose and packed objects.
447 """
448 return self.contains_loose(sha1)
450 @property
451 def packs(self) -> list[Pack]:
452 """Iterable of pack objects."""
453 raise NotImplementedError
455 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
456 """Obtain the raw text for an object.
458 Args:
459 name: sha for the object.
460 Returns: tuple with numeric type and object contents.
461 """
462 raise NotImplementedError(self.get_raw)
464 def __getitem__(self, sha1: ObjectID | RawObjectID) -> ShaFile:
465 """Obtain an object by SHA1.
467 Raises:
468 ChecksumMismatch: if the stored contents do not hash to the
469 requested object id.
470 """
471 if len(sha1) == self.object_format.oid_length:
472 hexsha = sha_to_hex(RawObjectID(sha1))
473 else:
474 hexsha = ObjectID(sha1)
475 type_num, uncomp = self.get_raw(sha1)
476 return ShaFile.from_raw_string(
477 type_num, uncomp, verify_sha=hexsha, object_format=self.object_format
478 )
480 def __iter__(self) -> Iterator[ObjectID]:
481 """Iterate over the SHAs that are present in this store."""
482 raise NotImplementedError(self.__iter__)
484 def add_object(self, obj: ShaFile) -> None:
485 """Add a single object to this object store."""
486 raise NotImplementedError(self.add_object)
488 def add_objects(
489 self,
490 objects: Sequence[tuple[ShaFile, str | None]],
491 progress: Callable[..., None] | None = None,
492 ) -> "Pack | None":
493 """Add a set of objects to this object store.
495 Args:
496 objects: Iterable over a list of (object, path) tuples
497 progress: Optional progress callback
498 """
499 raise NotImplementedError(self.add_objects)
501 def get_reachability_provider(
502 self, prefer_bitmaps: bool = True
503 ) -> ObjectReachabilityProvider:
504 """Get a reachability provider for this object store.
506 Returns an ObjectReachabilityProvider that can efficiently compute
507 object reachability queries. Subclasses can override this to provide
508 optimized implementations (e.g., using bitmap indexes).
510 Args:
511 prefer_bitmaps: Whether to prefer bitmap-based reachability if
512 available.
514 Returns:
515 ObjectReachabilityProvider instance
516 """
517 return GraphTraversalReachability(self)
519 def tree_changes(
520 self,
521 source: ObjectID | None,
522 target: ObjectID | None,
523 want_unchanged: bool = False,
524 include_trees: bool = False,
525 change_type_same: bool = False,
526 rename_detector: "RenameDetector | None" = None,
527 paths: Sequence[bytes] | None = None,
528 ) -> Iterator[
529 tuple[
530 tuple[bytes | None, bytes | None],
531 tuple[int | None, int | None],
532 tuple[ObjectID | None, ObjectID | None],
533 ]
534 ]:
535 """Find the differences between the contents of two trees.
537 Args:
538 source: SHA1 of the source tree
539 target: SHA1 of the target tree
540 want_unchanged: Whether unchanged files should be reported
541 include_trees: Whether to include trees
542 change_type_same: Whether to report files changing
543 type in the same entry.
544 rename_detector: RenameDetector object for detecting renames.
545 paths: Optional list of paths to filter to (as bytes).
546 Returns: Iterator over tuples with
547 (oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
548 """
549 from .diff_tree import tree_changes
551 for change in tree_changes(
552 self,
553 source,
554 target,
555 want_unchanged=want_unchanged,
556 include_trees=include_trees,
557 change_type_same=change_type_same,
558 rename_detector=rename_detector,
559 paths=paths,
560 ):
561 old_path = change.old.path if change.old is not None else None
562 new_path = change.new.path if change.new is not None else None
563 old_mode = change.old.mode if change.old is not None else None
564 new_mode = change.new.mode if change.new is not None else None
565 old_sha = change.old.sha if change.old is not None else None
566 new_sha = change.new.sha if change.new is not None else None
567 yield (
568 (old_path, new_path),
569 (old_mode, new_mode),
570 (old_sha, new_sha),
571 )
573 def iter_tree_contents(
574 self, tree_id: ObjectID, include_trees: bool = False
575 ) -> Iterator[TreeEntry]:
576 """Iterate the contents of a tree and all subtrees.
578 Iteration is depth-first pre-order, as in e.g. os.walk.
580 Args:
581 tree_id: SHA1 of the tree.
582 include_trees: If True, include tree objects in the iteration.
583 Returns: Iterator over TreeEntry namedtuples for all the objects in a
584 tree.
585 """
586 warnings.warn(
587 "Please use dulwich.object_store.iter_tree_contents",
588 DeprecationWarning,
589 stacklevel=2,
590 )
591 return iter_tree_contents(self, tree_id, include_trees=include_trees)
593 def iterobjects_subset(
594 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
595 ) -> Iterator[ShaFile]:
596 """Iterate over a subset of objects in the store.
598 Args:
599 shas: Iterable of object SHAs to retrieve
600 allow_missing: If True, skip missing objects; if False, raise KeyError
602 Returns:
603 Iterator of ShaFile objects
605 Raises:
606 KeyError: If an object is missing and allow_missing is False
607 """
608 for sha in shas:
609 try:
610 yield self[sha]
611 except KeyError:
612 if not allow_missing:
613 raise
615 def iter_unpacked_subset(
616 self,
617 shas: Iterable[ObjectID | RawObjectID],
618 *,
619 include_comp: bool = False,
620 allow_missing: bool = False,
621 convert_ofs_delta: bool = True,
622 ) -> "Iterator[UnpackedObject]":
623 """Iterate over unpacked objects for a subset of SHAs.
625 Default implementation that converts ShaFile objects to UnpackedObject.
626 Subclasses may override for more efficient unpacked access.
628 Args:
629 shas: Iterable of object SHAs to retrieve
630 include_comp: Whether to include compressed data (ignored in base
631 implementation)
632 allow_missing: If True, skip missing objects; if False, raise
633 KeyError
634 convert_ofs_delta: Whether to convert OFS_DELTA objects (ignored in
635 base implementation)
637 Returns:
638 Iterator of UnpackedObject instances
640 Raises:
641 KeyError: If an object is missing and allow_missing is False
642 """
643 from .pack import UnpackedObject
645 for sha in shas:
646 try:
647 obj = self[sha]
648 # Convert ShaFile to UnpackedObject
649 unpacked = UnpackedObject(
650 obj.type_num, decomp_chunks=obj.as_raw_chunks(), sha=obj.id
651 )
652 yield unpacked
653 except KeyError:
654 if not allow_missing:
655 raise
657 def find_missing_objects(
658 self,
659 haves: Iterable[ObjectID],
660 wants: Iterable[ObjectID],
661 shallow: Set[ObjectID] | None = None,
662 progress: Callable[..., None] | None = None,
663 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
664 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
665 ) -> Iterator[tuple[ObjectID, PackHint | None]]:
666 """Find the missing objects required for a set of revisions.
668 Args:
669 haves: Iterable over SHAs already in common.
670 wants: Iterable over SHAs of objects to fetch.
671 shallow: Set of shallow commit SHA1s to skip
672 progress: Simple progress function that will be called with
673 updated progress strings.
674 get_tagged: Function that returns a dict of pointed-to sha ->
675 tag sha for including tags.
676 get_parents: Optional function for getting the parents of a
677 commit.
678 Returns: Iterator over (sha, path) pairs.
679 """
680 warnings.warn("Please use MissingObjectFinder(store)", DeprecationWarning)
681 finder = MissingObjectFinder(
682 self,
683 haves=haves,
684 wants=wants,
685 shallow=shallow,
686 progress=progress,
687 get_tagged=get_tagged,
688 get_parents=get_parents,
689 )
690 return iter(finder)
692 def find_common_revisions(self, graphwalker: GraphWalker) -> list[ObjectID]:
693 """Find which revisions this store has in common using graphwalker.
695 Args:
696 graphwalker: A graphwalker object.
697 Returns: List of SHAs that are in common
698 """
699 haves = []
700 sha = next(graphwalker)
701 while sha:
702 if sha in self:
703 haves.append(sha)
704 graphwalker.ack(sha)
705 sha = next(graphwalker)
706 return haves
708 def generate_pack_data(
709 self,
710 have: Iterable[ObjectID],
711 want: Iterable[ObjectID],
712 *,
713 shallow: Set[ObjectID] | None = None,
714 progress: Callable[..., None] | None = None,
715 ofs_delta: bool = True,
716 ) -> tuple[int, Iterator[UnpackedObject]]:
717 """Generate pack data objects for a set of wants/haves.
719 Args:
720 have: List of SHA1s of objects that should not be sent
721 want: List of SHA1s of objects that should be sent
722 shallow: Set of shallow commit SHA1s to skip
723 ofs_delta: Whether OFS deltas can be included
724 progress: Optional progress reporting method
725 """
726 # Note that the pack-specific implementation below is more efficient,
727 # as it reuses deltas
728 missing_objects = MissingObjectFinder(
729 self, haves=have, wants=want, shallow=shallow, progress=progress
730 )
731 object_ids = list(missing_objects)
732 return pack_objects_to_data(
733 [(self[oid], path) for oid, path in object_ids],
734 ofs_delta=ofs_delta,
735 progress=progress,
736 )
738 def peel_sha(self, sha: ObjectID | RawObjectID) -> ObjectID:
739 """Peel all tags from a SHA.
741 Args:
742 sha: The object SHA to peel.
743 Returns: The fully-peeled SHA1 of a tag object, after peeling all
744 intermediate tags; if the original ref does not point to a tag,
745 this will equal the original SHA1.
746 """
747 warnings.warn(
748 "Please use dulwich.object_store.peel_sha()",
749 DeprecationWarning,
750 stacklevel=2,
751 )
752 return peel_sha(self, sha)[1].id
754 def _get_depth(
755 self,
756 head: ObjectID,
757 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
758 max_depth: int | None = None,
759 ) -> int:
760 """Return the current available depth for the given head.
762 For commits with multiple parents, the largest possible depth will be
763 returned.
765 Args:
766 head: commit to start from
767 get_parents: optional function for getting the parents of a commit
768 max_depth: maximum depth to search
769 """
770 return get_depth(self, head, get_parents=get_parents, max_depth=max_depth)
772 def close(self) -> None:
773 """Close any files opened by this object store."""
774 # Default implementation is a NO-OP
776 def prune(self, grace_period: int | None = None) -> None:
777 """Prune/clean up this object store.
779 This includes removing orphaned temporary files and other
780 housekeeping tasks. Default implementation is a NO-OP.
782 Args:
783 grace_period: Grace period in seconds for removing temporary files.
784 If None, uses the default grace period.
785 """
786 # Default implementation is a NO-OP
788 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
789 """Iterate over all SHA1s that start with a given prefix.
791 The default implementation is a naive iteration over all objects.
792 However, subclasses may override this method with more efficient
793 implementations.
794 """
795 for sha in self:
796 if sha.startswith(prefix):
797 yield sha
799 def get_commit_graph(self) -> "CommitGraph | None":
800 """Get the commit graph for this object store.
802 Returns:
803 CommitGraph object if available, None otherwise
804 """
805 return None
807 def write_commit_graph(
808 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
809 ) -> None:
810 """Write a commit graph file for this object store.
812 Args:
813 refs: List of refs to include. If None, includes all refs from object store.
814 reachable: If True, includes all commits reachable from refs.
815 If False, only includes the direct ref targets.
817 Note:
818 Default implementation does nothing. Subclasses should override
819 this method to provide commit graph writing functionality.
820 """
821 raise NotImplementedError(self.write_commit_graph)
823 def get_object_mtime(self, sha: ObjectID) -> float:
824 """Get the modification time of an object.
826 Args:
827 sha: SHA1 of the object
829 Returns:
830 Modification time as seconds since epoch
832 Raises:
833 KeyError: if the object is not found
834 """
835 # Default implementation raises KeyError
836 # Subclasses should override to provide actual mtime
837 raise KeyError(sha)
840class PackCapableObjectStore(BaseObjectStore, PackedObjectContainer):
841 """Object store that supports pack operations.
843 This is a base class for object stores that can handle pack files,
844 including both disk-based and memory-based stores.
845 """
847 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
848 """Add a new pack to this object store.
850 Returns: Tuple of (file, commit_func, abort_func)
851 """
852 raise NotImplementedError(self.add_pack)
854 def add_pack_data(
855 self,
856 count: int,
857 unpacked_objects: Iterator["UnpackedObject"],
858 progress: Callable[..., None] | None = None,
859 ) -> "Pack | None":
860 """Add pack data to this object store.
862 Args:
863 count: Number of objects
864 unpacked_objects: Iterator over unpacked objects
865 progress: Optional progress callback
866 """
867 raise NotImplementedError(self.add_pack_data)
869 def get_unpacked_object(
870 self, sha1: ObjectID | RawObjectID, *, include_comp: bool = False
871 ) -> "UnpackedObject":
872 """Get a raw unresolved object.
874 Args:
875 sha1: SHA-1 hash of the object
876 include_comp: Whether to include compressed data
878 Returns:
879 UnpackedObject instance
880 """
881 from .pack import UnpackedObject
883 obj = self[sha1]
884 return UnpackedObject(obj.type_num, sha=sha1, decomp_chunks=obj.as_raw_chunks())
886 def iterobjects_subset(
887 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
888 ) -> Iterator[ShaFile]:
889 """Iterate over a subset of objects.
891 Args:
892 shas: Iterable of object SHAs to retrieve
893 allow_missing: If True, skip missing objects
895 Returns:
896 Iterator of ShaFile objects
897 """
898 for sha in shas:
899 try:
900 yield self[sha]
901 except KeyError:
902 if not allow_missing:
903 raise
906class PackBasedObjectStore(PackCapableObjectStore, PackedObjectContainer):
907 """Object store that uses pack files for storage.
909 This class provides a base implementation for object stores that use
910 Git pack files as their primary storage mechanism. It handles caching
911 of open pack files and provides configuration for pack file operations.
912 """
914 def __init__(
915 self,
916 pack_compression_level: int = -1,
917 pack_index_version: int | None = None,
918 pack_delta_window_size: int | None = None,
919 pack_window_memory: int | None = None,
920 pack_delta_cache_size: int | None = None,
921 pack_depth: int | None = None,
922 pack_threads: int | None = None,
923 pack_big_file_threshold: int | None = None,
924 *,
925 packed_git_limit: int | None = None,
926 delta_base_cache_limit: int | None = None,
927 object_format: "ObjectFormat | None" = None,
928 ) -> None:
929 """Initialize a PackBasedObjectStore.
931 Args:
932 pack_compression_level: Compression level for pack files (-1 to 9)
933 pack_index_version: Pack index version to use
934 pack_delta_window_size: Window size for delta compression
935 pack_window_memory: Maximum memory to use for delta window
936 pack_delta_cache_size: Cache size for delta operations
937 pack_depth: Maximum depth for pack deltas
938 pack_threads: Number of threads to use for packing
939 pack_big_file_threshold: Threshold for treating files as "big"
940 packed_git_limit: Maximum total bytes for mmapped pack files.
941 When exceeded, least-recently-used packs are closed to free memory.
942 delta_base_cache_limit: Maximum bytes for caching delta base objects.
943 Controls memory used to cache resolved base objects during delta
944 unpacking, corresponding to Git's core.deltaBaseCacheLimit.
945 object_format: Hash algorithm to use
946 """
947 super().__init__(object_format=object_format)
948 self._pack_cache: dict[str, Pack] = {}
949 self._pack_access_order: list[str] = []
950 self.packed_git_limit = packed_git_limit
951 self.delta_base_cache_limit = delta_base_cache_limit
952 self.pack_compression_level = pack_compression_level
953 self.pack_index_version = pack_index_version
954 self.pack_delta_window_size = pack_delta_window_size
955 self.pack_window_memory = pack_window_memory
956 self.pack_delta_cache_size = pack_delta_cache_size
957 self.pack_depth = pack_depth
958 self.pack_threads = pack_threads
959 self.pack_big_file_threshold = pack_big_file_threshold
961 def get_reachability_provider(
962 self,
963 prefer_bitmaps: bool = True,
964 ) -> ObjectReachabilityProvider:
965 """Get the best reachability provider for the object store.
967 Args:
968 prefer_bitmaps: Whether to use bitmaps if available
970 Returns:
971 ObjectReachabilityProvider implementation (either bitmap-accelerated
972 or graph traversal)
973 """
974 if prefer_bitmaps:
975 # Check if any packs have bitmaps
976 has_bitmap = False
977 for pack in self.packs:
978 try:
979 # Try to access bitmap property
980 if pack.bitmap is not None:
981 has_bitmap = True
982 break
983 except FileNotFoundError:
984 # Bitmap file doesn't exist for this pack
985 continue
987 if has_bitmap:
988 return BitmapReachability(self)
990 # Fall back to graph traversal
991 return GraphTraversalReachability(self)
993 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
994 """Add a new pack to this object store."""
995 raise NotImplementedError(self.add_pack)
997 def add_pack_data(
998 self,
999 count: int,
1000 unpacked_objects: Iterator[UnpackedObject],
1001 progress: Callable[..., None] | None = None,
1002 ) -> "Pack | None":
1003 """Add pack data to this object store.
1005 Args:
1006 count: Number of items to add
1007 unpacked_objects: Iterator of UnpackedObject instances
1008 progress: Optional progress callback
1009 """
1010 if count == 0:
1011 # Don't bother writing an empty pack file
1012 return None
1013 f, commit, abort = self.add_pack()
1014 try:
1015 write_pack_data(
1016 f.write,
1017 unpacked_objects,
1018 num_records=count,
1019 progress=progress,
1020 compression_level=self.pack_compression_level,
1021 object_format=self.object_format,
1022 )
1023 except BaseException:
1024 abort()
1025 raise
1026 else:
1027 return commit()
1029 @property
1030 def alternates(self) -> list["BaseObjectStore"]:
1031 """Return list of alternate object stores."""
1032 return []
1034 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
1035 """Check if a particular object is present by SHA1 and is packed.
1037 This does not check alternates.
1038 """
1040 def lookup(p: "Pack") -> bool:
1041 if sha in p:
1042 return True
1043 raise KeyError
1045 try:
1046 return self._lookup_in_packs(lookup)
1047 except KeyError:
1048 return False
1050 def __contains__(self, sha: ObjectID) -> bool:
1051 """Check if a particular object is present by SHA1.
1053 This method makes no distinction between loose and packed objects.
1054 """
1055 if self.contains_packed(sha) or self.contains_loose(sha):
1056 return True
1057 for alternate in self.alternates:
1058 if sha in alternate:
1059 return True
1060 return False
1062 def _add_cached_pack(self, base_name: str, pack: Pack) -> None:
1063 """Add a newly appeared pack to the cache by path."""
1064 prev_pack = self._pack_cache.get(base_name)
1065 if prev_pack is not pack:
1066 self._pack_cache[base_name] = pack
1067 if prev_pack:
1068 prev_pack.close()
1069 self._mark_pack_used(base_name)
1070 self._enforce_packed_git_limit()
1072 def generate_pack_data(
1073 self,
1074 have: Iterable[ObjectID],
1075 want: Iterable[ObjectID],
1076 *,
1077 shallow: Set[ObjectID] | None = None,
1078 progress: Callable[..., None] | None = None,
1079 ofs_delta: bool = True,
1080 ) -> tuple[int, Iterator[UnpackedObject]]:
1081 """Generate pack data objects for a set of wants/haves.
1083 Args:
1084 have: List of SHA1s of objects that should not be sent
1085 want: List of SHA1s of objects that should be sent
1086 shallow: Set of shallow commit SHA1s to skip
1087 ofs_delta: Whether OFS deltas can be included
1088 progress: Optional progress reporting method
1089 """
1090 missing_objects = MissingObjectFinder(
1091 self, haves=have, wants=want, shallow=shallow, progress=progress
1092 )
1093 remote_has = missing_objects.get_remote_has()
1094 object_ids = list(missing_objects)
1095 return len(object_ids), generate_unpacked_objects(
1096 self,
1097 object_ids,
1098 progress=progress,
1099 ofs_delta=ofs_delta,
1100 other_haves=remote_has,
1101 )
1103 def _clear_cached_packs(self) -> None:
1104 pack_cache = self._pack_cache
1105 self._pack_cache = {}
1106 self._pack_access_order = []
1107 while pack_cache:
1108 (_name, pack) = pack_cache.popitem()
1109 pack.close()
1111 def _total_pack_mmap_size(self) -> int:
1112 """Return the total mmapped memory across all cached packs."""
1113 return sum(pack.mmap_size for pack in self._pack_cache.values())
1115 def _mark_pack_used(self, pack_hash: str) -> None:
1116 """Mark a pack as recently used for LRU tracking."""
1117 try:
1118 self._pack_access_order.remove(pack_hash)
1119 except ValueError:
1120 pass
1121 self._pack_access_order.append(pack_hash)
1123 def _enforce_packed_git_limit(self) -> None:
1124 """Evict least-recently-used packs if the memory limit is exceeded."""
1125 if self.packed_git_limit is None:
1126 return
1127 while (
1128 self._pack_access_order
1129 and self._total_pack_mmap_size() > self.packed_git_limit
1130 ):
1131 oldest = self._pack_access_order.pop(0)
1132 pack = self._pack_cache.get(oldest)
1133 if pack is not None:
1134 pack.close()
1135 del self._pack_cache[oldest]
1137 def _iter_cached_packs(self) -> Iterator[Pack]:
1138 return iter(list(self._pack_cache.values()))
1140 def _evict_pack(self, pack: "Pack | FilePackIndex") -> None:
1141 """Evict a pack from the cache after its backing file disappeared.
1143 ``pack`` may be a :class:`Pack` or a :class:`FilePackIndex`; in the
1144 latter case the index's owning ``Pack`` is matched via the cached
1145 pack's ``_idx`` reference.
1146 """
1147 for key, cached in list(self._pack_cache.items()):
1148 if cached is pack or cached._idx is pack:
1149 del self._pack_cache[key]
1150 try:
1151 self._pack_access_order.remove(key)
1152 except ValueError:
1153 pass
1154 try:
1155 cached.close()
1156 except OSError:
1157 pass
1158 break
1160 def _lookup_in_packs(self, lookup: "Callable[[Pack], _T]") -> "_T":
1161 """Run ``lookup(pack)`` against each cached pack and return the first hit.
1163 ``lookup`` should raise ``KeyError`` if the pack does not contain the
1164 target. ``PackFileDisappeared`` from a concurrent ``git repack`` /
1165 ``gc --auto`` is caught: the stale pack is evicted, the pack
1166 directory is rescanned, and the search retries — bounded, mirroring
1167 git's ``reprepare_packed_git()``. If no cached pack has the object
1168 the pack directory is rescanned once to pick up any newly-arrived
1169 packs (e.g. another writer just landed one). ``KeyError`` is raised
1170 if no pack — old or new — has the object.
1171 """
1172 rescanned = False
1173 for _attempt in range(_MAX_PACK_RESCAN_ATTEMPTS):
1174 disappeared = False
1175 for pack_hash, pack in list(self._pack_cache.items()):
1176 try:
1177 result = lookup(pack)
1178 except KeyError:
1179 continue
1180 except PackFileDisappeared as exc:
1181 self._evict_pack(exc.obj)
1182 disappeared = True
1183 continue
1184 self._mark_pack_used(pack_hash)
1185 self._enforce_packed_git_limit()
1186 return result
1187 if disappeared:
1188 self._update_pack_cache()
1189 rescanned = True
1190 continue
1191 if not rescanned:
1192 # Maybe another process just landed a pack with the object.
1193 if self._update_pack_cache():
1194 rescanned = True
1195 continue
1196 break
1197 raise KeyError
1199 def _update_pack_cache(self) -> list[Pack]:
1200 raise NotImplementedError(self._update_pack_cache)
1202 def close(self) -> None:
1203 """Close the object store and release resources.
1205 This method closes all cached pack files and frees associated resources.
1206 Can be called multiple times safely.
1207 """
1208 self._clear_cached_packs()
1210 def __del__(self) -> None:
1211 """Warn if the object store is being deleted with unclosed packs."""
1212 if self._pack_cache:
1213 import warnings
1215 warnings.warn(
1216 f"ObjectStore {self!r} was destroyed with {len(self._pack_cache)} "
1217 "unclosed pack(s). Please call close() explicitly.",
1218 ResourceWarning,
1219 stacklevel=2,
1220 )
1221 self.close()
1223 @property
1224 def packs(self) -> list[Pack]:
1225 """List with pack objects."""
1226 return list(self._iter_cached_packs()) + list(self._update_pack_cache())
1228 def count_pack_files(self) -> int:
1229 """Count the number of pack files.
1231 Returns:
1232 Number of pack files (excluding those with .keep files)
1233 """
1234 count = 0
1235 for pack in self.packs:
1236 # Check if there's a .keep file for this pack
1237 keep_path = pack._basename + ".keep"
1238 if not os.path.exists(keep_path):
1239 count += 1
1240 return count
1242 def _iter_alternate_objects(self) -> Iterator[ObjectID]:
1243 """Iterate over the SHAs of all the objects in alternate stores."""
1244 for alternate in self.alternates:
1245 yield from alternate
1247 def _iter_loose_objects(self) -> Iterator[ObjectID]:
1248 """Iterate over the SHAs of all loose objects."""
1249 raise NotImplementedError(self._iter_loose_objects)
1251 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
1252 raise NotImplementedError(self._get_loose_object)
1254 def delete_loose_object(self, sha: ObjectID) -> None:
1255 """Delete a loose object.
1257 This method only handles loose objects. For packed objects,
1258 use repack(exclude=...) to exclude them during repacking.
1259 """
1260 raise NotImplementedError(self.delete_loose_object)
1262 def _remove_pack(self, pack: "Pack") -> None:
1263 raise NotImplementedError(self._remove_pack)
1265 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
1266 """Pack loose objects.
1268 Args:
1269 progress: Optional progress reporting callback
1271 Returns: Number of objects packed
1272 """
1273 objects: list[tuple[ShaFile, None]] = []
1274 for sha in self._iter_loose_objects():
1275 obj = self._get_loose_object(sha)
1276 if obj is not None:
1277 objects.append((obj, None))
1278 self.add_objects(objects, progress=progress)
1279 for obj, path in objects:
1280 self.delete_loose_object(obj.id)
1281 return len(objects)
1283 def repack(
1284 self,
1285 exclude: Set[bytes] | None = None,
1286 progress: Callable[[str], None] | None = None,
1287 ) -> int:
1288 """Repack the packs in this repository.
1290 Note that this implementation is fairly naive and currently keeps all
1291 objects in memory while it repacks.
1293 Args:
1294 exclude: Optional set of object SHAs to exclude from repacking
1295 progress: Optional progress reporting callback
1296 """
1297 if exclude is None:
1298 exclude = set()
1300 loose_objects = set()
1301 excluded_loose_objects = set()
1302 for sha in self._iter_loose_objects():
1303 if sha not in exclude:
1304 obj = self._get_loose_object(sha)
1305 if obj is not None:
1306 loose_objects.add(obj)
1307 else:
1308 excluded_loose_objects.add(sha)
1310 objects: set[tuple[ShaFile, None]] = {(obj, None) for obj in loose_objects}
1311 old_packs = {p.name(): p for p in self.packs}
1312 for name, pack in old_packs.items():
1313 objects.update(
1314 (obj, None) for obj in pack.iterobjects() if obj.id not in exclude
1315 )
1317 # Only create a new pack if there are objects to pack
1318 if objects:
1319 # The name of the consolidated pack might match the name of a
1320 # pre-existing pack. Take care not to remove the newly created
1321 # consolidated pack.
1322 consolidated = self.add_objects(list(objects), progress=progress)
1323 if consolidated is not None:
1324 old_packs.pop(consolidated.name(), None)
1326 # Delete loose objects that were packed
1327 for obj in loose_objects:
1328 if obj is not None:
1329 self.delete_loose_object(obj.id)
1330 # Delete excluded loose objects
1331 for sha in excluded_loose_objects:
1332 self.delete_loose_object(sha)
1333 for name, pack in old_packs.items():
1334 self._remove_pack(pack)
1335 self._update_pack_cache()
1336 return len(objects)
1338 def generate_pack_bitmaps(
1339 self,
1340 refs: dict[Ref, ObjectID],
1341 *,
1342 commit_interval: int | None = None,
1343 progress: Callable[[str], None] | None = None,
1344 ) -> int:
1345 """Generate bitmap indexes for all packs that don't have them.
1347 This generates .bitmap files for packfiles, enabling fast reachability
1348 queries. Equivalent to the bitmap generation part of 'git repack -b'.
1350 Args:
1351 refs: Dictionary of ref names to commit SHAs
1352 commit_interval: Include every Nth commit in bitmap index (None for default)
1353 progress: Optional progress reporting callback
1355 Returns:
1356 Number of bitmaps generated
1357 """
1358 count = 0
1359 for pack in self.packs:
1360 pack.ensure_bitmap(
1361 self, refs, commit_interval=commit_interval, progress=progress
1362 )
1363 count += 1
1365 # Update cache to pick up new bitmaps
1366 self._update_pack_cache()
1368 return count
1370 def __iter__(self) -> Iterator[ObjectID]:
1371 """Iterate over the SHAs that are present in this store."""
1372 self._update_pack_cache()
1373 for pack in self._iter_cached_packs():
1374 try:
1375 yield from pack
1376 except PackFileDisappeared as exc:
1377 self._evict_pack(exc.obj)
1378 yield from self._iter_loose_objects()
1379 yield from self._iter_alternate_objects()
1381 def contains_loose(self, sha: ObjectID) -> bool:
1382 """Check if a particular object is present by SHA1 and is loose.
1384 This does not check alternates.
1385 """
1386 return self._get_loose_object(sha) is not None
1388 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
1389 """Obtain the raw fulltext for an object.
1391 Args:
1392 name: sha for the object.
1393 Returns: tuple with numeric type and object contents.
1394 """
1395 sha: RawObjectID
1396 hexsha: ObjectID | None
1397 if len(name) == self.object_format.hex_length:
1398 sha = hex_to_sha(ObjectID(name))
1399 hexsha = cast(ObjectID, name)
1400 elif len(name) == self.object_format.oid_length:
1401 sha = RawObjectID(name)
1402 hexsha = None
1403 else:
1404 raise AssertionError(f"Invalid object name {name!r}")
1405 try:
1406 return self._lookup_in_packs(lambda p: p.get_raw(sha))
1407 except KeyError:
1408 pass
1409 if hexsha is None:
1410 hexsha = sha_to_hex(sha)
1411 ret = self._get_loose_object(hexsha)
1412 if ret is not None:
1413 return ret.type_num, ret.as_raw_string()
1414 for alternate in self.alternates:
1415 try:
1416 return alternate.get_raw(hexsha)
1417 except KeyError:
1418 pass
1419 raise KeyError(hexsha)
1421 def iter_unpacked_subset(
1422 self,
1423 shas: Iterable[ObjectID | RawObjectID],
1424 *,
1425 include_comp: bool = False,
1426 allow_missing: bool = False,
1427 convert_ofs_delta: bool = True,
1428 ) -> Iterator[UnpackedObject]:
1429 """Iterate over a subset of objects, yielding UnpackedObject instances.
1431 Args:
1432 shas: Set of object SHAs to retrieve
1433 include_comp: Whether to include compressed data
1434 allow_missing: If True, skip missing objects; if False, raise KeyError
1435 convert_ofs_delta: Whether to convert OFS_DELTA objects
1437 Returns:
1438 Iterator of UnpackedObject instances
1440 Raises:
1441 KeyError: If an object is missing and allow_missing is False
1442 """
1443 todo: set[ObjectID | RawObjectID] = set(shas)
1444 for p in self._iter_cached_packs():
1445 try:
1446 for unpacked in p.iter_unpacked_subset(
1447 todo,
1448 include_comp=include_comp,
1449 allow_missing=True,
1450 convert_ofs_delta=convert_ofs_delta,
1451 ):
1452 yield unpacked
1453 hexsha = sha_to_hex(unpacked.sha())
1454 todo.remove(hexsha)
1455 except PackFileDisappeared as exc:
1456 self._evict_pack(exc.obj)
1457 # Maybe something else has added a pack with the object
1458 # in the mean time?
1459 for p in self._update_pack_cache():
1460 try:
1461 for unpacked in p.iter_unpacked_subset(
1462 todo,
1463 include_comp=include_comp,
1464 allow_missing=True,
1465 convert_ofs_delta=convert_ofs_delta,
1466 ):
1467 yield unpacked
1468 hexsha = sha_to_hex(unpacked.sha())
1469 todo.remove(hexsha)
1470 except PackFileDisappeared as exc:
1471 self._evict_pack(exc.obj)
1472 for alternate in self.alternates:
1473 assert isinstance(alternate, PackBasedObjectStore)
1474 for unpacked in alternate.iter_unpacked_subset(
1475 todo,
1476 include_comp=include_comp,
1477 allow_missing=True,
1478 convert_ofs_delta=convert_ofs_delta,
1479 ):
1480 yield unpacked
1481 hexsha = sha_to_hex(unpacked.sha())
1482 todo.remove(hexsha)
1484 def iterobjects_subset(
1485 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
1486 ) -> Iterator[ShaFile]:
1487 """Iterate over a subset of objects in the store.
1489 This method searches for objects in pack files, alternates, and loose storage.
1491 Args:
1492 shas: Iterable of object SHAs to retrieve
1493 allow_missing: If True, skip missing objects; if False, raise KeyError
1495 Returns:
1496 Iterator of ShaFile objects
1498 Raises:
1499 KeyError: If an object is missing and allow_missing is False
1500 """
1501 todo: set[ObjectID] = set(shas)
1502 for p in self._iter_cached_packs():
1503 try:
1504 for o in p.iterobjects_subset(todo, allow_missing=True):
1505 yield o
1506 todo.remove(o.id)
1507 except PackFileDisappeared as exc:
1508 self._evict_pack(exc.obj)
1509 # Maybe something else has added a pack with the object
1510 # in the mean time?
1511 for p in self._update_pack_cache():
1512 try:
1513 for o in p.iterobjects_subset(todo, allow_missing=True):
1514 yield o
1515 todo.remove(o.id)
1516 except PackFileDisappeared as exc:
1517 self._evict_pack(exc.obj)
1518 for alternate in self.alternates:
1519 for o in alternate.iterobjects_subset(todo, allow_missing=True):
1520 yield o
1521 todo.remove(o.id)
1522 for oid in todo:
1523 loose_obj: ShaFile | None = self._get_loose_object(oid)
1524 if loose_obj is not None:
1525 yield loose_obj
1526 elif not allow_missing:
1527 raise KeyError(oid)
1529 def get_unpacked_object(
1530 self, sha1: bytes, *, include_comp: bool = False
1531 ) -> UnpackedObject:
1532 """Obtain the unpacked object.
1534 Args:
1535 sha1: sha for the object.
1536 include_comp: Whether to include compression metadata.
1537 """
1538 if len(sha1) == self.object_format.hex_length:
1539 sha = hex_to_sha(cast(ObjectID, sha1))
1540 hexsha = cast(ObjectID, sha1)
1541 elif len(sha1) == self.object_format.oid_length:
1542 sha = cast(RawObjectID, sha1)
1543 hexsha = None
1544 else:
1545 raise AssertionError(f"Invalid object sha1 {sha1!r}")
1546 try:
1547 return self._lookup_in_packs(
1548 lambda p: p.get_unpacked_object(sha, include_comp=include_comp)
1549 )
1550 except KeyError:
1551 pass
1552 if hexsha is None:
1553 hexsha = sha_to_hex(sha)
1554 for alternate in self.alternates:
1555 assert isinstance(alternate, PackBasedObjectStore)
1556 try:
1557 return alternate.get_unpacked_object(hexsha, include_comp=include_comp)
1558 except KeyError:
1559 pass
1560 raise KeyError(hexsha)
1562 def add_objects(
1563 self,
1564 objects: Sequence[tuple[ShaFile, str | None]],
1565 progress: Callable[[str], None] | None = None,
1566 ) -> "Pack | None":
1567 """Add a set of objects to this object store.
1569 Args:
1570 objects: Iterable over (object, path) tuples, should support
1571 __len__.
1572 progress: Optional progress reporting function.
1573 Returns: Pack object of the objects written.
1574 """
1575 count = len(objects)
1576 record_iter = (full_unpacked_object(o) for (o, p) in objects)
1577 return self.add_pack_data(count, record_iter, progress=progress)
1580class DiskObjectStore(PackBasedObjectStore):
1581 """Git-style object store that exists on disk."""
1583 path: str | os.PathLike[str]
1584 pack_dir: str | os.PathLike[str]
1585 _alternates: "list[BaseObjectStore] | None"
1586 _commit_graph: "CommitGraph | None"
1588 def __init__(
1589 self,
1590 path: str | os.PathLike[str],
1591 *,
1592 loose_compression_level: int = -1,
1593 pack_compression_level: int = -1,
1594 pack_index_version: int | None = None,
1595 pack_delta_window_size: int | None = None,
1596 pack_window_memory: int | None = None,
1597 pack_delta_cache_size: int | None = None,
1598 pack_depth: int | None = None,
1599 pack_threads: int | None = None,
1600 pack_big_file_threshold: int | None = None,
1601 packed_git_limit: int | None = None,
1602 delta_base_cache_limit: int | None = None,
1603 fsync_object_files: bool = False,
1604 pack_write_bitmaps: bool = False,
1605 pack_write_bitmap_hash_cache: bool = True,
1606 pack_write_bitmap_lookup_table: bool = True,
1607 file_mode: int | None = None,
1608 dir_mode: int | None = None,
1609 object_format: "ObjectFormat | None" = None,
1610 loose_object_size_limit: int | None = None,
1611 ) -> None:
1612 """Open an object store.
1614 Args:
1615 path: Path of the object store.
1616 loose_compression_level: zlib compression level for loose objects
1617 pack_compression_level: zlib compression level for pack objects
1618 pack_index_version: pack index version to use (1, 2, or 3)
1619 pack_delta_window_size: sliding window size for delta compression
1620 pack_window_memory: memory limit for delta window operations
1621 pack_delta_cache_size: size of cache for delta operations
1622 pack_depth: maximum delta chain depth
1623 pack_threads: number of threads for pack operations
1624 pack_big_file_threshold: threshold for treating files as big
1625 packed_git_limit: maximum total bytes for mmapped pack files
1626 delta_base_cache_limit: maximum bytes for delta base object cache
1627 fsync_object_files: whether to fsync object files for durability
1628 pack_write_bitmaps: whether to write bitmap indexes for packs
1629 pack_write_bitmap_hash_cache: whether to include name-hash cache in bitmaps
1630 pack_write_bitmap_lookup_table: whether to include lookup table in bitmaps
1631 file_mode: File permission mask for shared repository
1632 dir_mode: Directory permission mask for shared repository
1633 object_format: Hash algorithm to use (SHA1 or SHA256)
1634 loose_object_size_limit: Maximum inflated size of a single loose
1635 object. Defaults to core.bigFileThreshold's Git default (512 MiB)
1636 via :data:`DEFAULT_LOOSE_OBJECT_SIZE_LIMIT` when None. Guards against
1637 decompression-bomb attacks.
1638 """
1639 # Import here to avoid circular dependency
1640 from .object_format import DEFAULT_OBJECT_FORMAT
1642 super().__init__(
1643 pack_compression_level=pack_compression_level,
1644 pack_index_version=pack_index_version,
1645 pack_delta_window_size=pack_delta_window_size,
1646 pack_window_memory=pack_window_memory,
1647 pack_delta_cache_size=pack_delta_cache_size,
1648 pack_depth=pack_depth,
1649 pack_threads=pack_threads,
1650 pack_big_file_threshold=pack_big_file_threshold,
1651 packed_git_limit=packed_git_limit,
1652 delta_base_cache_limit=delta_base_cache_limit,
1653 object_format=object_format if object_format else DEFAULT_OBJECT_FORMAT,
1654 )
1655 self.path = path
1656 self.pack_dir = os.path.join(self.path, PACKDIR)
1657 self._alternates = None
1658 self.loose_compression_level = loose_compression_level
1659 self.pack_compression_level = pack_compression_level
1660 self.pack_index_version = pack_index_version
1661 self.fsync_object_files = fsync_object_files
1662 self.pack_write_bitmaps = pack_write_bitmaps
1663 self.pack_write_bitmap_hash_cache = pack_write_bitmap_hash_cache
1664 self.pack_write_bitmap_lookup_table = pack_write_bitmap_lookup_table
1665 self.file_mode = file_mode
1666 self.dir_mode = dir_mode
1667 self.loose_object_size_limit = (
1668 loose_object_size_limit
1669 if loose_object_size_limit is not None
1670 else DEFAULT_LOOSE_OBJECT_SIZE_LIMIT
1671 )
1673 # Commit graph support - lazy loaded
1674 self._commit_graph = None
1675 self._use_commit_graph = True # Default to true
1677 # Multi-pack-index support - lazy loaded
1678 self._midx: MultiPackIndex | None = None
1679 self._use_midx = True # Default to true
1681 def __repr__(self) -> str:
1682 """Return string representation of DiskObjectStore.
1684 Returns:
1685 String representation including the store path
1686 """
1687 return f"<{self.__class__.__name__}({self.path!r})>"
1689 @classmethod
1690 def from_config(
1691 cls,
1692 path: str | os.PathLike[str],
1693 config: "Config",
1694 *,
1695 file_mode: int | None = None,
1696 dir_mode: int | None = None,
1697 ) -> "DiskObjectStore":
1698 """Create a DiskObjectStore from a configuration object.
1700 Args:
1701 path: Path to the object store directory
1702 config: Configuration object to read settings from
1703 file_mode: Optional file permission mask for shared repository
1704 dir_mode: Optional directory permission mask for shared repository
1706 Returns:
1707 New DiskObjectStore instance configured according to config
1708 """
1709 try:
1710 default_compression_level = int(
1711 config.get((b"core",), b"compression").decode()
1712 )
1713 except KeyError:
1714 default_compression_level = -1
1715 try:
1716 loose_compression_level = int(
1717 config.get((b"core",), b"looseCompression").decode()
1718 )
1719 except KeyError:
1720 loose_compression_level = default_compression_level
1721 try:
1722 pack_compression_level = int(
1723 config.get((b"core",), "packCompression").decode()
1724 )
1725 except KeyError:
1726 pack_compression_level = default_compression_level
1727 try:
1728 pack_index_version = int(config.get((b"pack",), b"indexVersion").decode())
1729 except KeyError:
1730 pack_index_version = None
1732 # Read pack configuration options
1733 try:
1734 pack_delta_window_size = int(
1735 config.get((b"pack",), b"deltaWindowSize").decode()
1736 )
1737 except KeyError:
1738 pack_delta_window_size = None
1739 try:
1740 pack_window_memory = int(config.get((b"pack",), b"windowMemory").decode())
1741 except KeyError:
1742 pack_window_memory = None
1743 try:
1744 pack_delta_cache_size = int(
1745 config.get((b"pack",), b"deltaCacheSize").decode()
1746 )
1747 except KeyError:
1748 pack_delta_cache_size = None
1749 try:
1750 pack_depth = int(config.get((b"pack",), b"depth").decode())
1751 except KeyError:
1752 pack_depth = None
1753 try:
1754 pack_threads = int(config.get((b"pack",), b"threads").decode())
1755 except KeyError:
1756 pack_threads = None
1757 try:
1758 pack_big_file_threshold = int(
1759 config.get((b"pack",), b"bigFileThreshold").decode()
1760 )
1761 except KeyError:
1762 pack_big_file_threshold = None
1764 # Read core.packedGitLimit setting
1765 try:
1766 packed_git_limit = int(config.get((b"core",), b"packedGitLimit").decode())
1767 except KeyError:
1768 packed_git_limit = None
1770 # Read core.deltaBaseCacheLimit setting
1771 try:
1772 delta_base_cache_limit = int(
1773 config.get((b"core",), b"deltaBaseCacheLimit").decode()
1774 )
1775 except KeyError:
1776 delta_base_cache_limit = None
1778 # Read core.bigFileThreshold setting; used as the upper bound for
1779 # inflating a single loose object, guarding against decompression bombs.
1780 try:
1781 loose_object_size_limit: int | None = int(
1782 config.get((b"core",), b"bigFileThreshold").decode()
1783 )
1784 except KeyError:
1785 loose_object_size_limit = None
1787 # Read core.commitGraph setting
1788 use_commit_graph = config.get_boolean((b"core",), b"commitGraph", True)
1790 # Read core.multiPackIndex setting
1791 use_midx = config.get_boolean((b"core",), b"multiPackIndex", True)
1793 # Read core.fsyncObjectFiles setting
1794 fsync_object_files = config.get_boolean((b"core",), b"fsyncObjectFiles", False)
1796 # Read bitmap settings
1797 pack_write_bitmaps = config.get_boolean((b"pack",), b"writeBitmaps", False)
1798 pack_write_bitmap_hash_cache = config.get_boolean(
1799 (b"pack",), b"writeBitmapHashCache", True
1800 )
1801 pack_write_bitmap_lookup_table = config.get_boolean(
1802 (b"pack",), b"writeBitmapLookupTable", True
1803 )
1804 # Also check repack.writeBitmaps for backwards compatibility
1805 if not pack_write_bitmaps:
1806 pack_write_bitmaps = config.get_boolean(
1807 (b"repack",), b"writeBitmaps", False
1808 )
1810 # Get hash algorithm from config
1811 from .object_format import get_object_format
1813 object_format = None
1814 try:
1815 try:
1816 version = int(config.get((b"core",), b"repositoryformatversion"))
1817 except KeyError:
1818 version = 0
1819 if version == 1:
1820 try:
1821 object_format_name = config.get((b"extensions",), b"objectformat")
1822 except KeyError:
1823 object_format_name = b"sha1"
1824 object_format = get_object_format(object_format_name.decode("ascii"))
1825 except (KeyError, ValueError):
1826 pass
1828 instance = cls(
1829 path,
1830 loose_compression_level=loose_compression_level,
1831 pack_compression_level=pack_compression_level,
1832 pack_index_version=pack_index_version,
1833 pack_delta_window_size=pack_delta_window_size,
1834 pack_window_memory=pack_window_memory,
1835 pack_delta_cache_size=pack_delta_cache_size,
1836 pack_depth=pack_depth,
1837 pack_threads=pack_threads,
1838 pack_big_file_threshold=pack_big_file_threshold,
1839 packed_git_limit=packed_git_limit,
1840 delta_base_cache_limit=delta_base_cache_limit,
1841 fsync_object_files=fsync_object_files,
1842 pack_write_bitmaps=pack_write_bitmaps,
1843 pack_write_bitmap_hash_cache=pack_write_bitmap_hash_cache,
1844 pack_write_bitmap_lookup_table=pack_write_bitmap_lookup_table,
1845 file_mode=file_mode,
1846 dir_mode=dir_mode,
1847 object_format=object_format,
1848 loose_object_size_limit=loose_object_size_limit,
1849 )
1850 instance._use_commit_graph = use_commit_graph
1851 instance._use_midx = use_midx
1852 return instance
1854 @property
1855 def alternates(self) -> list["BaseObjectStore"]:
1856 """Get the list of alternate object stores.
1858 Reads from .git/objects/info/alternates if not already cached.
1860 Returns:
1861 List of DiskObjectStore instances for alternate object directories
1862 """
1863 if self._alternates is not None:
1864 return self._alternates
1865 self._alternates = []
1866 for path in self._read_alternate_paths():
1867 self._alternates.append(DiskObjectStore(path))
1868 return self._alternates
1870 def _read_alternate_paths(self) -> Iterator[str]:
1871 try:
1872 f = GitFile(os.path.join(self.path, INFODIR, "alternates"), "rb")
1873 except FileNotFoundError:
1874 return
1875 with f:
1876 for line in f.readlines():
1877 line = line.rstrip(b"\n")
1878 if line.startswith(b"#"):
1879 continue
1880 if os.path.isabs(line):
1881 yield os.fsdecode(line)
1882 else:
1883 yield os.fsdecode(os.path.join(os.fsencode(self.path), line))
1885 def add_alternate_path(self, path: str | os.PathLike[str]) -> None:
1886 """Add an alternate path to this object store."""
1887 info_dir = os.path.join(self.path, INFODIR)
1888 try:
1889 os.mkdir(info_dir)
1890 if self.dir_mode is not None:
1891 os.chmod(info_dir, self.dir_mode)
1892 except FileExistsError:
1893 pass
1894 alternates_path = os.path.join(self.path, INFODIR, "alternates")
1895 mask = self.file_mode if self.file_mode is not None else 0o644
1896 with GitFile(alternates_path, "wb", mask=mask) as f:
1897 try:
1898 orig_f = open(alternates_path, "rb")
1899 except FileNotFoundError:
1900 pass
1901 else:
1902 with orig_f:
1903 f.write(orig_f.read())
1904 f.write(os.fsencode(path) + b"\n")
1906 if not os.path.isabs(path):
1907 path = os.path.join(self.path, path)
1908 self.alternates.append(DiskObjectStore(path))
1910 def _update_pack_cache(self) -> list[Pack]:
1911 """Read and iterate over new pack files and cache them."""
1912 try:
1913 pack_dir_contents = set(os.listdir(self.pack_dir))
1914 except FileNotFoundError:
1915 return []
1916 pack_files = set()
1917 for name in pack_dir_contents:
1918 # Index any ".pack" file with a matching ".idx", not just
1919 # "pack-<hash>". ``git maintenance`` writes packs named
1920 # "loose-<hash>.pack"; these are ordinary packs and Git indexes
1921 # any .pack file present. The matching ".idx" also confirms the
1922 # pack is fully written.
1923 if name.endswith(".pack"):
1924 basename = name[: -len(".pack")]
1925 if basename + ".idx" in pack_dir_contents:
1926 pack_files.add(basename)
1928 # Open newly appeared pack files
1929 new_packs = []
1930 for basename in pack_files:
1931 if basename not in self._pack_cache:
1932 pack = Pack(
1933 os.path.join(self.pack_dir, basename),
1934 object_format=self.object_format,
1935 delta_window_size=self.pack_delta_window_size,
1936 window_memory=self.pack_window_memory,
1937 delta_cache_size=self.pack_delta_cache_size,
1938 depth=self.pack_depth,
1939 threads=self.pack_threads,
1940 big_file_threshold=self.pack_big_file_threshold,
1941 delta_base_cache_limit=self.delta_base_cache_limit,
1942 )
1943 new_packs.append(pack)
1944 self._pack_cache[basename] = pack
1945 self._mark_pack_used(basename)
1946 # Remove disappeared pack files
1947 for f in set(self._pack_cache) - pack_files:
1948 self._pack_cache.pop(f).close()
1949 try:
1950 self._pack_access_order.remove(f)
1951 except ValueError:
1952 pass
1953 self._enforce_packed_git_limit()
1954 return new_packs
1956 def _get_shafile_path(self, sha: ObjectID) -> str:
1957 # Reject anything that is neither a valid hex object id nor a raw
1958 # binary oid before building a path. Otherwise a malformed id (e.g.
1959 # one containing path separators) would be joined into a filename
1960 # that escapes the objects directory.
1961 if not valid_hexsha(sha):
1962 raise ValueError(f"Invalid object id {sha!r}")
1963 # Check from object dir
1964 return hex_to_filename(os.fspath(self.path), sha)
1966 def _iter_loose_objects(self) -> Iterator[ObjectID]:
1967 for base in os.listdir(self.path):
1968 if len(base) != 2:
1969 continue
1970 for rest in os.listdir(os.path.join(self.path, base)):
1971 sha = os.fsencode(base + rest)
1972 if not valid_hexsha(sha):
1973 continue
1974 yield ObjectID(sha)
1976 def count_loose_objects(self) -> int:
1977 """Count the number of loose objects in the object store.
1979 Returns:
1980 Number of loose objects
1981 """
1982 # Calculate expected filename length for loose
1983 # objects (excluding directory)
1984 fn_length = self.object_format.hex_length - 2
1985 count = 0
1986 if not os.path.exists(self.path):
1987 return 0
1989 for i in range(256):
1990 subdir = os.path.join(self.path, f"{i:02x}")
1991 try:
1992 count += len(
1993 [name for name in os.listdir(subdir) if len(name) == fn_length]
1994 )
1995 except FileNotFoundError:
1996 # Directory may have been removed or is inaccessible
1997 continue
1999 return count
2001 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
2002 try:
2003 # Load the object from path with SHA and hash algorithm from object store
2004 # Convert to hex ObjectID if needed
2005 if len(sha) == self.object_format.oid_length:
2006 hex_sha: ObjectID = sha_to_hex(RawObjectID(sha))
2007 else:
2008 hex_sha = ObjectID(sha)
2009 path = self._get_shafile_path(hex_sha)
2010 return ShaFile.from_path(
2011 path,
2012 hex_sha,
2013 object_format=self.object_format,
2014 max_size=self.loose_object_size_limit,
2015 )
2016 except FileNotFoundError:
2017 return None
2019 def delete_loose_object(self, sha: ObjectID) -> None:
2020 """Delete a loose object from disk.
2022 Args:
2023 sha: SHA1 of the object to delete
2025 Raises:
2026 FileNotFoundError: If the object file doesn't exist
2027 """
2028 _remove_readonly(self._get_shafile_path(sha))
2030 def get_object_mtime(self, sha: ObjectID) -> float:
2031 """Get the modification time of an object.
2033 Args:
2034 sha: SHA1 of the object
2036 Returns:
2037 Modification time as seconds since epoch
2039 Raises:
2040 KeyError: if the object is not found
2041 """
2042 # First check if it's a loose object
2043 if self.contains_loose(sha):
2044 path = self._get_shafile_path(sha)
2045 try:
2046 return os.path.getmtime(path)
2047 except FileNotFoundError:
2048 pass
2050 # Check if it's in a pack file
2051 for pack in self.packs:
2052 try:
2053 if sha in pack:
2054 # Use the pack file's mtime for packed objects
2055 pack_path = pack._data_path
2056 try:
2057 return os.path.getmtime(pack_path)
2058 except (FileNotFoundError, AttributeError):
2059 pass
2060 except PackFileDisappeared:
2061 pass
2063 raise KeyError(sha)
2065 def _remove_pack(self, pack: Pack) -> None:
2066 # _pack_cache is keyed by the full pack basename (e.g. "pack-<hash>"
2067 # or "loose-<hash>"), matching pack._basename.
2068 basename = os.path.basename(pack._basename)
2069 self._pack_cache.pop(basename, None)
2070 try:
2071 self._pack_access_order.remove(basename)
2072 except ValueError:
2073 pass
2074 # Store paths before closing to avoid re-opening files on Windows
2075 data_path = pack._data_path
2076 idx_path = pack._idx_path
2077 pack.close()
2078 _remove_readonly(data_path)
2079 if os.path.exists(idx_path):
2080 _remove_readonly(idx_path)
2082 def _get_pack_basepath(
2083 self, entries: Iterable[tuple[bytes, int, int | None]]
2084 ) -> str:
2085 suffix_bytes = iter_sha1(entry[0] for entry in entries)
2086 # TODO: Handle self.pack_dir being bytes
2087 suffix = suffix_bytes.decode("ascii")
2088 return os.path.join(self.pack_dir, "pack-" + suffix)
2090 def _complete_pack(
2091 self,
2092 f: BinaryIO,
2093 path: str,
2094 num_objects: int,
2095 indexer: PackIndexer,
2096 progress: Callable[..., None] | None = None,
2097 refs: dict[Ref, ObjectID] | None = None,
2098 ) -> Pack:
2099 """Move a specific file containing a pack into the pack directory.
2101 Note: The file should be on the same file system as the
2102 packs directory.
2104 Args:
2105 f: Open file object for the pack.
2106 path: Path to the pack file.
2107 num_objects: Number of objects in the pack.
2108 indexer: A PackIndexer for indexing the pack.
2109 progress: Optional progress reporting function.
2110 refs: Optional dictionary of refs for bitmap generation.
2111 """
2112 entries = []
2113 for i, entry in enumerate(indexer):
2114 if progress is not None:
2115 progress(f"generating index: {i}/{num_objects}\r".encode("ascii"))
2116 entries.append(entry)
2118 pack_sha, extra_entries = extend_pack(
2119 f,
2120 set(indexer.ext_refs()),
2121 get_raw=self.get_raw,
2122 compression_level=self.pack_compression_level,
2123 progress=progress,
2124 object_format=self.object_format,
2125 )
2126 f.flush()
2127 if self.fsync_object_files:
2128 try:
2129 fileno = f.fileno()
2130 except AttributeError as e:
2131 raise OSError("fsync requested but file has no fileno()") from e
2132 else:
2133 os.fsync(fileno)
2134 f.close()
2136 entries.extend(extra_entries)
2138 # Move the pack in.
2139 entries.sort()
2140 pack_base_name = self._get_pack_basepath(entries)
2142 # A pack's identity is the SHA over its object SHAs, which is the
2143 # "<hash>" suffix _get_pack_basepath builds the name from. Compare by
2144 # that rather than by basename so an existing pack holding the same
2145 # objects under a different prefix (e.g. a "loose-<hash>" pack written
2146 # by git maintenance) is recognised and not duplicated.
2147 pack_name = os.path.basename(pack_base_name)[len("pack-") :].encode("ascii")
2148 for pack in self.packs:
2149 if pack.name() == pack_name:
2150 # The objects are already packed; drop the temporary pack we
2151 # were about to move in rather than leaking it into pack_dir.
2152 _remove_readonly(path)
2153 return pack
2155 target_pack_path = pack_base_name + ".pack"
2156 target_index_path = pack_base_name + ".idx"
2157 if sys.platform == "win32":
2158 # Windows might have the target pack file lingering. Attempt
2159 # removal, silently passing if the target does not exist.
2160 with suppress(FileNotFoundError):
2161 os.remove(target_pack_path)
2162 os.rename(path, target_pack_path)
2164 # Write the index.
2165 mask = self.file_mode if self.file_mode is not None else PACK_MODE
2166 with GitFile(
2167 target_index_path,
2168 "wb",
2169 mask=mask,
2170 fsync=self.fsync_object_files,
2171 ) as index_file:
2172 write_pack_index(
2173 index_file, entries, pack_sha, version=self.pack_index_version
2174 )
2176 # Generate bitmap if configured and refs are available
2177 if self.pack_write_bitmaps and refs:
2178 from .bitmap import generate_bitmap, write_bitmap
2179 from .pack import load_pack_index
2181 if progress:
2182 progress("Generating bitmap index\r".encode("ascii"))
2184 # Load the index we just wrote. load_pack_index keeps the file
2185 # open for the lifetime of the index (it mmaps it), so close it
2186 # once the bitmap is generated rather than leaving the .idx
2187 # mapped, which would lock it on Windows.
2188 with closing(
2189 load_pack_index(target_index_path, self.object_format)
2190 ) as pack_index:
2191 bitmap = generate_bitmap(
2192 pack_index=pack_index,
2193 object_store=self,
2194 refs=refs,
2195 pack_checksum=pack_sha,
2196 include_hash_cache=self.pack_write_bitmap_hash_cache,
2197 include_lookup_table=self.pack_write_bitmap_lookup_table,
2198 progress=lambda msg: (
2199 progress(msg.encode("ascii"))
2200 if progress and isinstance(msg, str)
2201 else None
2202 ),
2203 )
2205 # Write the bitmap
2206 target_bitmap_path = pack_base_name + ".bitmap"
2207 write_bitmap(target_bitmap_path, bitmap)
2209 if progress:
2210 progress("Bitmap index written\r".encode("ascii"))
2212 # Add the pack to the store and return it.
2213 final_pack = Pack(
2214 pack_base_name,
2215 object_format=self.object_format,
2216 delta_window_size=self.pack_delta_window_size,
2217 window_memory=self.pack_window_memory,
2218 delta_cache_size=self.pack_delta_cache_size,
2219 depth=self.pack_depth,
2220 threads=self.pack_threads,
2221 big_file_threshold=self.pack_big_file_threshold,
2222 delta_base_cache_limit=self.delta_base_cache_limit,
2223 )
2224 try:
2225 final_pack.check_length_and_checksum()
2226 # Materialise every object so payloads that fail to parse
2227 # (e.g. tree entries with garbage modes) are rejected rather
2228 # than silently landed on disk. MemoryObjectStore already
2229 # validates ingested objects this way via PackInflater; without
2230 # the same check DiskObjectStore was strictly weaker.
2231 for _obj in PackInflater.for_pack_data(
2232 final_pack.data, resolve_ext_ref=self.get_raw
2233 ):
2234 pass
2235 except BaseException:
2236 final_pack.close()
2237 with suppress(FileNotFoundError):
2238 os.remove(target_pack_path)
2239 with suppress(FileNotFoundError):
2240 os.remove(target_index_path)
2241 if self.pack_write_bitmaps and refs:
2242 with suppress(FileNotFoundError):
2243 os.remove(pack_base_name + ".bitmap")
2244 raise
2245 # _pack_cache is keyed by the full basename (/path/to/pack-HASH -> pack-HASH)
2246 self._add_cached_pack(os.path.basename(pack_base_name), final_pack)
2247 return final_pack
2249 def add_thin_pack(
2250 self,
2251 read_all: Callable[[int], bytes],
2252 read_some: Callable[[int], bytes] | None,
2253 progress: Callable[..., None] | None = None,
2254 *,
2255 max_input_size: int | None = None,
2256 ) -> "Pack":
2257 """Add a new thin pack to this object store.
2259 Thin packs are packs that contain deltas with parents that exist
2260 outside the pack. They should never be placed in the object store
2261 directly, and always indexed and completed as they are copied.
2263 Args:
2264 read_all: Read function that blocks until the number of
2265 requested bytes are read.
2266 read_some: Read function that returns at least one byte, but may
2267 not return the number of bytes requested.
2268 progress: Optional progress reporting function.
2269 max_input_size: Maximum number of bytes that may be read from
2270 the wire while ingesting this pack. Matches git's
2271 ``receive.maxInputSize`` / ``index-pack --max-input-size``
2272 semantics: ``None`` (the default) or ``0`` mean unlimited.
2273 Exceeding the cap raises ``PackInputTooLarge``.
2274 Returns: A Pack object pointing at the now-completed thin pack in the
2275 objects/pack directory.
2276 """
2277 import tempfile
2279 if max_input_size:
2280 read_all, read_some = _bound_read_callables(
2281 read_all, read_some, max_input_size
2282 )
2284 fd, path = tempfile.mkstemp(dir=self.path, prefix="tmp_pack_")
2285 with os.fdopen(fd, "w+b") as f:
2286 os.chmod(path, PACK_MODE)
2287 indexer = PackIndexer(
2288 f,
2289 self.object_format.hash_func,
2290 resolve_ext_ref=self.get_raw,
2291 )
2292 copier = PackStreamCopier(
2293 self.object_format.hash_func,
2294 read_all,
2295 read_some,
2296 f,
2297 delta_iter=indexer, # type: ignore[arg-type]
2298 )
2299 copier.verify(progress=progress)
2300 return self._complete_pack(f, path, len(copier), indexer, progress=progress)
2302 def add_pack(
2303 self,
2304 ) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2305 """Add a new pack to this object store.
2307 Returns: Fileobject to write to, a commit function to
2308 call when the pack is finished and an abort
2309 function.
2310 """
2311 import tempfile
2313 fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
2314 f = os.fdopen(fd, "w+b")
2315 mask = self.file_mode if self.file_mode is not None else PACK_MODE
2316 os.chmod(path, mask)
2318 def commit() -> "Pack | None":
2319 if f.tell() > 0:
2320 f.seek(0)
2322 with PackData(path, file=f, object_format=self.object_format) as pd:
2323 indexer = PackIndexer.for_pack_data(
2324 pd,
2325 resolve_ext_ref=self.get_raw,
2326 )
2327 return self._complete_pack(f, path, len(pd), indexer) # type: ignore[arg-type]
2328 else:
2329 f.close()
2330 os.remove(path)
2331 return None
2333 def abort() -> None:
2334 f.close()
2335 os.remove(path)
2337 return f, commit, abort # type: ignore[return-value]
2339 def add_object(self, obj: ShaFile) -> None:
2340 """Add a single object to this object store.
2342 Args:
2343 obj: Object to add
2344 """
2345 # Use the correct hash algorithm for the object ID
2346 obj_id = ObjectID(obj.get_id(self.object_format))
2347 path = self._get_shafile_path(obj_id)
2348 dir = os.path.dirname(path)
2349 try:
2350 os.mkdir(dir)
2351 if self.dir_mode is not None:
2352 os.chmod(dir, self.dir_mode)
2353 except FileExistsError:
2354 pass
2355 if os.path.exists(path):
2356 return # Already there, no need to write again
2357 mask = self.file_mode if self.file_mode is not None else PACK_MODE
2358 with GitFile(path, "wb", mask=mask, fsync=self.fsync_object_files) as f:
2359 f.write(
2360 obj.as_legacy_object(compression_level=self.loose_compression_level)
2361 )
2363 @classmethod
2364 def init(
2365 cls,
2366 path: str | os.PathLike[str],
2367 *,
2368 file_mode: int | None = None,
2369 dir_mode: int | None = None,
2370 object_format: "ObjectFormat | None" = None,
2371 ) -> "DiskObjectStore":
2372 """Initialize a new disk object store.
2374 Creates the necessary directory structure for a Git object store.
2376 Args:
2377 path: Path where the object store should be created
2378 file_mode: Optional file permission mask for shared repository
2379 dir_mode: Optional directory permission mask for shared repository
2380 object_format: Hash algorithm to use (SHA1 or SHA256)
2382 Returns:
2383 New DiskObjectStore instance
2384 """
2385 try:
2386 os.mkdir(path)
2387 if dir_mode is not None:
2388 os.chmod(path, dir_mode)
2389 except FileExistsError:
2390 pass
2391 info_path = os.path.join(path, "info")
2392 pack_path = os.path.join(path, PACKDIR)
2393 os.mkdir(info_path)
2394 os.mkdir(pack_path)
2395 if dir_mode is not None:
2396 os.chmod(info_path, dir_mode)
2397 os.chmod(pack_path, dir_mode)
2398 return cls(
2399 path, file_mode=file_mode, dir_mode=dir_mode, object_format=object_format
2400 )
2402 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
2403 """Iterate over all object SHAs with the given prefix.
2405 Args:
2406 prefix: Hex prefix to search for (as bytes)
2408 Returns:
2409 Iterator of object SHAs (as ObjectID) matching the prefix
2410 """
2411 if len(prefix) < 2:
2412 yield from super().iter_prefix(prefix)
2413 return
2414 seen = set()
2415 dir = prefix[:2].decode()
2416 rest = prefix[2:].decode()
2417 try:
2418 for name in os.listdir(os.path.join(self.path, dir)):
2419 if name.startswith(rest):
2420 sha = ObjectID(os.fsencode(dir + name))
2421 if sha not in seen:
2422 seen.add(sha)
2423 yield sha
2424 except FileNotFoundError:
2425 pass
2427 for p in self.packs:
2428 bin_prefix = (
2429 binascii.unhexlify(prefix)
2430 if len(prefix) % 2 == 0
2431 else binascii.unhexlify(prefix[:-1])
2432 )
2433 for bin_sha in p.index.iter_prefix(bin_prefix):
2434 sha = sha_to_hex(bin_sha)
2435 if sha.startswith(prefix) and sha not in seen:
2436 seen.add(sha)
2437 yield sha
2438 for alternate in self.alternates:
2439 for sha in alternate.iter_prefix(prefix):
2440 if sha not in seen:
2441 seen.add(sha)
2442 yield sha
2444 def get_commit_graph(self) -> "CommitGraph | None":
2445 """Get the commit graph for this object store.
2447 Returns:
2448 CommitGraph object if available, None otherwise
2449 """
2450 if not self._use_commit_graph:
2451 return None
2453 if self._commit_graph is None:
2454 from .commit_graph import read_commit_graph
2456 # Look for commit graph in our objects directory
2457 graph_file = os.path.join(self.path, "info", "commit-graph")
2458 if os.path.exists(graph_file):
2459 self._commit_graph = read_commit_graph(graph_file)
2460 return self._commit_graph
2462 def get_midx(self) -> MultiPackIndex | None:
2463 """Get the multi-pack-index for this object store.
2465 Returns:
2466 MultiPackIndex object if available, None otherwise
2468 Raises:
2469 ValueError: If MIDX file is corrupt
2470 OSError: If MIDX file cannot be read
2471 """
2472 if not self._use_midx:
2473 return None
2475 if self._midx is None:
2476 # Look for MIDX in pack directory
2477 midx_file = os.path.join(self.pack_dir, "multi-pack-index")
2478 if os.path.exists(midx_file):
2479 self._midx = load_midx(midx_file)
2480 return self._midx
2482 def _get_pack_by_name(self, pack_name: str) -> Pack:
2483 """Get a pack referenced by a multi-pack-index entry.
2485 Args:
2486 pack_name: Pack index file name as stored in the MIDX. Usually
2487 ``pack-<hash>.idx``, but ``git maintenance`` writes packs
2488 named ``loose-<hash>.idx``, so any ``.idx`` basename is
2489 accepted.
2491 Returns:
2492 Pack object
2494 Raises:
2495 KeyError: If pack doesn't exist
2496 """
2497 if not pack_name.endswith(".idx"):
2498 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2499 # The name is joined under pack_dir below, so reject any path
2500 # separators that a corrupt or hostile MIDX could use to traverse
2501 # directories (backslash matters on Windows).
2502 if "/" in pack_name or "\\" in pack_name:
2503 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2504 basename = pack_name[: -len(".idx")]
2506 # _pack_cache is keyed by full basename and _update_pack_cache
2507 # discovers every "<name>-<hash>.pack" file (including the
2508 # "loose-<hash>" packs that git maintenance writes and the MIDX
2509 # references), so a referenced pack is normally already cached.
2510 try:
2511 return self._pack_cache[basename]
2512 except KeyError:
2513 pass
2515 pack_path = os.path.join(self.pack_dir, basename)
2516 if not os.path.exists(pack_path + ".pack"):
2517 raise KeyError(f"Pack {pack_name} not found")
2519 pack = Pack(
2520 pack_path,
2521 object_format=self.object_format,
2522 delta_window_size=self.pack_delta_window_size,
2523 window_memory=self.pack_window_memory,
2524 delta_cache_size=self.pack_delta_cache_size,
2525 depth=self.pack_depth,
2526 threads=self.pack_threads,
2527 big_file_threshold=self.pack_big_file_threshold,
2528 delta_base_cache_limit=self.delta_base_cache_limit,
2529 )
2530 self._pack_cache[basename] = pack
2531 self._mark_pack_used(basename)
2532 return pack
2534 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2535 """Check if a particular object is present by SHA1 and is packed.
2537 This checks the MIDX first if available, then falls back to checking
2538 individual pack indexes.
2540 Args:
2541 sha: SHA of the object (20/32 bytes binary or 40/64 bytes hex)
2543 Returns:
2544 True if the object is in a pack file
2545 """
2546 # Normalise to binary once: MIDX requires it, and passing binary to
2547 # the per-pack fallback avoids N redundant hex->binary conversions
2548 # inside PackIndex.object_offset. Mirrors ``get_raw`` below.
2549 if len(sha) == self.object_format.hex_length:
2550 sha = hex_to_sha(cast(ObjectID, sha))
2552 midx = self.get_midx()
2553 if midx is not None and sha in midx:
2554 return True
2556 # Fall back to checking individual packs
2557 return super().contains_packed(sha)
2559 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2560 """Obtain the raw fulltext for an object.
2562 This uses the MIDX if available for faster lookups.
2564 Args:
2565 name: SHA for the object (20 bytes binary or 40 bytes hex)
2567 Returns:
2568 Tuple with numeric type and object contents
2570 Raises:
2571 KeyError: If object not found
2572 """
2573 sha: RawObjectID
2574 if len(name) in (40, 64):
2575 # name is ObjectID (hex), convert to RawObjectID
2576 # Support both SHA1 (40) and SHA256 (64)
2577 sha = hex_to_sha(cast(ObjectID, name))
2578 elif len(name) in (20, 32):
2579 # name is already RawObjectID (binary)
2580 # Support both SHA1 (20) and SHA256 (32)
2581 sha = RawObjectID(name)
2582 else:
2583 raise AssertionError(f"Invalid object name {name!r}")
2585 # Try MIDX first for faster lookup
2586 midx = self.get_midx()
2587 if midx is not None:
2588 result = midx.object_offset(sha)
2589 if result is not None:
2590 pack_name, _offset = result
2591 try:
2592 pack = self._get_pack_by_name(pack_name)
2593 return pack.get_raw(sha)
2594 except (KeyError, PackFileDisappeared):
2595 # Pack disappeared or object not found, fall through to standard lookup
2596 pass
2598 # Fall back to the standard implementation
2599 return super().get_raw(name)
2601 def write_midx(self) -> bytes:
2602 """Write a multi-pack-index file for this object store.
2604 Creates a MIDX file that indexes all pack files in the pack directory.
2606 Returns:
2607 SHA-1 checksum of the written MIDX file
2609 Raises:
2610 OSError: If the pack directory doesn't exist or MIDX can't be written
2611 """
2612 from .midx import write_midx_file
2614 midx_path = os.path.join(self.pack_dir, "multi-pack-index")
2615 # Skip packs that vanish mid-collection (e.g. concurrent
2616 # ``git repack``); the survivors still produce a valid MIDX.
2617 pack_entries: list[tuple[str, list[tuple[RawObjectID, int, int | None]]]] = []
2618 for pack in self.packs:
2619 try:
2620 entries = list(pack.index.iterentries())
2621 except PackFileDisappeared as exc:
2622 self._evict_pack(exc.obj)
2623 continue
2624 pack_entries.append((os.path.basename(pack._basename) + ".idx", entries))
2625 if not pack_entries:
2626 return b"\x00" * 20
2627 return write_midx_file(midx_path, pack_entries)
2629 def write_commit_graph(
2630 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
2631 ) -> None:
2632 """Write a commit graph file for this object store.
2634 Args:
2635 refs: List of refs to include. If None, includes all refs from object store.
2636 reachable: If True, includes all commits reachable from refs.
2637 If False, only includes the direct ref targets.
2638 """
2639 from .commit_graph import get_reachable_commits
2641 if refs is None:
2642 # Get all commit objects from the object store
2643 all_refs = []
2644 # Iterate through all objects to find commits
2645 for sha in self:
2646 try:
2647 obj = self[sha]
2648 if obj.type_name == b"commit":
2649 all_refs.append(sha)
2650 except KeyError:
2651 continue
2652 else:
2653 # Use provided refs
2654 all_refs = list(refs)
2656 if not all_refs:
2657 return # No commits to include
2659 if reachable:
2660 # Get all reachable commits
2661 commit_ids = get_reachable_commits(self, all_refs)
2662 else:
2663 # Just use the direct ref targets - ensure they're hex ObjectIDs
2664 commit_ids = []
2665 for ref in all_refs:
2666 if isinstance(ref, bytes) and len(ref) == self.object_format.hex_length:
2667 # Already hex ObjectID
2668 commit_ids.append(ref)
2669 elif (
2670 isinstance(ref, bytes) and len(ref) == self.object_format.oid_length
2671 ):
2672 # Binary SHA, convert to hex ObjectID
2673 commit_ids.append(sha_to_hex(RawObjectID(ref)))
2674 else:
2675 # Assume it's already correct format
2676 commit_ids.append(ref)
2678 if commit_ids:
2679 # Write commit graph directly to our object store path
2680 # Generate the commit graph
2681 from .commit_graph import generate_commit_graph
2683 graph = generate_commit_graph(self, commit_ids)
2685 if graph.entries:
2686 # Ensure the info directory exists
2687 info_dir = os.path.join(self.path, "info")
2688 os.makedirs(info_dir, exist_ok=True)
2689 if self.dir_mode is not None:
2690 os.chmod(info_dir, self.dir_mode)
2692 # Write using GitFile for atomic operation
2693 graph_path = os.path.join(info_dir, "commit-graph")
2694 mask = self.file_mode if self.file_mode is not None else 0o644
2695 with GitFile(graph_path, "wb", mask=mask) as f:
2696 assert isinstance(
2697 f, _GitFile
2698 ) # GitFile in write mode always returns _GitFile
2699 graph.write_to_file(f)
2701 # Clear cached commit graph so it gets reloaded
2702 self._commit_graph = None
2704 def prune(self, grace_period: int | None = None) -> None:
2705 """Prune/clean up this object store.
2707 This removes temporary files that were left behind by interrupted
2708 pack operations. These are files that start with ``tmp_pack_`` in the
2709 repository directory or files with .pack extension but no corresponding
2710 .idx file in the pack directory.
2712 Args:
2713 grace_period: Grace period in seconds for removing temporary files.
2714 If None, uses DEFAULT_TEMPFILE_GRACE_PERIOD.
2715 """
2716 import glob
2718 if grace_period is None:
2719 grace_period = DEFAULT_TEMPFILE_GRACE_PERIOD
2721 # Clean up tmp_pack_* files in the repository directory
2722 for tmp_file in glob.glob(os.path.join(self.path, "tmp_pack_*")):
2723 # Check if file is old enough (more than grace period)
2724 mtime = os.path.getmtime(tmp_file)
2725 if time.time() - mtime > grace_period:
2726 os.remove(tmp_file)
2728 # Clean up orphaned .pack files without corresponding .idx files
2729 try:
2730 pack_dir_contents = os.listdir(self.pack_dir)
2731 except FileNotFoundError:
2732 return
2734 pack_files = {}
2735 idx_files = set()
2737 for name in pack_dir_contents:
2738 if name.endswith(".pack"):
2739 base_name = name[:-5] # Remove .pack extension
2740 pack_files[base_name] = name
2741 elif name.endswith(".idx"):
2742 base_name = name[:-4] # Remove .idx extension
2743 idx_files.add(base_name)
2745 # Remove .pack files without corresponding .idx files
2746 for base_name, pack_name in pack_files.items():
2747 if base_name not in idx_files:
2748 pack_path = os.path.join(self.pack_dir, pack_name)
2749 # Check if file is old enough (more than grace period)
2750 mtime = os.path.getmtime(pack_path)
2751 if time.time() - mtime > grace_period:
2752 os.remove(pack_path)
2754 def close(self) -> None:
2755 """Close the object store and release resources.
2757 This method closes all cached pack files, MIDX, and frees associated resources.
2758 Can be called multiple times safely.
2759 """
2760 # Close MIDX if it's loaded
2761 if self._midx is not None:
2762 self._midx.close()
2763 self._midx = None
2765 # Close alternates
2766 if self._alternates is not None:
2767 for alt in self._alternates:
2768 alt.close()
2769 self._alternates = None
2771 # Call parent class close to handle pack files
2772 super().close()
2775class MemoryObjectStore(PackCapableObjectStore):
2776 """Object store that keeps all objects in memory."""
2778 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
2779 """Initialize a MemoryObjectStore.
2781 Creates an empty in-memory object store.
2783 Args:
2784 object_format: Hash algorithm to use (defaults to SHA1)
2785 """
2786 super().__init__(object_format=object_format)
2787 self._data: dict[ObjectID, ShaFile] = {}
2788 self.pack_compression_level = -1
2790 def _to_hexsha(self, sha: ObjectID | RawObjectID) -> ObjectID:
2791 if len(sha) == self.object_format.hex_length:
2792 return cast(ObjectID, sha)
2793 elif len(sha) == self.object_format.oid_length:
2794 return sha_to_hex(cast(RawObjectID, sha))
2795 else:
2796 raise ValueError(f"Invalid sha {sha!r}")
2798 def contains_loose(self, sha: ObjectID) -> bool:
2799 """Check if a particular object is present by SHA1 and is loose."""
2800 return self._to_hexsha(sha) in self._data
2802 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2803 """Check if a particular object is present by SHA1 and is packed."""
2804 return False
2806 def __iter__(self) -> Iterator[ObjectID]:
2807 """Iterate over the SHAs that are present in this store."""
2808 return iter(self._data.keys())
2810 @property
2811 def packs(self) -> list[Pack]:
2812 """List with pack objects."""
2813 return []
2815 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2816 """Obtain the raw text for an object.
2818 Args:
2819 name: sha for the object.
2820 Returns: tuple with numeric type and object contents.
2821 """
2822 obj = self[self._to_hexsha(name)]
2823 return obj.type_num, obj.as_raw_string()
2825 def __getitem__(self, name: ObjectID | RawObjectID) -> ShaFile:
2826 """Retrieve an object by SHA.
2828 Args:
2829 name: SHA of the object (as hex string or bytes)
2831 Returns:
2832 Copy of the ShaFile object
2834 Raises:
2835 KeyError: If the object is not found
2836 """
2837 return self._data[self._to_hexsha(name)].copy()
2839 def __delitem__(self, name: ObjectID) -> None:
2840 """Delete an object from this store, for testing only."""
2841 del self._data[self._to_hexsha(name)]
2843 def add_object(self, obj: ShaFile) -> None:
2844 """Add a single object to this object store."""
2845 self._data[obj.id] = obj.copy()
2847 def add_objects(
2848 self,
2849 objects: Iterable[tuple[ShaFile, str | None]],
2850 progress: Callable[[str], None] | None = None,
2851 ) -> None:
2852 """Add a set of objects to this object store.
2854 Args:
2855 objects: Iterable over a list of (object, path) tuples
2856 progress: Optional progress reporting function.
2857 """
2858 for obj, path in objects:
2859 self.add_object(obj)
2861 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2862 """Add a new pack to this object store.
2864 Because this object store doesn't support packs, we extract and add the
2865 individual objects.
2867 Returns: Fileobject to write to and a commit function to
2868 call when the pack is finished.
2869 """
2870 from tempfile import SpooledTemporaryFile
2872 f = SpooledTemporaryFile(max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-")
2874 def commit() -> None:
2875 size = f.tell()
2876 if size > 0:
2877 f.seek(0)
2879 p = PackData.from_file(f, self.object_format, size)
2880 try:
2881 # Verify the trailing pack checksum before extracting
2882 # objects. Without this, a fetch that delivered a
2883 # truncated pack would still be accepted: ``add_pack``
2884 # iterates objects by offset and never reaches the
2885 # trailing bytes, so a stream that lost the last few
2886 # bytes of its trailer slipped through silently.
2887 # ``add_thin_pack`` already validates via
2888 # ``PackStreamCopier.verify``; do the equivalent here.
2889 p.check()
2890 for obj in PackInflater.for_pack_data(p, self.get_raw):
2891 self.add_object(obj)
2892 finally:
2893 p.close()
2894 f.close()
2895 else:
2896 f.close()
2898 def abort() -> None:
2899 f.close()
2901 return f, commit, abort # type: ignore[return-value]
2903 def add_pack_data(
2904 self,
2905 count: int,
2906 unpacked_objects: Iterator[UnpackedObject],
2907 progress: Callable[[str], None] | None = None,
2908 ) -> None:
2909 """Add pack data to this object store.
2911 Args:
2912 count: Number of items to add
2913 unpacked_objects: Iterator of UnpackedObject instances
2914 progress: Optional progress reporting function.
2915 """
2916 if count == 0:
2917 return
2919 # Since MemoryObjectStore doesn't support pack files, we need to
2920 # extract individual objects. To handle deltas properly, we write
2921 # to a temporary pack and then use PackInflater to resolve them.
2922 f, commit, abort = self.add_pack()
2923 try:
2924 write_pack_data(
2925 f.write,
2926 unpacked_objects,
2927 num_records=count,
2928 progress=progress,
2929 object_format=self.object_format,
2930 )
2931 except BaseException:
2932 abort()
2933 raise
2934 else:
2935 commit()
2937 def add_thin_pack(
2938 self,
2939 read_all: Callable[[int], bytes],
2940 read_some: Callable[[int], bytes] | None,
2941 progress: Callable[[str], None] | None = None,
2942 ) -> None:
2943 """Add a new thin pack to this object store.
2945 Thin packs are packs that contain deltas with parents that exist
2946 outside the pack. Because this object store doesn't support packs, we
2947 extract and add the individual objects.
2949 Args:
2950 read_all: Read function that blocks until the number of
2951 requested bytes are read.
2952 read_some: Read function that returns at least one byte, but may
2953 not return the number of bytes requested.
2954 progress: Optional progress reporting function.
2955 """
2956 f, commit, abort = self.add_pack()
2957 try:
2958 copier = PackStreamCopier(
2959 self.object_format.hash_func,
2960 read_all,
2961 read_some,
2962 f,
2963 )
2964 copier.verify()
2965 except BaseException:
2966 abort()
2967 raise
2968 else:
2969 commit()
2972class ObjectIterator(Protocol):
2973 """Interface for iterating over objects."""
2975 def iterobjects(self) -> Iterator[ShaFile]:
2976 """Iterate over all objects.
2978 Returns:
2979 Iterator of ShaFile objects
2980 """
2981 raise NotImplementedError(self.iterobjects)
2984def tree_lookup_path(
2985 lookup_obj: Callable[[ObjectID | RawObjectID], ShaFile],
2986 root_sha: ObjectID | RawObjectID,
2987 path: bytes,
2988) -> tuple[int, ObjectID]:
2989 """Look up an object in a Git tree.
2991 Args:
2992 lookup_obj: Callback for retrieving object by SHA1
2993 root_sha: SHA1 of the root tree
2994 path: Path to lookup
2995 Returns: A tuple of (mode, SHA) of the resulting path.
2996 """
2997 tree = lookup_obj(root_sha)
2998 if not isinstance(tree, Tree):
2999 raise NotTreeError(root_sha)
3000 return tree.lookup_path(lookup_obj, path)
3003def _collect_filetree_revs(
3004 obj_store: ObjectContainer, tree_sha: ObjectID, kset: set[ObjectID]
3005) -> None:
3006 """Collect SHA1s of files and directories for specified tree.
3008 Args:
3009 obj_store: Object store to get objects by SHA from
3010 tree_sha: tree reference to walk
3011 kset: set to fill with references to files and directories
3012 """
3013 filetree = obj_store[tree_sha]
3014 assert isinstance(filetree, Tree)
3015 for name, mode, sha in filetree.iteritems():
3016 assert mode is not None
3017 assert sha is not None
3018 if not S_ISGITLINK(mode) and sha not in kset:
3019 kset.add(sha)
3020 if stat.S_ISDIR(mode):
3021 _collect_filetree_revs(obj_store, sha, kset)
3024def _split_commits_and_tags(
3025 obj_store: ObjectContainer,
3026 lst: Iterable[ObjectID],
3027 *,
3028 unknown: str = "error",
3029) -> tuple[set[ObjectID], set[ObjectID], set[ObjectID]]:
3030 """Split object id list into three lists with commit, tag, and other SHAs.
3032 Commits referenced by tags are included into commits
3033 list as well. Only SHA1s known in this repository will get
3034 through, controlled by the unknown parameter.
3036 Args:
3037 obj_store: Object store to get objects by SHA1 from
3038 lst: Collection of commit and tag SHAs
3039 unknown: How to handle unknown objects: "error", "warn", or "ignore"
3040 Returns: A tuple of (commits, tags, others) SHA1s
3041 """
3042 if unknown not in ("error", "warn", "ignore"):
3043 raise ValueError(
3044 f"unknown must be 'error', 'warn', or 'ignore', got {unknown!r}"
3045 )
3047 commits: set[ObjectID] = set()
3048 tags: set[ObjectID] = set()
3049 others: set[ObjectID] = set()
3050 for e in lst:
3051 try:
3052 o = obj_store[e]
3053 except KeyError:
3054 if unknown == "error":
3055 raise
3056 elif unknown == "warn":
3057 logger.warning("Object %s not found in object store", e.decode("ascii"))
3058 # else: ignore
3059 else:
3060 if isinstance(o, Commit):
3061 commits.add(e)
3062 elif isinstance(o, Tag):
3063 tags.add(e)
3064 tagged = o.object[1]
3065 c, t, os = _split_commits_and_tags(obj_store, [tagged], unknown=unknown)
3066 commits |= c
3067 tags |= t
3068 others |= os
3069 else:
3070 others.add(e)
3071 return (commits, tags, others)
3074class MissingObjectFinder:
3075 """Find the objects missing from another object store.
3077 Args:
3078 object_store: Object store containing at least all objects to be
3079 sent
3080 haves: SHA1s of commits not to send (already present in target)
3081 wants: SHA1s of commits to send
3082 progress: Optional function to report progress to.
3083 get_tagged: Function that returns a dict of pointed-to sha -> tag
3084 sha for including tags.
3085 get_parents: Optional function for getting the parents of a commit.
3086 """
3088 def __init__(
3089 self,
3090 object_store: BaseObjectStore,
3091 haves: Iterable[ObjectID],
3092 wants: Iterable[ObjectID],
3093 *,
3094 shallow: Set[ObjectID] | None = None,
3095 progress: Callable[[bytes], None] | None = None,
3096 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
3097 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3098 ) -> None:
3099 """Initialize a MissingObjectFinder.
3101 Args:
3102 object_store: Object store containing objects
3103 haves: SHA1s of objects already present in target
3104 wants: SHA1s of objects to send
3105 shallow: Set of shallow commit SHA1s
3106 progress: Optional progress reporting callback
3107 get_tagged: Function returning dict of pointed-to sha -> tag sha
3108 get_parents: Function for getting commit parents
3109 """
3110 self.object_store = object_store
3111 if shallow is None:
3112 shallow = set()
3113 self._get_parents = get_parents
3114 reachability = object_store.get_reachability_provider()
3115 # process Commits and Tags differently
3116 # haves may list commits/tags not available locally (silently ignore them).
3117 # wants should only contain valid SHAs (fail fast if not).
3118 have_commits, have_tags, have_others = _split_commits_and_tags(
3119 object_store, haves, unknown="ignore"
3120 )
3121 want_commits, want_tags, want_others = _split_commits_and_tags(
3122 object_store, wants, unknown="error"
3123 )
3124 # all_ancestors is a set of commits that shall not be sent
3125 # (complete repository up to 'haves')
3126 all_ancestors = reachability.get_reachable_commits(
3127 have_commits, exclude=None, shallow=shallow
3128 )
3129 # all_missing - complete set of commits between haves and wants
3130 # common_commits - boundary commits directly encountered when traversing wants
3131 # We use _collect_ancestors here because we need the exact boundary behavior:
3132 # commits that are in all_ancestors and directly reachable from wants,
3133 # but we don't traverse past them. This is hard to express with the
3134 # reachability abstraction alone.
3135 missing_commits, common_commits = _collect_ancestors(
3136 object_store,
3137 want_commits,
3138 frozenset(all_ancestors),
3139 shallow=frozenset(shallow),
3140 get_parents=self._get_parents,
3141 )
3143 self.remote_has: set[ObjectID] = set()
3144 # Now, fill sha_done with commits and revisions of
3145 # files and directories known to be both locally
3146 # and on target. Thus these commits and files
3147 # won't get selected for fetch
3148 for h in common_commits:
3149 self.remote_has.add(h)
3150 cmt = object_store[h]
3151 assert isinstance(cmt, Commit)
3152 # Get tree objects for this commit
3153 tree_objects = reachability.get_tree_objects([cmt.tree])
3154 self.remote_has.update(tree_objects)
3156 # record tags we have as visited, too
3157 for t in have_tags:
3158 self.remote_has.add(t)
3159 self.sha_done = set(self.remote_has)
3161 # in fact, what we 'want' is commits, tags, and others
3162 # we've found missing
3163 self.objects_to_send: set[tuple[ObjectID, bytes | None, int | None, bool]] = {
3164 (w, None, Commit.type_num, False) for w in missing_commits
3165 }
3166 missing_tags = want_tags.difference(have_tags)
3167 self.objects_to_send.update(
3168 {(w, None, Tag.type_num, False) for w in missing_tags}
3169 )
3170 missing_others = want_others.difference(have_others)
3171 self.objects_to_send.update({(w, None, None, False) for w in missing_others})
3173 if progress is None:
3174 self.progress: Callable[[bytes], None] = lambda x: None
3175 else:
3176 self.progress = progress
3177 self._tagged = (get_tagged and get_tagged()) or {}
3179 def get_remote_has(self) -> set[ObjectID]:
3180 """Get the set of SHAs the remote has.
3182 Returns:
3183 Set of SHA1s that the remote side already has
3184 """
3185 return self.remote_has
3187 def add_todo(
3188 self, entries: Iterable[tuple[ObjectID, bytes | None, int | None, bool]]
3189 ) -> None:
3190 """Add objects to the todo list.
3192 Args:
3193 entries: Iterable of tuples (sha, name, type_num, is_leaf)
3194 """
3195 self.objects_to_send.update([e for e in entries if e[0] not in self.sha_done])
3197 def __next__(self) -> tuple[ObjectID, PackHint | None]:
3198 """Get the next object to send.
3200 Returns:
3201 Tuple of (sha, pack_hint)
3203 Raises:
3204 StopIteration: When no more objects to send
3205 """
3206 while True:
3207 if not self.objects_to_send:
3208 self.progress(
3209 f"counting objects: {len(self.sha_done)}, done.\n".encode("ascii")
3210 )
3211 raise StopIteration
3212 (sha, name, type_num, leaf) = self.objects_to_send.pop()
3213 if sha not in self.sha_done:
3214 break
3215 if not leaf:
3216 o = self.object_store[sha]
3217 if isinstance(o, Commit):
3218 self.add_todo([(o.tree, b"", Tree.type_num, False)])
3219 elif isinstance(o, Tree):
3220 todos = []
3221 for n, m, s in o.iteritems():
3222 assert m is not None
3223 assert n is not None
3224 assert s is not None
3225 if not S_ISGITLINK(m):
3226 todos.append(
3227 (
3228 s,
3229 n,
3230 (Blob.type_num if stat.S_ISREG(m) else Tree.type_num),
3231 not stat.S_ISDIR(m),
3232 )
3233 )
3234 self.add_todo(todos)
3235 elif isinstance(o, Tag):
3236 self.add_todo([(o.object[1], None, o.object[0].type_num, False)])
3237 if sha in self._tagged:
3238 self.add_todo([(self._tagged[sha], None, None, True)])
3239 self.sha_done.add(sha)
3240 if len(self.sha_done) % 1000 == 0:
3241 self.progress(f"counting objects: {len(self.sha_done)}\r".encode("ascii"))
3242 if type_num is None:
3243 pack_hint = None
3244 else:
3245 pack_hint = (type_num, name)
3246 return (sha, pack_hint)
3248 def __iter__(self) -> Iterator[tuple[ObjectID, PackHint | None]]:
3249 """Return iterator over objects to send.
3251 Returns:
3252 Self (this class implements the iterator protocol)
3253 """
3254 return self
3257class ObjectStoreGraphWalker:
3258 """Graph walker that finds what commits are missing from an object store."""
3260 heads: set[ObjectID]
3261 """Revisions without descendants in the local repo."""
3263 get_parents: Callable[[ObjectID], list[ObjectID]]
3264 """Function to retrieve parents in the local repo."""
3266 shallow: set[ObjectID]
3268 def __init__(
3269 self,
3270 local_heads: Iterable[ObjectID],
3271 get_parents: Callable[[ObjectID], list[ObjectID]],
3272 shallow: set[ObjectID] | None = None,
3273 update_shallow: Callable[[set[ObjectID] | None, set[ObjectID] | None], None]
3274 | None = None,
3275 ) -> None:
3276 """Create a new instance.
3278 Args:
3279 local_heads: Heads to start search with
3280 get_parents: Function for finding the parents of a SHA1.
3281 shallow: Set of shallow commits.
3282 update_shallow: Function to update shallow commits.
3283 """
3284 self.heads = set(local_heads)
3285 self.get_parents = get_parents
3286 self.parents: dict[ObjectID, list[ObjectID] | None] = {}
3287 if shallow is None:
3288 shallow = set()
3289 self.shallow = shallow
3290 self.update_shallow = update_shallow
3292 def nak(self) -> None:
3293 """Nothing in common was found."""
3295 def ack(self, sha: ObjectID) -> None:
3296 """Ack that a revision and its ancestors are present in the source."""
3297 if len(sha) != 40:
3298 # TODO: support SHA256
3299 raise ValueError(f"unexpected sha {sha!r} received")
3300 ancestors = {sha}
3302 # stop if we run out of heads to remove
3303 while self.heads:
3304 for a in ancestors:
3305 if a in self.heads:
3306 self.heads.remove(a)
3308 # collect all ancestors
3309 new_ancestors = set()
3310 for a in ancestors:
3311 ps = self.parents.get(a)
3312 if ps is not None:
3313 new_ancestors.update(ps)
3314 self.parents[a] = None
3316 # no more ancestors; stop
3317 if not new_ancestors:
3318 break
3320 ancestors = new_ancestors
3322 def next(self) -> ObjectID | None:
3323 """Iterate over ancestors of heads in the target."""
3324 if self.heads:
3325 ret = self.heads.pop()
3326 try:
3327 ps = self.get_parents(ret)
3328 except KeyError:
3329 return None
3330 self.parents[ret] = ps
3331 self.heads.update([p for p in ps if p not in self.parents])
3332 return ret
3333 return None
3335 __next__ = next
3338def commit_tree_changes(
3339 object_store: BaseObjectStore,
3340 tree: ObjectID | Tree,
3341 changes: Sequence[tuple[bytes, int | None, ObjectID | None]],
3342) -> ObjectID:
3343 """Commit a specified set of changes to a tree structure.
3345 This will apply a set of changes on top of an existing tree, storing new
3346 objects in object_store.
3348 changes are a list of tuples with (path, mode, object_sha).
3349 Paths can be both blobs and trees. See the mode and
3350 object sha to None deletes the path.
3352 This method works especially well if there are only a small
3353 number of changes to a big tree. For a large number of changes
3354 to a large tree, use e.g. commit_tree.
3356 Args:
3357 object_store: Object store to store new objects in
3358 and retrieve old ones from.
3359 tree: Original tree root (SHA or Tree object)
3360 changes: changes to apply
3361 Returns: New tree root object
3362 """
3363 # TODO(jelmer): Save up the objects and add them using .add_objects
3364 # rather than with individual calls to .add_object.
3365 # Handle both Tree object and SHA
3366 if isinstance(tree, Tree):
3367 tree_obj: Tree = tree
3368 else:
3369 sha_obj = object_store[tree]
3370 assert isinstance(sha_obj, Tree)
3371 tree_obj = sha_obj
3372 nested_changes: dict[bytes, list[tuple[bytes, int | None, ObjectID | None]]] = {}
3373 for path, new_mode, new_sha in changes:
3374 try:
3375 (dirname, subpath) = path.split(b"/", 1)
3376 except ValueError:
3377 if new_sha is None:
3378 del tree_obj[path]
3379 else:
3380 assert new_mode is not None
3381 tree_obj[path] = (new_mode, new_sha)
3382 else:
3383 nested_changes.setdefault(dirname, []).append((subpath, new_mode, new_sha))
3384 for name, subchanges in nested_changes.items():
3385 try:
3386 orig_subtree_id: ObjectID | Tree = tree_obj[name][1]
3387 except KeyError:
3388 # For new directories, pass an empty Tree object
3389 orig_subtree_id = Tree()
3390 subtree_id = commit_tree_changes(object_store, orig_subtree_id, subchanges)
3391 subtree = object_store[subtree_id]
3392 assert isinstance(subtree, Tree)
3393 if len(subtree) == 0:
3394 del tree_obj[name]
3395 else:
3396 tree_obj[name] = (stat.S_IFDIR, subtree.id)
3397 object_store.add_object(tree_obj)
3398 return tree_obj.id
3401class OverlayObjectStore(BaseObjectStore):
3402 """Object store that can overlay multiple object stores."""
3404 def __init__(
3405 self,
3406 bases: list[BaseObjectStore],
3407 add_store: BaseObjectStore | None = None,
3408 ) -> None:
3409 """Initialize an OverlayObjectStore.
3411 Args:
3412 bases: List of base object stores to overlay
3413 add_store: Optional store to write new objects to
3415 Raises:
3416 ValueError: If stores have different hash algorithms
3417 """
3418 from .object_format import verify_same_object_format
3420 # Verify all stores use the same hash algorithm
3421 store_algorithms = [store.object_format for store in bases]
3422 if add_store:
3423 store_algorithms.append(add_store.object_format)
3425 object_format = verify_same_object_format(*store_algorithms)
3427 super().__init__(object_format=object_format)
3428 self.bases = bases
3429 self.add_store = add_store
3431 def add_object(self, obj: ShaFile) -> None:
3432 """Add a single object to the store.
3434 Args:
3435 obj: Object to add
3437 Raises:
3438 NotImplementedError: If no add_store was provided
3439 """
3440 if self.add_store is None:
3441 raise NotImplementedError(self.add_object)
3442 return self.add_store.add_object(obj)
3444 def add_objects(
3445 self,
3446 objects: Sequence[tuple[ShaFile, str | None]],
3447 progress: Callable[[str], None] | None = None,
3448 ) -> Pack | None:
3449 """Add multiple objects to the store.
3451 Args:
3452 objects: Iterator of objects to add
3453 progress: Optional progress reporting callback
3455 Raises:
3456 NotImplementedError: If no add_store was provided
3457 """
3458 if self.add_store is None:
3459 raise NotImplementedError(self.add_object)
3460 return self.add_store.add_objects(objects, progress)
3462 @property
3463 def packs(self) -> list[Pack]:
3464 """Get the list of packs from all overlaid stores.
3466 Returns:
3467 Combined list of packs from all base stores
3468 """
3469 ret = []
3470 for b in self.bases:
3471 ret.extend(b.packs)
3472 return ret
3474 def __iter__(self) -> Iterator[ObjectID]:
3475 """Iterate over all object SHAs in the overlaid stores.
3477 Returns:
3478 Iterator of object SHAs (deduped across stores)
3479 """
3480 done = set()
3481 for b in self.bases:
3482 for o_id in b:
3483 if o_id not in done:
3484 yield o_id
3485 done.add(o_id)
3487 def iterobjects_subset(
3488 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
3489 ) -> Iterator[ShaFile]:
3490 """Iterate over a subset of objects from the overlaid stores.
3492 Args:
3493 shas: Iterable of object SHAs to retrieve
3494 allow_missing: If True, skip missing objects; if False, raise KeyError
3496 Returns:
3497 Iterator of ShaFile objects
3499 Raises:
3500 KeyError: If an object is missing and allow_missing is False
3501 """
3502 todo = set(shas)
3503 found: set[ObjectID] = set()
3505 for b in self.bases:
3506 # Create a copy of todo for each base to avoid modifying
3507 # the set while iterating through it
3508 current_todo = todo - found
3509 for o in b.iterobjects_subset(current_todo, allow_missing=True):
3510 yield o
3511 found.add(o.id)
3513 # Check for any remaining objects not found
3514 missing = todo - found
3515 if missing and not allow_missing:
3516 raise KeyError(next(iter(missing)))
3518 def iter_unpacked_subset(
3519 self,
3520 shas: Iterable[ObjectID | RawObjectID],
3521 *,
3522 include_comp: bool = False,
3523 allow_missing: bool = False,
3524 convert_ofs_delta: bool = True,
3525 ) -> Iterator[UnpackedObject]:
3526 """Iterate over unpacked objects from the overlaid stores.
3528 Args:
3529 shas: Iterable of object SHAs to retrieve
3530 include_comp: Whether to include compressed data
3531 allow_missing: If True, skip missing objects; if False, raise KeyError
3532 convert_ofs_delta: Whether to convert OFS_DELTA objects
3534 Returns:
3535 Iterator of unpacked objects
3537 Raises:
3538 KeyError: If an object is missing and allow_missing is False
3539 """
3540 todo: set[ObjectID | RawObjectID] = set(shas)
3541 for b in self.bases:
3542 for o in b.iter_unpacked_subset(
3543 todo,
3544 include_comp=include_comp,
3545 allow_missing=True,
3546 convert_ofs_delta=convert_ofs_delta,
3547 ):
3548 yield o
3549 todo.remove(o.sha())
3550 if todo and not allow_missing:
3551 raise KeyError(next(iter(todo)))
3553 def get_raw(self, name: ObjectID | RawObjectID) -> tuple[int, bytes]:
3554 """Get the raw object data from the overlaid stores.
3556 Args:
3557 name: SHA of the object
3559 Returns:
3560 Tuple of (type_num, raw_data)
3562 Raises:
3563 KeyError: If object not found in any base store
3564 """
3565 for b in self.bases:
3566 try:
3567 return b.get_raw(name)
3568 except KeyError:
3569 pass
3570 raise KeyError(name)
3572 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
3573 """Check if an object is packed in any base store.
3575 Args:
3576 sha: SHA of the object
3578 Returns:
3579 True if object is packed in any base store
3580 """
3581 for b in self.bases:
3582 if b.contains_packed(sha):
3583 return True
3584 return False
3586 def contains_loose(self, sha: ObjectID) -> bool:
3587 """Check if an object is loose in any base store.
3589 Args:
3590 sha: SHA of the object
3592 Returns:
3593 True if object is loose in any base store
3594 """
3595 for b in self.bases:
3596 if b.contains_loose(sha):
3597 return True
3598 return False
3601def read_packs_file(f: BinaryIO) -> Iterator[str]:
3602 """Yield the packs listed in a packs file."""
3603 for line in f.read().splitlines():
3604 if not line:
3605 continue
3606 (kind, name) = line.split(b" ", 1)
3607 if kind != b"P":
3608 continue
3609 yield os.fsdecode(name)
3612class BucketBasedObjectStore(PackBasedObjectStore):
3613 """Object store implementation that uses a bucket store like S3 as backend."""
3615 def _iter_loose_objects(self) -> Iterator[ObjectID]:
3616 """Iterate over the SHAs of all loose objects."""
3617 return iter([])
3619 def _get_loose_object(self, sha: ObjectID) -> None:
3620 return None
3622 def delete_loose_object(self, sha: ObjectID) -> None:
3623 """Delete a loose object (no-op for bucket stores).
3625 Bucket-based stores don't have loose objects, so this is a no-op.
3627 Args:
3628 sha: SHA of the object to delete
3629 """
3630 # Doesn't exist..
3632 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
3633 """Pack loose objects. Returns number of objects packed.
3635 BucketBasedObjectStore doesn't support loose objects, so this is a no-op.
3637 Args:
3638 progress: Optional progress reporting callback (ignored)
3639 """
3640 return 0
3642 def _remove_pack_by_name(self, name: str) -> None:
3643 """Remove a pack by name. Subclasses should implement this."""
3644 raise NotImplementedError(self._remove_pack_by_name)
3646 def _iter_pack_names(self) -> Iterator[str]:
3647 raise NotImplementedError(self._iter_pack_names)
3649 def _get_pack(self, name: str) -> Pack:
3650 raise NotImplementedError(self._get_pack)
3652 def _update_pack_cache(self) -> list[Pack]:
3653 pack_files = set(self._iter_pack_names())
3655 # Open newly appeared pack files
3656 new_packs = []
3657 for f in pack_files:
3658 if f not in self._pack_cache:
3659 pack = self._get_pack(f)
3660 new_packs.append(pack)
3661 self._pack_cache[f] = pack
3662 # Remove disappeared pack files
3663 for f in set(self._pack_cache) - pack_files:
3664 self._pack_cache.pop(f).close()
3665 return new_packs
3667 def _upload_pack(
3668 self, basename: str, pack_file: BinaryIO, index_file: BinaryIO
3669 ) -> None:
3670 raise NotImplementedError
3672 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
3673 """Add a new pack to this object store.
3675 Returns: Fileobject to write to, a commit function to
3676 call when the pack is finished and an abort
3677 function.
3678 """
3679 import tempfile
3681 pf = tempfile.SpooledTemporaryFile(
3682 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3683 )
3685 def commit() -> Pack | None:
3686 if pf.tell() == 0:
3687 pf.close()
3688 return None
3690 pf.seek(0)
3692 p = PackData(pf.name, file=pf, object_format=self.object_format)
3693 entries = p.sorted_entries()
3694 basename = iter_sha1(entry[0] for entry in entries).decode("ascii")
3695 idxf = tempfile.SpooledTemporaryFile(
3696 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3697 )
3698 checksum = p.get_stored_checksum()
3699 write_pack_index(idxf, entries, checksum, version=self.pack_index_version)
3700 idxf.seek(0)
3701 idx = load_pack_index_file(basename + ".idx", idxf, self.object_format)
3702 for pack in self.packs:
3703 if pack.get_stored_checksum() == p.get_stored_checksum():
3704 p.close()
3705 idx.close()
3706 pf.close()
3707 idxf.close()
3708 return pack
3709 pf.seek(0)
3710 idxf.seek(0)
3711 self._upload_pack(basename, pf, idxf) # type: ignore[arg-type]
3712 final_pack = Pack.from_objects(p, idx)
3713 self._add_cached_pack(basename, final_pack)
3714 pf.close()
3715 idxf.close()
3716 return final_pack
3718 return pf, commit, pf.close # type: ignore[return-value]
3721def _collect_ancestors(
3722 store: ObjectContainer,
3723 heads: Iterable[ObjectID],
3724 common: frozenset[ObjectID] = frozenset(),
3725 shallow: frozenset[ObjectID] = frozenset(),
3726 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3727) -> tuple[set[ObjectID], set[ObjectID]]:
3728 """Collect all ancestors of heads up to (excluding) those in common.
3730 Args:
3731 store: Object store to get commits from
3732 heads: commits to start from
3733 common: commits to end at, or empty set to walk repository
3734 completely
3735 shallow: Set of shallow commits
3736 get_parents: Optional function for getting the parents of a
3737 commit.
3738 Returns: a tuple (A, B) where A - all commits reachable
3739 from heads but not present in common, B - common (shared) elements
3740 that are directly reachable from heads
3741 """
3742 bases = set()
3743 commits = set()
3744 queue: list[ObjectID] = []
3745 queue.extend(heads)
3747 # Try to use commit graph if available
3748 commit_graph = store.get_commit_graph()
3750 while queue:
3751 e = queue.pop(0)
3752 if e in common:
3753 bases.add(e)
3754 elif e not in commits:
3755 commits.add(e)
3756 if e in shallow:
3757 continue
3759 # Try to use commit graph for parent lookup
3760 parents = None
3761 if commit_graph:
3762 parents = commit_graph.get_parents(e)
3764 if parents is None:
3765 # Fall back to loading the object
3766 cmt = store[e]
3767 assert isinstance(cmt, Commit)
3768 parents = get_parents(cmt)
3770 queue.extend(parents)
3771 return (commits, bases)
3774def iter_tree_contents(
3775 store: ObjectContainer, tree_id: ObjectID | None, *, include_trees: bool = False
3776) -> Iterator[TreeEntry]:
3777 """Iterate the contents of a tree and all subtrees.
3779 Iteration is depth-first pre-order, as in e.g. os.walk.
3781 Args:
3782 store: Object store to get trees from
3783 tree_id: SHA1 of the tree.
3784 include_trees: If True, include tree objects in the iteration.
3786 Yields: TreeEntry namedtuples for all the objects in a tree.
3787 """
3788 if tree_id is None:
3789 return
3790 # This could be fairly easily generalized to >2 trees if we find a use
3791 # case.
3792 todo = [TreeEntry(b"", stat.S_IFDIR, tree_id)]
3793 while todo:
3794 entry = todo.pop()
3795 assert entry.mode is not None
3796 if stat.S_ISDIR(entry.mode):
3797 extra = []
3798 assert entry.sha is not None
3799 tree = store[entry.sha]
3800 assert isinstance(tree, Tree)
3801 for subentry in tree.iteritems(name_order=True):
3802 assert entry.path is not None
3803 extra.append(subentry.in_path(entry.path))
3804 todo.extend(reversed(extra))
3805 if not stat.S_ISDIR(entry.mode) or include_trees:
3806 yield entry
3809def iter_commit_contents(
3810 store: ObjectContainer,
3811 commit: Commit | ObjectID | RawObjectID,
3812 *,
3813 include: Sequence[str | bytes | Path] | None = None,
3814) -> Iterator[TreeEntry]:
3815 """Iterate the contents of the repository at the specified commit.
3817 This is a wrapper around iter_tree_contents() and
3818 tree_lookup_path() to simplify the common task of getting the
3819 contest of a repo at a particular commit. See also
3820 dulwich.index.build_file_from_blob() for writing individual files
3821 to disk.
3823 Args:
3824 store: Object store to get trees from
3825 commit: Commit object, or SHA1 of a commit
3826 include: if provided, only the entries whose paths are in the
3827 list, or whose parent tree is in the list, will be
3828 included. Note that duplicate or overlapping paths
3829 (e.g. ["foo", "foo/bar"]) may result in duplicate entries
3831 Yields: TreeEntry namedtuples for all matching files in a commit.
3832 """
3833 sha = commit.id if isinstance(commit, Commit) else commit
3834 if not isinstance(obj := store[sha], Commit):
3835 raise TypeError(
3836 f"{sha.decode('ascii')} should be ID of a Commit, but is {type(obj)}"
3837 )
3838 commit = obj
3839 encoding = commit.encoding or "utf-8"
3840 include_bytes: list[bytes] = (
3841 [
3842 path if isinstance(path, bytes) else str(path).encode(encoding)
3843 for path in include
3844 ]
3845 if include is not None
3846 else [b""]
3847 )
3849 for path in include_bytes:
3850 mode, obj_id = tree_lookup_path(store.__getitem__, commit.tree, path)
3851 # Iterate all contained files if path points to a dir, otherwise just get that
3852 # single file
3853 if isinstance(store[obj_id], Tree):
3854 for entry in iter_tree_contents(store, obj_id):
3855 yield entry.in_path(path)
3856 else:
3857 yield TreeEntry(path, mode, obj_id)
3860def peel_sha(
3861 store: ObjectContainer, sha: ObjectID | RawObjectID
3862) -> tuple[ShaFile, ShaFile]:
3863 """Peel all tags from a SHA.
3865 Args:
3866 store: Object store to get objects from
3867 sha: The object SHA to peel.
3868 Returns: The fully-peeled SHA1 of a tag object, after peeling all
3869 intermediate tags; if the original ref does not point to a tag,
3870 this will equal the original SHA1.
3871 """
3872 unpeeled = obj = store[sha]
3873 obj_class = object_class(obj.type_name)
3874 while obj_class is Tag:
3875 assert isinstance(obj, Tag)
3876 obj_class, sha = obj.object
3877 obj = store[sha]
3878 return unpeeled, obj
3881class GraphTraversalReachability:
3882 """Naive graph traversal implementation of ObjectReachabilityProvider.
3884 This implementation wraps existing graph traversal functions
3885 (_collect_ancestors, _collect_filetree_revs) to provide the standard
3886 reachability interface without any performance optimizations.
3887 """
3889 def __init__(self, object_store: BaseObjectStore) -> None:
3890 """Initialize the graph traversal provider.
3892 Args:
3893 object_store: Object store to query
3894 """
3895 self.store = object_store
3897 def get_reachable_commits(
3898 self,
3899 heads: Iterable[ObjectID],
3900 exclude: Iterable[ObjectID] | None = None,
3901 shallow: Set[ObjectID] | None = None,
3902 ) -> set[ObjectID]:
3903 """Get all commits reachable from heads, excluding those in exclude.
3905 Uses _collect_ancestors for commit traversal.
3907 Args:
3908 heads: Starting commit SHAs
3909 exclude: Commit SHAs to exclude (and their ancestors)
3910 shallow: Set of shallow commit boundaries
3912 Returns:
3913 Set of commit SHAs reachable from heads but not from exclude
3914 """
3915 exclude_set = frozenset(exclude) if exclude else frozenset()
3916 shallow_set = frozenset(shallow) if shallow else frozenset()
3917 commits, _bases = _collect_ancestors(
3918 self.store, heads, exclude_set, shallow_set
3919 )
3920 return commits
3922 def get_tree_objects(
3923 self,
3924 tree_shas: Iterable[ObjectID],
3925 ) -> set[ObjectID]:
3926 """Get all trees and blobs reachable from the given trees.
3928 Uses _collect_filetree_revs for tree traversal.
3930 Args:
3931 tree_shas: Starting tree SHAs
3933 Returns:
3934 Set of tree and blob SHAs
3935 """
3936 result: set[ObjectID] = set()
3937 for tree_sha in tree_shas:
3938 _collect_filetree_revs(self.store, tree_sha, result)
3939 return result
3941 def get_reachable_objects(
3942 self,
3943 commits: Iterable[ObjectID],
3944 exclude_commits: Iterable[ObjectID] | None = None,
3945 ) -> set[ObjectID]:
3946 """Get all objects (commits + trees + blobs) reachable from commits.
3948 Args:
3949 commits: Starting commit SHAs
3950 exclude_commits: Commits whose objects should be excluded
3952 Returns:
3953 Set of all object SHAs (commits, trees, blobs)
3954 """
3955 commits_set = set(commits)
3956 result = set(commits_set)
3958 # Get trees for all commits
3959 tree_shas = []
3960 for commit_sha in commits_set:
3961 try:
3962 commit = self.store[commit_sha]
3963 if isinstance(commit, Commit):
3964 tree_shas.append(commit.tree)
3965 except KeyError:
3966 # Commit not in store, skip
3967 continue
3969 # Collect all tree/blob objects
3970 result.update(self.get_tree_objects(tree_shas))
3972 # Exclude objects from exclude_commits if needed
3973 if exclude_commits:
3974 exclude_objects = self.get_reachable_objects(exclude_commits, None)
3975 result -= exclude_objects
3977 return result
3980class BitmapReachability:
3981 """Bitmap-accelerated implementation of ObjectReachabilityProvider.
3983 This implementation uses packfile bitmap indexes where available to
3984 accelerate reachability queries. Falls back to graph traversal when
3985 bitmaps don't cover the requested commits.
3986 """
3988 def __init__(self, object_store: "PackBasedObjectStore") -> None:
3989 """Initialize the bitmap provider.
3991 Args:
3992 object_store: Pack-based object store with bitmap support
3993 """
3994 self.store = object_store
3995 # Fallback to graph traversal for operations not yet optimized
3996 self._fallback = GraphTraversalReachability(object_store)
3998 def _combine_commit_bitmaps(
3999 self,
4000 commit_shas: set[ObjectID],
4001 exclude_shas: set[ObjectID] | None = None,
4002 ) -> tuple["EWAHBitmap", "Pack"] | None:
4003 """Combine bitmaps for multiple commits using OR, with optional exclusion.
4005 Args:
4006 commit_shas: Set of commit SHAs to combine
4007 exclude_shas: Optional set of commit SHAs to exclude
4009 Returns:
4010 Tuple of (combined_bitmap, pack) or None if bitmaps unavailable
4011 """
4012 from .bitmap import find_commit_bitmaps
4014 # Find bitmaps for the commits
4015 commit_bitmaps = find_commit_bitmaps(commit_shas, self.store.packs)
4017 # If we can't find bitmaps for all commits, return None
4018 if len(commit_bitmaps) < len(commit_shas):
4019 return None
4021 # Combine bitmaps using OR
4022 combined_bitmap = None
4023 result_pack = None
4025 for commit_sha in commit_shas:
4026 pack, pack_bitmap, _sha_to_pos = commit_bitmaps[commit_sha]
4027 commit_bitmap = pack_bitmap.get_bitmap(commit_sha)
4029 if commit_bitmap is None:
4030 return None
4032 if combined_bitmap is None:
4033 combined_bitmap = commit_bitmap
4034 result_pack = pack
4035 elif pack == result_pack:
4036 # Same pack, can OR directly
4037 combined_bitmap = combined_bitmap | commit_bitmap
4038 else:
4039 # Different packs, can't combine
4040 return None
4042 # Handle exclusions if provided
4043 if exclude_shas and result_pack and combined_bitmap:
4044 exclude_bitmaps = find_commit_bitmaps(exclude_shas, [result_pack])
4046 if len(exclude_bitmaps) == len(exclude_shas):
4047 # All excludes have bitmaps, compute exclusion
4048 exclude_combined = None
4050 for commit_sha in exclude_shas:
4051 _pack, pack_bitmap, _sha_to_pos = exclude_bitmaps[commit_sha]
4052 exclude_bitmap = pack_bitmap.get_bitmap(commit_sha)
4054 if exclude_bitmap is None:
4055 break
4057 if exclude_combined is None:
4058 exclude_combined = exclude_bitmap
4059 else:
4060 exclude_combined = exclude_combined | exclude_bitmap
4062 # Subtract excludes using set difference
4063 if exclude_combined:
4064 combined_bitmap = combined_bitmap - exclude_combined
4066 if combined_bitmap and result_pack:
4067 return (combined_bitmap, result_pack)
4068 return None
4070 def get_reachable_commits(
4071 self,
4072 heads: Iterable[ObjectID],
4073 exclude: Iterable[ObjectID] | None = None,
4074 shallow: Set[ObjectID] | None = None,
4075 ) -> set[ObjectID]:
4076 """Get all commits reachable from heads using bitmaps where possible.
4078 Args:
4079 heads: Starting commit SHAs
4080 exclude: Commit SHAs to exclude (and their ancestors)
4081 shallow: Set of shallow commit boundaries
4083 Returns:
4084 Set of commit SHAs reachable from heads but not from exclude
4085 """
4086 from .bitmap import bitmap_to_object_shas
4088 # If shallow is specified, fall back to graph traversal
4089 # (bitmaps don't support shallow boundaries well)
4090 if shallow:
4091 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4093 heads_set = set(heads)
4094 exclude_set = set(exclude) if exclude else None
4096 # Try to combine bitmaps
4097 result = self._combine_commit_bitmaps(heads_set, exclude_set)
4098 if result is None:
4099 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4101 combined_bitmap, result_pack = result
4103 # Convert bitmap to commit SHAs, filtering for commits only
4104 pack_bitmap = result_pack.bitmap
4105 if pack_bitmap is None:
4106 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4107 commit_type_filter = pack_bitmap.commit_bitmap
4108 return bitmap_to_object_shas(
4109 combined_bitmap, result_pack.index, commit_type_filter
4110 )
4112 def get_tree_objects(
4113 self,
4114 tree_shas: Iterable[ObjectID],
4115 ) -> set[ObjectID]:
4116 """Get all trees and blobs reachable from the given trees.
4118 Args:
4119 tree_shas: Starting tree SHAs
4121 Returns:
4122 Set of tree and blob SHAs
4123 """
4124 # Tree traversal doesn't benefit much from bitmaps, use fallback
4125 return self._fallback.get_tree_objects(tree_shas)
4127 def get_reachable_objects(
4128 self,
4129 commits: Iterable[ObjectID],
4130 exclude_commits: Iterable[ObjectID] | None = None,
4131 ) -> set[ObjectID]:
4132 """Get all objects reachable from commits using bitmaps.
4134 Args:
4135 commits: Starting commit SHAs
4136 exclude_commits: Commits whose objects should be excluded
4138 Returns:
4139 Set of all object SHAs (commits, trees, blobs)
4140 """
4141 from .bitmap import bitmap_to_object_shas
4143 commits_set = set(commits)
4144 exclude_set = set(exclude_commits) if exclude_commits else None
4146 # Try to combine bitmaps
4147 result = self._combine_commit_bitmaps(commits_set, exclude_set)
4148 if result is None:
4149 return self._fallback.get_reachable_objects(commits, exclude_commits)
4151 combined_bitmap, result_pack = result
4153 # Convert bitmap to all object SHAs (no type filter)
4154 return bitmap_to_object_shas(combined_bitmap, result_pack.index, None)