Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/object_store.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# object_store.py -- Object store for git objects
2# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
3# and others
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
24"""Git object store interfaces and implementation."""
26__all__ = [
27 "DEFAULT_TEMPFILE_GRACE_PERIOD",
28 "INFODIR",
29 "PACKDIR",
30 "PACK_MODE",
31 "PEEL_CACHE_SIZE",
32 "BaseObjectStore",
33 "BitmapReachability",
34 "BucketBasedObjectStore",
35 "DiskObjectStore",
36 "GraphTraversalReachability",
37 "GraphWalker",
38 "MemoryObjectStore",
39 "MissingObjectFinder",
40 "ObjectIterator",
41 "ObjectReachabilityProvider",
42 "ObjectStoreGraphWalker",
43 "OverlayObjectStore",
44 "PackBasedObjectStore",
45 "PackCapableObjectStore",
46 "PackContainer",
47 "PackInputTooLarge",
48 "PeelCache",
49 "commit_tree_changes",
50 "find_shallow",
51 "get_depth",
52 "iter_commit_contents",
53 "iter_tree_contents",
54 "peel_sha",
55 "read_packs_file",
56 "tree_lookup_path",
57]
59import binascii
60import logging
61import os
62import stat
63import sys
64import time
65import warnings
66from collections import deque
67from collections.abc import (
68 Callable,
69 Iterable,
70 Iterator,
71 Mapping,
72 Sequence,
73 Set,
74)
75from contextlib import closing, suppress
76from io import BytesIO
77from pathlib import Path
78from typing import (
79 TYPE_CHECKING,
80 BinaryIO,
81 Protocol,
82 TypeVar,
83 cast,
84)
86if TYPE_CHECKING:
87 from .object_format import ObjectFormat
89from .errors import NotTreeError
90from .file import GitFile, SharedPerm, _GitFile, adjust_shared_perm
91from .lru_cache import LRUCache
92from .midx import MultiPackIndex, load_midx
93from .objects import (
94 DEFAULT_LOOSE_OBJECT_SIZE_LIMIT,
95 S_ISGITLINK,
96 Blob,
97 Commit,
98 ObjectID,
99 RawObjectID,
100 ShaFile,
101 Tag,
102 Tree,
103 TreeEntry,
104 hex_to_filename,
105 hex_to_sha,
106 object_class,
107 sha_to_hex,
108 valid_hexsha,
109)
110from .pack import (
111 PACK_SPOOL_FILE_MAX_SIZE,
112 ObjectContainer,
113 Pack,
114 PackData,
115 PackedObjectContainer,
116 PackFileDisappeared,
117 PackHint,
118 PackIndexEntry,
119 PackIndexer,
120 PackInflater,
121 PackStreamCopier,
122 UnpackedObject,
123 extend_pack,
124 full_unpacked_object,
125 generate_unpacked_objects,
126 iter_sha1,
127 load_pack_index_file,
128 pack_objects_to_data,
129 write_pack_data,
130 write_pack_index,
131)
132from .protocol import DEPTH_INFINITE, PEELED_TAG_SUFFIX
133from .refs import Ref
135if TYPE_CHECKING:
136 from .bitmap import EWAHBitmap
137 from .commit_graph import CommitGraph
138 from .config import Config
139 from .diff_tree import RenameDetector
140 from .pack import FilePackIndex, Pack
143logger = logging.getLogger(__name__)
145# Maximum number of times to rescan the pack directory after a pack file
146# disappears between snapshot and lazy open (e.g. concurrent repack).
147# Mirrors git's bounded reprepare_packed_git() retry.
148_MAX_PACK_RESCAN_ATTEMPTS = 3
150# Number of tag-peel results to memoize per object store. Entries are tiny
151# (two object ids), so this mainly bounds growth in long-lived processes.
152PEEL_CACHE_SIZE = 1000
155class PeelCache(Protocol):
156 """Minimal mapping interface peel_sha() needs from a cache."""
158 def get(self, key: ObjectID, default: None = None) -> ObjectID | None:
159 """Return the peeled id for key, or None."""
161 def __setitem__(self, key: ObjectID, value: ObjectID) -> None:
162 """Record the peeled id for key."""
165_T = TypeVar("_T")
168class GraphWalker(Protocol):
169 """Protocol for graph walker objects.
171 Implementations may also expose a ``shallow`` set, an ``unshallow`` set,
172 and an ``update_shallow`` callable for shallow-clone negotiation. These
173 are not part of the minimal protocol and callers must use ``hasattr`` or
174 ``getattr`` to access them.
175 """
177 def __next__(self) -> ObjectID | None:
178 """Return the next object SHA to visit."""
179 ...
181 def ack(self, sha: ObjectID, /) -> None:
182 """Acknowledge that an object has been received."""
183 ...
185 def nak(self) -> None:
186 """Nothing in common was found."""
187 ...
190class ObjectReachabilityProvider(Protocol):
191 """Protocol for computing object reachability queries.
193 This abstraction allows reachability computations to be backed by either
194 naive graph traversal or optimized bitmap indexes, with a consistent interface.
195 """
197 def get_reachable_commits(
198 self,
199 heads: Iterable[ObjectID],
200 exclude: Iterable[ObjectID] | None = None,
201 shallow: Set[ObjectID] | None = None,
202 ) -> set[ObjectID]:
203 """Get all commits reachable from heads, excluding those in exclude.
205 Args:
206 heads: Starting commit SHAs
207 exclude: Commit SHAs to exclude (and their ancestors)
208 shallow: Set of shallow commit boundaries (traversal stops here)
210 Returns:
211 Set of commit SHAs reachable from heads but not from exclude
212 """
213 ...
215 def get_reachable_objects(
216 self,
217 commits: Iterable[ObjectID],
218 exclude_commits: Iterable[ObjectID] | None = None,
219 ) -> set[ObjectID]:
220 """Get all objects (commits + trees + blobs) reachable from commits.
222 Args:
223 commits: Starting commit SHAs
224 exclude_commits: Commits whose objects should be excluded
226 Returns:
227 Set of all object SHAs (commits, trees, blobs, tags)
228 """
229 ...
231 def get_tree_objects(
232 self,
233 tree_shas: Iterable[ObjectID],
234 ) -> set[ObjectID]:
235 """Get all trees and blobs reachable from the given trees.
237 Args:
238 tree_shas: Starting tree SHAs
240 Returns:
241 Set of tree and blob SHAs
242 """
243 ...
246INFODIR = "info"
247PACKDIR = "pack"
249# use permissions consistent with Git; just readable by everyone
250# TODO: should packs also be non-writable on Windows? if so, that
251# would requite some rather significant adjustments to the test suite
252PACK_MODE = 0o444 if sys.platform != "win32" else 0o644
254# Grace period for cleaning up temporary pack files (in seconds)
255# Matches git's default of 2 weeks
256DEFAULT_TEMPFILE_GRACE_PERIOD = 14 * 24 * 60 * 60 # 2 weeks
259def _remove_readonly(path: str) -> None:
260 """Remove a file, clearing the read-only attribute first on Windows.
262 git stores pack files and loose objects read-only. Unix lets you unlink a
263 read-only file in a writable directory, but Windows refuses with
264 PermissionError, so clear the attribute and retry there.
265 """
266 try:
267 os.remove(path)
268 except PermissionError:
269 if sys.platform != "win32":
270 raise
271 os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
272 os.remove(path)
275class PackInputTooLarge(OSError):
276 """Raised when a received pack exceeds the configured input size cap.
278 Mirrors the failure mode of git's ``receive.maxInputSize`` /
279 ``git index-pack --max-input-size``.
280 """
283def _bound_read_callables(
284 read_all: Callable[[int], bytes],
285 read_some: Callable[[int], bytes] | None,
286 max_input_size: int,
287) -> tuple[Callable[[int], bytes], Callable[[int], bytes] | None]:
288 """Wrap pack-stream read callbacks so total bytes are capped.
290 When the cumulative number of bytes returned across ``read_all`` and
291 ``read_some`` exceeds ``max_input_size``, the next read raises
292 ``PackInputTooLarge``. This is the in-process analogue of
293 ``git index-pack --max-input-size``.
294 """
295 bytes_read = [0]
297 def _check(n: int) -> None:
298 bytes_read[0] += n
299 if bytes_read[0] > max_input_size:
300 raise PackInputTooLarge(
301 f"pack exceeds maximum input size of {max_input_size} bytes"
302 )
304 def wrapped_read_all(n: int) -> bytes:
305 data = read_all(n)
306 _check(len(data))
307 return data
309 if read_some is None:
310 return wrapped_read_all, None
312 def wrapped_read_some(n: int) -> bytes:
313 data = read_some(n)
314 _check(len(data))
315 return data
317 return wrapped_read_all, wrapped_read_some
320def find_shallow(
321 store: ObjectContainer, heads: Iterable[ObjectID], depth: int
322) -> tuple[set[ObjectID], set[ObjectID]]:
323 """Find shallow commits according to a given depth.
325 Args:
326 store: An ObjectStore for looking up objects.
327 heads: Iterable of head SHAs to start walking from.
328 depth: The depth of ancestors to include. A depth of one includes
329 only the heads themselves.
330 Returns: A tuple of (shallow, not_shallow), sets of SHAs that should be
331 considered shallow and unshallow according to the arguments. Note that
332 these sets may overlap if a commit is reachable along multiple paths.
333 """
334 parents: dict[ObjectID, list[ObjectID]] = {}
335 commit_graph = store.get_commit_graph()
337 def get_parents(sha: ObjectID) -> list[ObjectID]:
338 result = parents.get(sha, None)
339 if not result:
340 # Try to use commit graph first if available
341 if commit_graph:
342 graph_parents = commit_graph.get_parents(sha)
343 if graph_parents is not None:
344 result = graph_parents
345 parents[sha] = result
346 return result
347 # Fall back to loading the object
348 commit = store[sha]
349 assert isinstance(commit, Commit)
350 result = commit.parents
351 parents[sha] = result
352 return result
354 todo = [] # stack of (sha, depth)
355 for head_sha in heads:
356 obj = store[head_sha]
357 # Peel tags if necessary
358 while isinstance(obj, Tag):
359 _, sha = obj.object
360 obj = store[sha]
361 if isinstance(obj, Commit):
362 todo.append((obj.id, 1))
364 not_shallow = set()
365 shallow = set()
366 # A commit reachable along N distinct paths was popped and re-expanded N
367 # times, so a merge-heavy history walked in exponential time. Deduplicate
368 # on the (sha, depth) state: re-processing a state adds to the same set and
369 # pushes the same parents, so skipping repeats leaves the result unchanged.
370 seen: set[tuple[ObjectID, int]] = set()
371 while todo:
372 state = todo.pop()
373 if state in seen:
374 continue
375 seen.add(state)
376 sha, cur_depth = state
377 if cur_depth < depth:
378 not_shallow.add(sha)
379 new_depth = cur_depth + 1
380 todo.extend((p, new_depth) for p in get_parents(sha))
381 else:
382 shallow.add(sha)
384 return shallow, not_shallow
387def get_depth(
388 store: ObjectContainer,
389 head: ObjectID,
390 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
391 max_depth: int | None = None,
392) -> int:
393 """Return the current available depth for the given head.
395 For commits with multiple parents, the largest possible depth will be
396 returned.
398 Args:
399 store: Object store to search in
400 head: commit to start from
401 get_parents: optional function for getting the parents of a commit
402 max_depth: maximum depth to search
403 """
404 if head not in store:
405 return 0
406 current_depth = 1
407 queue = deque([(head, current_depth)])
408 commit_graph = store.get_commit_graph()
410 # Without deduplication a commit reachable along several paths is expanded
411 # once per path, so a merge-heavy history is walked in exponential time.
412 # Track the (sha, depth) states already queued; re-visiting one only
413 # recomputes the same max and re-queues the same parents. deque.popleft()
414 # keeps the O(1) breadth-first order that list.pop(0) made O(n).
415 seen: set[tuple[ObjectID, int]] = set()
416 while queue and (max_depth is None or current_depth < max_depth):
417 e, depth = queue.popleft()
418 if (e, depth) in seen:
419 continue
420 seen.add((e, depth))
421 current_depth = max(current_depth, depth)
423 # Try to use commit graph for parent lookup if available
424 parents = None
425 if commit_graph:
426 parents = commit_graph.get_parents(e)
428 if parents is None:
429 # Fall back to loading the object
430 cmt = store[e]
431 if isinstance(cmt, Tag):
432 _cls, sha = cmt.object
433 cmt = store[sha]
434 parents = get_parents(cmt)
436 queue.extend((parent, depth + 1) for parent in parents if parent in store)
437 return current_depth
440class PackContainer(Protocol):
441 """Protocol for containers that can accept pack files."""
443 def add_pack(self) -> tuple[BytesIO, Callable[[], None], Callable[[], None]]:
444 """Add a new pack."""
445 ...
448class BaseObjectStore:
449 """Object store interface."""
451 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
452 """Initialize object store.
454 Args:
455 object_format: Object format to use (defaults to DEFAULT_OBJECT_FORMAT)
456 """
457 from .object_format import DEFAULT_OBJECT_FORMAT
459 self.object_format = object_format if object_format else DEFAULT_OBJECT_FORMAT
461 def peel(self, sha: ObjectID | RawObjectID) -> tuple[ShaFile, ShaFile]:
462 """Peel all tags from a SHA, memoizing shared tag chains.
464 Peeling each ref separately is quadratic when many refs share one tag
465 chain, so results are kept in a bounded cache on the store. Git objects
466 are immutable, so an entry can only go stale if the objects behind it
467 are deleted; the deletion paths call _invalidate_peel_cache().
469 Args:
470 sha: The object SHA to peel.
471 Returns: Tuple of (unpeeled, peeled) objects.
472 """
473 return peel_sha(self, sha, cache=self._peel_cache)
475 @property
476 def _peel_cache(self) -> "LRUCache[ObjectID, ObjectID]":
477 cache: LRUCache[ObjectID, ObjectID] | None = getattr(
478 self, "_BaseObjectStore__peel_cache", None
479 )
480 if cache is None:
481 cache = LRUCache(max_cache=PEEL_CACHE_SIZE)
482 self.__peel_cache = cache
483 return cache
485 def _invalidate_peel_cache(self) -> None:
486 """Drop memoized peel results, after objects may have been removed."""
487 self._peel_cache.clear()
489 def determine_wants_all(
490 self, refs: Mapping[Ref, ObjectID], depth: int | None = None
491 ) -> list[ObjectID]:
492 """Determine which objects are wanted based on refs."""
494 def _want_deepen(sha: ObjectID) -> bool:
495 if not depth:
496 return False
497 if depth == DEPTH_INFINITE:
498 return True
499 return depth > self._get_depth(sha)
501 return [
502 sha
503 for (ref, sha) in refs.items()
504 if (sha not in self or _want_deepen(sha))
505 and not ref.endswith(PEELED_TAG_SUFFIX)
506 ]
508 def contains_loose(self, sha: ObjectID) -> bool:
509 """Check if a particular object is present by SHA1 and is loose."""
510 raise NotImplementedError(self.contains_loose)
512 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
513 """Check if a particular object is present by SHA1 and is packed."""
514 return False # Default implementation for stores that don't support packing
516 def __contains__(self, sha1: ObjectID) -> bool:
517 """Check if a particular object is present by SHA1.
519 This method makes no distinction between loose and packed objects.
520 """
521 return self.contains_loose(sha1)
523 @property
524 def packs(self) -> list[Pack]:
525 """Iterable of pack objects."""
526 raise NotImplementedError
528 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
529 """Obtain the raw text for an object.
531 Args:
532 name: sha for the object.
533 Returns: tuple with numeric type and object contents.
534 """
535 raise NotImplementedError(self.get_raw)
537 def __getitem__(self, sha1: ObjectID | RawObjectID) -> ShaFile:
538 """Obtain an object by SHA1.
540 Raises:
541 ChecksumMismatch: if the stored contents do not hash to the
542 requested object id.
543 """
544 if len(sha1) == self.object_format.oid_length:
545 hexsha = sha_to_hex(RawObjectID(sha1))
546 else:
547 hexsha = ObjectID(sha1)
548 type_num, uncomp = self.get_raw(sha1)
549 return ShaFile.from_raw_string(
550 type_num, uncomp, verify_sha=hexsha, object_format=self.object_format
551 )
553 def __iter__(self) -> Iterator[ObjectID]:
554 """Iterate over the SHAs that are present in this store."""
555 raise NotImplementedError(self.__iter__)
557 def add_object(self, obj: ShaFile) -> None:
558 """Add a single object to this object store."""
559 raise NotImplementedError(self.add_object)
561 def add_objects(
562 self,
563 objects: Sequence[tuple[ShaFile, str | None]],
564 progress: Callable[..., None] | None = None,
565 ) -> "Pack | None":
566 """Add a set of objects to this object store.
568 Args:
569 objects: Iterable over a list of (object, path) tuples
570 progress: Optional progress callback
571 """
572 raise NotImplementedError(self.add_objects)
574 def get_reachability_provider(
575 self, prefer_bitmaps: bool = True
576 ) -> ObjectReachabilityProvider:
577 """Get a reachability provider for this object store.
579 Returns an ObjectReachabilityProvider that can efficiently compute
580 object reachability queries. Subclasses can override this to provide
581 optimized implementations (e.g., using bitmap indexes).
583 Args:
584 prefer_bitmaps: Whether to prefer bitmap-based reachability if
585 available.
587 Returns:
588 ObjectReachabilityProvider instance
589 """
590 return GraphTraversalReachability(self)
592 def tree_changes(
593 self,
594 source: ObjectID | None,
595 target: ObjectID | None,
596 want_unchanged: bool = False,
597 include_trees: bool = False,
598 change_type_same: bool = False,
599 rename_detector: "RenameDetector | None" = None,
600 paths: Sequence[bytes] | None = None,
601 ) -> Iterator[
602 tuple[
603 tuple[bytes | None, bytes | None],
604 tuple[int | None, int | None],
605 tuple[ObjectID | None, ObjectID | None],
606 ]
607 ]:
608 """Find the differences between the contents of two trees.
610 Args:
611 source: SHA1 of the source tree
612 target: SHA1 of the target tree
613 want_unchanged: Whether unchanged files should be reported
614 include_trees: Whether to include trees
615 change_type_same: Whether to report files changing
616 type in the same entry.
617 rename_detector: RenameDetector object for detecting renames.
618 paths: Optional list of paths to filter to (as bytes).
619 Returns: Iterator over tuples with
620 (oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
621 """
622 from .diff_tree import tree_changes
624 for change in tree_changes(
625 self,
626 source,
627 target,
628 want_unchanged=want_unchanged,
629 include_trees=include_trees,
630 change_type_same=change_type_same,
631 rename_detector=rename_detector,
632 paths=paths,
633 ):
634 old_path = change.old.path if change.old is not None else None
635 new_path = change.new.path if change.new is not None else None
636 old_mode = change.old.mode if change.old is not None else None
637 new_mode = change.new.mode if change.new is not None else None
638 old_sha = change.old.sha if change.old is not None else None
639 new_sha = change.new.sha if change.new is not None else None
640 yield (
641 (old_path, new_path),
642 (old_mode, new_mode),
643 (old_sha, new_sha),
644 )
646 def iter_tree_contents(
647 self, tree_id: ObjectID, include_trees: bool = False
648 ) -> Iterator[TreeEntry]:
649 """Iterate the contents of a tree and all subtrees.
651 Iteration is depth-first pre-order, as in e.g. os.walk.
653 Args:
654 tree_id: SHA1 of the tree.
655 include_trees: If True, include tree objects in the iteration.
656 Returns: Iterator over TreeEntry namedtuples for all the objects in a
657 tree.
658 """
659 warnings.warn(
660 "Please use dulwich.object_store.iter_tree_contents",
661 DeprecationWarning,
662 stacklevel=2,
663 )
664 return iter_tree_contents(self, tree_id, include_trees=include_trees)
666 def iterobjects_subset(
667 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
668 ) -> Iterator[ShaFile]:
669 """Iterate over a subset of objects in the store.
671 Args:
672 shas: Iterable of object SHAs to retrieve
673 allow_missing: If True, skip missing objects; if False, raise KeyError
675 Returns:
676 Iterator of ShaFile objects
678 Raises:
679 KeyError: If an object is missing and allow_missing is False
680 """
681 for sha in shas:
682 try:
683 yield self[sha]
684 except KeyError:
685 if not allow_missing:
686 raise
688 def iter_unpacked_subset(
689 self,
690 shas: Iterable[ObjectID | RawObjectID],
691 *,
692 include_comp: bool = False,
693 allow_missing: bool = False,
694 convert_ofs_delta: bool = True,
695 ) -> "Iterator[UnpackedObject]":
696 """Iterate over unpacked objects for a subset of SHAs.
698 Default implementation that converts ShaFile objects to UnpackedObject.
699 Subclasses may override for more efficient unpacked access.
701 Args:
702 shas: Iterable of object SHAs to retrieve
703 include_comp: Whether to include compressed data (ignored in base
704 implementation)
705 allow_missing: If True, skip missing objects; if False, raise
706 KeyError
707 convert_ofs_delta: Whether to convert OFS_DELTA objects (ignored in
708 base implementation)
710 Returns:
711 Iterator of UnpackedObject instances
713 Raises:
714 KeyError: If an object is missing and allow_missing is False
715 """
716 from .pack import UnpackedObject
718 for sha in shas:
719 try:
720 obj = self[sha]
721 # Convert ShaFile to UnpackedObject
722 unpacked = UnpackedObject(
723 obj.type_num, decomp_chunks=obj.as_raw_chunks(), sha=obj.id
724 )
725 yield unpacked
726 except KeyError:
727 if not allow_missing:
728 raise
730 def find_missing_objects(
731 self,
732 haves: Iterable[ObjectID],
733 wants: Iterable[ObjectID],
734 shallow: Set[ObjectID] | None = None,
735 progress: Callable[..., None] | None = None,
736 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
737 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
738 ) -> Iterator[tuple[ObjectID, PackHint | None]]:
739 """Find the missing objects required for a set of revisions.
741 Args:
742 haves: Iterable over SHAs already in common.
743 wants: Iterable over SHAs of objects to fetch.
744 shallow: Set of shallow commit SHA1s to skip
745 progress: Simple progress function that will be called with
746 updated progress strings.
747 get_tagged: Function that returns a dict of pointed-to sha ->
748 tag sha for including tags.
749 get_parents: Optional function for getting the parents of a
750 commit.
751 Returns: Iterator over (sha, path) pairs.
752 """
753 warnings.warn("Please use MissingObjectFinder(store)", DeprecationWarning)
754 finder = MissingObjectFinder(
755 self,
756 haves=haves,
757 wants=wants,
758 shallow=shallow,
759 progress=progress,
760 get_tagged=get_tagged,
761 get_parents=get_parents,
762 )
763 return iter(finder)
765 def find_common_revisions(self, graphwalker: GraphWalker) -> list[ObjectID]:
766 """Find which revisions this store has in common using graphwalker.
768 Args:
769 graphwalker: A graphwalker object.
770 Returns: List of SHAs that are in common
771 """
772 haves = []
773 sha = next(graphwalker)
774 while sha:
775 if sha in self:
776 haves.append(sha)
777 graphwalker.ack(sha)
778 sha = next(graphwalker)
779 return haves
781 def generate_pack_data(
782 self,
783 have: Iterable[ObjectID],
784 want: Iterable[ObjectID],
785 *,
786 shallow: Set[ObjectID] | None = None,
787 progress: Callable[..., None] | None = None,
788 ofs_delta: bool = True,
789 ) -> tuple[int, Iterator[UnpackedObject]]:
790 """Generate pack data objects for a set of wants/haves.
792 Args:
793 have: List of SHA1s of objects that should not be sent
794 want: List of SHA1s of objects that should be sent
795 shallow: Set of shallow commit SHA1s to skip
796 ofs_delta: Whether OFS deltas can be included
797 progress: Optional progress reporting method
798 """
799 # Note that the pack-specific implementation below is more efficient,
800 # as it reuses deltas
801 missing_objects = MissingObjectFinder(
802 self, haves=have, wants=want, shallow=shallow, progress=progress
803 )
804 object_ids = list(missing_objects)
805 return pack_objects_to_data(
806 [(self[oid], path) for oid, path in object_ids],
807 ofs_delta=ofs_delta,
808 progress=progress,
809 )
811 def peel_sha(self, sha: ObjectID | RawObjectID) -> ObjectID:
812 """Peel all tags from a SHA.
814 Args:
815 sha: The object SHA to peel.
816 Returns: The fully-peeled SHA1 of a tag object, after peeling all
817 intermediate tags; if the original ref does not point to a tag,
818 this will equal the original SHA1.
819 """
820 warnings.warn(
821 "Please use dulwich.object_store.peel_sha()",
822 DeprecationWarning,
823 stacklevel=2,
824 )
825 return peel_sha(self, sha)[1].id
827 def _get_depth(
828 self,
829 head: ObjectID,
830 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
831 max_depth: int | None = None,
832 ) -> int:
833 """Return the current available depth for the given head.
835 For commits with multiple parents, the largest possible depth will be
836 returned.
838 Args:
839 head: commit to start from
840 get_parents: optional function for getting the parents of a commit
841 max_depth: maximum depth to search
842 """
843 return get_depth(self, head, get_parents=get_parents, max_depth=max_depth)
845 def close(self) -> None:
846 """Close any files opened by this object store."""
847 # Default implementation is a NO-OP
849 def prune(self, grace_period: int | None = None) -> None:
850 """Prune/clean up this object store.
852 This includes removing orphaned temporary files and other
853 housekeeping tasks. Default implementation is a NO-OP.
855 Args:
856 grace_period: Grace period in seconds for removing temporary files.
857 If None, uses the default grace period.
858 """
859 # Default implementation is a NO-OP
861 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
862 """Iterate over all SHA1s that start with a given prefix.
864 The default implementation is a naive iteration over all objects.
865 However, subclasses may override this method with more efficient
866 implementations.
867 """
868 for sha in self:
869 if sha.startswith(prefix):
870 yield sha
872 def get_commit_graph(self) -> "CommitGraph | None":
873 """Get the commit graph for this object store.
875 Returns:
876 CommitGraph object if available, None otherwise
877 """
878 return None
880 def write_commit_graph(
881 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
882 ) -> None:
883 """Write a commit graph file for this object store.
885 Args:
886 refs: List of refs to include. If None, includes all refs from object store.
887 reachable: If True, includes all commits reachable from refs.
888 If False, only includes the direct ref targets.
890 Note:
891 Default implementation does nothing. Subclasses should override
892 this method to provide commit graph writing functionality.
893 """
894 raise NotImplementedError(self.write_commit_graph)
896 def get_object_mtime(self, sha: ObjectID) -> float:
897 """Get the modification time of an object.
899 Args:
900 sha: SHA1 of the object
902 Returns:
903 Modification time as seconds since epoch
905 Raises:
906 KeyError: if the object is not found
907 """
908 # Default implementation raises KeyError
909 # Subclasses should override to provide actual mtime
910 raise KeyError(sha)
913class PackCapableObjectStore(BaseObjectStore, PackedObjectContainer):
914 """Object store that supports pack operations.
916 This is a base class for object stores that can handle pack files,
917 including both disk-based and memory-based stores.
918 """
920 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
921 """Add a new pack to this object store.
923 Returns: Tuple of (file, commit_func, abort_func)
924 """
925 raise NotImplementedError(self.add_pack)
927 def add_pack_data(
928 self,
929 count: int,
930 unpacked_objects: Iterator["UnpackedObject"],
931 progress: Callable[..., None] | None = None,
932 ) -> "Pack | None":
933 """Add pack data to this object store.
935 Args:
936 count: Number of objects
937 unpacked_objects: Iterator over unpacked objects
938 progress: Optional progress callback
939 """
940 raise NotImplementedError(self.add_pack_data)
942 def get_unpacked_object(
943 self, sha1: ObjectID | RawObjectID, *, include_comp: bool = False
944 ) -> "UnpackedObject":
945 """Get a raw unresolved object.
947 Args:
948 sha1: SHA-1 hash of the object
949 include_comp: Whether to include compressed data
951 Returns:
952 UnpackedObject instance
953 """
954 from .pack import UnpackedObject
956 obj = self[sha1]
957 return UnpackedObject(obj.type_num, sha=sha1, decomp_chunks=obj.as_raw_chunks())
959 def iterobjects_subset(
960 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
961 ) -> Iterator[ShaFile]:
962 """Iterate over a subset of objects.
964 Args:
965 shas: Iterable of object SHAs to retrieve
966 allow_missing: If True, skip missing objects
968 Returns:
969 Iterator of ShaFile objects
970 """
971 for sha in shas:
972 try:
973 yield self[sha]
974 except KeyError:
975 if not allow_missing:
976 raise
979class PackBasedObjectStore(PackCapableObjectStore, PackedObjectContainer):
980 """Object store that uses pack files for storage.
982 This class provides a base implementation for object stores that use
983 Git pack files as their primary storage mechanism. It handles caching
984 of open pack files and provides configuration for pack file operations.
985 """
987 def __init__(
988 self,
989 pack_compression_level: int = -1,
990 pack_index_version: int | None = None,
991 pack_delta_window_size: int | None = None,
992 pack_window_memory: int | None = None,
993 pack_delta_cache_size: int | None = None,
994 pack_depth: int | None = None,
995 pack_threads: int | None = None,
996 pack_big_file_threshold: int | None = None,
997 *,
998 packed_git_limit: int | None = None,
999 delta_base_cache_limit: int | None = None,
1000 object_format: "ObjectFormat | None" = None,
1001 ) -> None:
1002 """Initialize a PackBasedObjectStore.
1004 Args:
1005 pack_compression_level: Compression level for pack files (-1 to 9)
1006 pack_index_version: Pack index version to use
1007 pack_delta_window_size: Window size for delta compression
1008 pack_window_memory: Maximum memory to use for delta window
1009 pack_delta_cache_size: Cache size for delta operations
1010 pack_depth: Maximum depth for pack deltas
1011 pack_threads: Number of threads to use for packing
1012 pack_big_file_threshold: Threshold for treating files as "big"
1013 packed_git_limit: Maximum total bytes for mmapped pack files.
1014 When exceeded, least-recently-used packs are closed to free memory.
1015 delta_base_cache_limit: Maximum bytes for caching delta base objects.
1016 Controls memory used to cache resolved base objects during delta
1017 unpacking, corresponding to Git's core.deltaBaseCacheLimit.
1018 object_format: Hash algorithm to use
1019 """
1020 super().__init__(object_format=object_format)
1021 self._pack_cache: dict[str, Pack] = {}
1022 self._pack_access_order: list[str] = []
1023 self.packed_git_limit = packed_git_limit
1024 self.delta_base_cache_limit = delta_base_cache_limit
1025 self.pack_compression_level = pack_compression_level
1026 self.pack_index_version = pack_index_version
1027 self.pack_delta_window_size = pack_delta_window_size
1028 self.pack_window_memory = pack_window_memory
1029 self.pack_delta_cache_size = pack_delta_cache_size
1030 self.pack_depth = pack_depth
1031 self.pack_threads = pack_threads
1032 self.pack_big_file_threshold = pack_big_file_threshold
1034 def get_reachability_provider(
1035 self,
1036 prefer_bitmaps: bool = True,
1037 ) -> ObjectReachabilityProvider:
1038 """Get the best reachability provider for the object store.
1040 Args:
1041 prefer_bitmaps: Whether to use bitmaps if available
1043 Returns:
1044 ObjectReachabilityProvider implementation (either bitmap-accelerated
1045 or graph traversal)
1046 """
1047 if prefer_bitmaps:
1048 # Check if any packs have bitmaps. ``self.packs`` rescans the pack
1049 # directory, so a pack removed by a concurrent repack is dropped
1050 # from the cache and its replacement, if any, is probed here too.
1051 for pack in self.packs:
1052 try:
1053 if pack.bitmap is not None:
1054 return BitmapReachability(self)
1055 except FileNotFoundError:
1056 # Bitmap file doesn't exist for this pack
1057 continue
1058 except PackFileDisappeared as exc:
1059 # The pack vanished between the scan and the bitmap probe.
1060 self._evict_pack(exc.obj)
1061 continue
1063 # Fall back to graph traversal
1064 return GraphTraversalReachability(self)
1066 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
1067 """Add a new pack to this object store."""
1068 raise NotImplementedError(self.add_pack)
1070 def add_pack_data(
1071 self,
1072 count: int,
1073 unpacked_objects: Iterator[UnpackedObject],
1074 progress: Callable[..., None] | None = None,
1075 ) -> "Pack | None":
1076 """Add pack data to this object store.
1078 Args:
1079 count: Number of items to add
1080 unpacked_objects: Iterator of UnpackedObject instances
1081 progress: Optional progress callback
1082 """
1083 if count == 0:
1084 # Don't bother writing an empty pack file
1085 return None
1086 f, commit, abort = self.add_pack()
1087 try:
1088 write_pack_data(
1089 f.write,
1090 unpacked_objects,
1091 num_records=count,
1092 progress=progress,
1093 compression_level=self.pack_compression_level,
1094 object_format=self.object_format,
1095 )
1096 except BaseException:
1097 abort()
1098 raise
1099 else:
1100 return commit()
1102 @property
1103 def alternates(self) -> list["BaseObjectStore"]:
1104 """Return list of alternate object stores."""
1105 return []
1107 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
1108 """Check if a particular object is present by SHA1 and is packed.
1110 This does not check alternates.
1111 """
1113 def lookup(p: "Pack") -> bool:
1114 if sha in p:
1115 return True
1116 raise KeyError
1118 try:
1119 return self._lookup_in_packs(lookup)
1120 except KeyError:
1121 return False
1123 def __contains__(self, sha: ObjectID) -> bool:
1124 """Check if a particular object is present by SHA1.
1126 This method makes no distinction between loose and packed objects.
1127 """
1128 if self.contains_packed(sha) or self.contains_loose(sha):
1129 return True
1130 for alternate in self.alternates:
1131 if sha in alternate:
1132 return True
1133 return False
1135 def _add_cached_pack(self, base_name: str, pack: Pack) -> None:
1136 """Add a newly appeared pack to the cache by path."""
1137 prev_pack = self._pack_cache.get(base_name)
1138 if prev_pack is not pack:
1139 self._pack_cache[base_name] = pack
1140 if prev_pack:
1141 prev_pack.close()
1142 self._mark_pack_used(base_name)
1143 self._enforce_packed_git_limit()
1145 def generate_pack_data(
1146 self,
1147 have: Iterable[ObjectID],
1148 want: Iterable[ObjectID],
1149 *,
1150 shallow: Set[ObjectID] | None = None,
1151 progress: Callable[..., None] | None = None,
1152 ofs_delta: bool = True,
1153 ) -> tuple[int, Iterator[UnpackedObject]]:
1154 """Generate pack data objects for a set of wants/haves.
1156 Args:
1157 have: List of SHA1s of objects that should not be sent
1158 want: List of SHA1s of objects that should be sent
1159 shallow: Set of shallow commit SHA1s to skip
1160 ofs_delta: Whether OFS deltas can be included
1161 progress: Optional progress reporting method
1162 """
1163 missing_objects = MissingObjectFinder(
1164 self, haves=have, wants=want, shallow=shallow, progress=progress
1165 )
1166 remote_has = missing_objects.get_remote_has()
1167 object_ids = list(missing_objects)
1168 return len(object_ids), generate_unpacked_objects(
1169 self,
1170 object_ids,
1171 progress=progress,
1172 ofs_delta=ofs_delta,
1173 other_haves=remote_has,
1174 )
1176 def _clear_cached_packs(self) -> None:
1177 pack_cache = self._pack_cache
1178 self._pack_cache = {}
1179 self._pack_access_order = []
1180 while pack_cache:
1181 (_name, pack) = pack_cache.popitem()
1182 pack.close()
1184 def _total_pack_mmap_size(self) -> int:
1185 """Return the total mmapped memory across all cached packs."""
1186 return sum(pack.mmap_size for pack in self._pack_cache.values())
1188 def _mark_pack_used(self, pack_hash: str) -> None:
1189 """Mark a pack as recently used for LRU tracking."""
1190 try:
1191 self._pack_access_order.remove(pack_hash)
1192 except ValueError:
1193 pass
1194 self._pack_access_order.append(pack_hash)
1196 def _enforce_packed_git_limit(self) -> None:
1197 """Evict least-recently-used packs if the memory limit is exceeded."""
1198 if self.packed_git_limit is None:
1199 return
1200 while (
1201 self._pack_access_order
1202 and self._total_pack_mmap_size() > self.packed_git_limit
1203 ):
1204 oldest = self._pack_access_order.pop(0)
1205 pack = self._pack_cache.get(oldest)
1206 if pack is not None:
1207 pack.close()
1208 del self._pack_cache[oldest]
1210 def _iter_cached_packs(self) -> Iterator[Pack]:
1211 return iter(list(self._pack_cache.values()))
1213 def _evict_pack(self, pack: "Pack | FilePackIndex") -> None:
1214 """Evict a pack from the cache after its backing file disappeared.
1216 ``pack`` may be a :class:`Pack` or a :class:`FilePackIndex`; in the
1217 latter case the index's owning ``Pack`` is matched via the cached
1218 pack's ``_idx`` reference.
1219 """
1220 for key, cached in list(self._pack_cache.items()):
1221 if cached is pack or cached._idx is pack:
1222 del self._pack_cache[key]
1223 try:
1224 self._pack_access_order.remove(key)
1225 except ValueError:
1226 pass
1227 try:
1228 cached.close()
1229 except OSError:
1230 pass
1231 break
1233 def _lookup_in_packs(self, lookup: "Callable[[Pack], _T]") -> "_T":
1234 """Run ``lookup(pack)`` against each cached pack and return the first hit.
1236 ``lookup`` should raise ``KeyError`` if the pack does not contain the
1237 target. ``PackFileDisappeared`` from a concurrent ``git repack`` /
1238 ``gc --auto`` is caught: the stale pack is evicted, the pack
1239 directory is rescanned, and the search retries — bounded, mirroring
1240 git's ``reprepare_packed_git()``. If no cached pack has the object
1241 the pack directory is rescanned once to pick up any newly-arrived
1242 packs (e.g. another writer just landed one). ``KeyError`` is raised
1243 if no pack — old or new — has the object.
1244 """
1245 rescanned = False
1246 for _attempt in range(_MAX_PACK_RESCAN_ATTEMPTS):
1247 disappeared = False
1248 for pack_hash, pack in list(self._pack_cache.items()):
1249 try:
1250 result = lookup(pack)
1251 except KeyError:
1252 continue
1253 except PackFileDisappeared as exc:
1254 self._evict_pack(exc.obj)
1255 disappeared = True
1256 continue
1257 self._mark_pack_used(pack_hash)
1258 self._enforce_packed_git_limit()
1259 return result
1260 if disappeared:
1261 self._update_pack_cache()
1262 rescanned = True
1263 continue
1264 if not rescanned:
1265 # Maybe another process just landed a pack with the object.
1266 if self._update_pack_cache():
1267 rescanned = True
1268 continue
1269 break
1270 raise KeyError
1272 def _update_pack_cache(self) -> list[Pack]:
1273 raise NotImplementedError(self._update_pack_cache)
1275 def close(self) -> None:
1276 """Close the object store and release resources.
1278 This method closes all cached pack files and frees associated resources.
1279 Can be called multiple times safely.
1280 """
1281 self._clear_cached_packs()
1283 def __del__(self) -> None:
1284 """Warn if the object store is being deleted with unclosed packs."""
1285 if self._pack_cache:
1286 import warnings
1288 warnings.warn(
1289 f"ObjectStore {self!r} was destroyed with {len(self._pack_cache)} "
1290 "unclosed pack(s). Please call close() explicitly.",
1291 ResourceWarning,
1292 stacklevel=2,
1293 )
1294 self.close()
1296 @property
1297 def packs(self) -> list[Pack]:
1298 """List with pack objects."""
1299 return list(self._iter_cached_packs()) + list(self._update_pack_cache())
1301 def count_pack_files(self) -> int:
1302 """Count the number of pack files.
1304 Returns:
1305 Number of pack files (excluding those with .keep files)
1306 """
1307 count = 0
1308 for pack in self.packs:
1309 # Check if there's a .keep file for this pack
1310 keep_path = pack._basename + ".keep"
1311 if not os.path.exists(keep_path):
1312 count += 1
1313 return count
1315 def _iter_alternate_objects(self) -> Iterator[ObjectID]:
1316 """Iterate over the SHAs of all the objects in alternate stores."""
1317 for alternate in self.alternates:
1318 yield from alternate
1320 def _iter_loose_objects(self) -> Iterator[ObjectID]:
1321 """Iterate over the SHAs of all loose objects."""
1322 raise NotImplementedError(self._iter_loose_objects)
1324 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
1325 raise NotImplementedError(self._get_loose_object)
1327 def delete_loose_object(self, sha: ObjectID) -> None:
1328 """Delete a loose object.
1330 This method only handles loose objects. For packed objects,
1331 use repack(exclude=...) to exclude them during repacking.
1332 """
1333 raise NotImplementedError(self.delete_loose_object)
1335 def _remove_pack(self, pack: "Pack") -> None:
1336 raise NotImplementedError(self._remove_pack)
1338 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
1339 """Pack loose objects.
1341 Args:
1342 progress: Optional progress reporting callback
1344 Returns: Number of objects packed
1345 """
1346 objects: list[tuple[ShaFile, None]] = []
1347 for sha in self._iter_loose_objects():
1348 obj = self._get_loose_object(sha)
1349 if obj is not None:
1350 objects.append((obj, None))
1351 self.add_objects(objects, progress=progress)
1352 for obj, path in objects:
1353 self.delete_loose_object(obj.id)
1354 return len(objects)
1356 def repack(
1357 self,
1358 exclude: Set[bytes] | None = None,
1359 progress: Callable[[str], None] | None = None,
1360 ) -> int:
1361 """Repack the packs in this repository.
1363 Note that this implementation is fairly naive and currently keeps all
1364 objects in memory while it repacks.
1366 Args:
1367 exclude: Optional set of object SHAs to exclude from repacking
1368 progress: Optional progress reporting callback
1369 """
1370 if exclude is None:
1371 exclude = set()
1373 loose_objects = set()
1374 excluded_loose_objects = set()
1375 for sha in self._iter_loose_objects():
1376 if sha not in exclude:
1377 obj = self._get_loose_object(sha)
1378 if obj is not None:
1379 loose_objects.add(obj)
1380 else:
1381 excluded_loose_objects.add(sha)
1383 objects: set[tuple[ShaFile, None]] = {(obj, None) for obj in loose_objects}
1384 old_packs = {p.name(): p for p in self.packs}
1385 for name, pack in old_packs.items():
1386 objects.update(
1387 (obj, None) for obj in pack.iterobjects() if obj.id not in exclude
1388 )
1390 # Only create a new pack if there are objects to pack
1391 if objects:
1392 # The name of the consolidated pack might match the name of a
1393 # pre-existing pack. Take care not to remove the newly created
1394 # consolidated pack.
1395 consolidated = self.add_objects(list(objects), progress=progress)
1396 if consolidated is not None:
1397 old_packs.pop(consolidated.name(), None)
1399 # Delete loose objects that were packed
1400 for obj in loose_objects:
1401 if obj is not None:
1402 self.delete_loose_object(obj.id)
1403 # Delete excluded loose objects
1404 for sha in excluded_loose_objects:
1405 self.delete_loose_object(sha)
1406 for name, pack in old_packs.items():
1407 self._remove_pack(pack)
1408 self._update_pack_cache()
1409 return len(objects)
1411 def generate_pack_bitmaps(
1412 self,
1413 refs: dict[Ref, ObjectID],
1414 *,
1415 commit_interval: int | None = None,
1416 progress: Callable[[str], None] | None = None,
1417 ) -> int:
1418 """Generate bitmap indexes for all packs that don't have them.
1420 This generates .bitmap files for packfiles, enabling fast reachability
1421 queries. Equivalent to the bitmap generation part of 'git repack -b'.
1423 Args:
1424 refs: Dictionary of ref names to commit SHAs
1425 commit_interval: Include every Nth commit in bitmap index (None for default)
1426 progress: Optional progress reporting callback
1428 Returns:
1429 Number of bitmaps generated
1430 """
1431 count = 0
1432 for pack in self.packs:
1433 pack.ensure_bitmap(
1434 self, refs, commit_interval=commit_interval, progress=progress
1435 )
1436 count += 1
1438 # Update cache to pick up new bitmaps
1439 self._update_pack_cache()
1441 return count
1443 def __iter__(self) -> Iterator[ObjectID]:
1444 """Iterate over the SHAs that are present in this store."""
1445 self._update_pack_cache()
1446 for pack in self._iter_cached_packs():
1447 try:
1448 yield from pack
1449 except PackFileDisappeared as exc:
1450 self._evict_pack(exc.obj)
1451 yield from self._iter_loose_objects()
1452 yield from self._iter_alternate_objects()
1454 def contains_loose(self, sha: ObjectID) -> bool:
1455 """Check if a particular object is present by SHA1 and is loose.
1457 This does not check alternates.
1458 """
1459 return self._get_loose_object(sha) is not None
1461 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
1462 """Obtain the raw fulltext for an object.
1464 Args:
1465 name: sha for the object.
1466 Returns: tuple with numeric type and object contents.
1467 """
1468 sha: RawObjectID
1469 hexsha: ObjectID | None
1470 if len(name) == self.object_format.hex_length:
1471 sha = hex_to_sha(ObjectID(name))
1472 hexsha = cast(ObjectID, name)
1473 elif len(name) == self.object_format.oid_length:
1474 sha = RawObjectID(name)
1475 hexsha = None
1476 else:
1477 raise AssertionError(f"Invalid object name {name!r}")
1478 try:
1479 return self._lookup_in_packs(lambda p: p.get_raw(sha))
1480 except KeyError:
1481 pass
1482 if hexsha is None:
1483 hexsha = sha_to_hex(sha)
1484 ret = self._get_loose_object(hexsha)
1485 if ret is not None:
1486 return ret.type_num, ret.as_raw_string()
1487 for alternate in self.alternates:
1488 try:
1489 return alternate.get_raw(hexsha)
1490 except KeyError:
1491 pass
1492 raise KeyError(hexsha)
1494 def iter_unpacked_subset(
1495 self,
1496 shas: Iterable[ObjectID | RawObjectID],
1497 *,
1498 include_comp: bool = False,
1499 allow_missing: bool = False,
1500 convert_ofs_delta: bool = True,
1501 ) -> Iterator[UnpackedObject]:
1502 """Iterate over a subset of objects, yielding UnpackedObject instances.
1504 Args:
1505 shas: Set of object SHAs to retrieve
1506 include_comp: Whether to include compressed data
1507 allow_missing: If True, skip missing objects; if False, raise KeyError
1508 convert_ofs_delta: Whether to convert OFS_DELTA objects
1510 Returns:
1511 Iterator of UnpackedObject instances
1513 Raises:
1514 KeyError: If an object is missing and allow_missing is False
1515 """
1516 todo: set[ObjectID | RawObjectID] = set(shas)
1517 for p in self._iter_cached_packs():
1518 try:
1519 for unpacked in p.iter_unpacked_subset(
1520 todo,
1521 include_comp=include_comp,
1522 allow_missing=True,
1523 convert_ofs_delta=convert_ofs_delta,
1524 ):
1525 yield unpacked
1526 hexsha = sha_to_hex(unpacked.sha())
1527 todo.remove(hexsha)
1528 except PackFileDisappeared as exc:
1529 self._evict_pack(exc.obj)
1530 # Maybe something else has added a pack with the object
1531 # in the mean time?
1532 for p in self._update_pack_cache():
1533 try:
1534 for unpacked in p.iter_unpacked_subset(
1535 todo,
1536 include_comp=include_comp,
1537 allow_missing=True,
1538 convert_ofs_delta=convert_ofs_delta,
1539 ):
1540 yield unpacked
1541 hexsha = sha_to_hex(unpacked.sha())
1542 todo.remove(hexsha)
1543 except PackFileDisappeared as exc:
1544 self._evict_pack(exc.obj)
1545 for alternate in self.alternates:
1546 assert isinstance(alternate, PackBasedObjectStore)
1547 for unpacked in alternate.iter_unpacked_subset(
1548 todo,
1549 include_comp=include_comp,
1550 allow_missing=True,
1551 convert_ofs_delta=convert_ofs_delta,
1552 ):
1553 yield unpacked
1554 hexsha = sha_to_hex(unpacked.sha())
1555 todo.remove(hexsha)
1557 def iterobjects_subset(
1558 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
1559 ) -> Iterator[ShaFile]:
1560 """Iterate over a subset of objects in the store.
1562 This method searches for objects in pack files, alternates, and loose storage.
1564 Args:
1565 shas: Iterable of object SHAs to retrieve
1566 allow_missing: If True, skip missing objects; if False, raise KeyError
1568 Returns:
1569 Iterator of ShaFile objects
1571 Raises:
1572 KeyError: If an object is missing and allow_missing is False
1573 """
1574 todo: set[ObjectID] = set(shas)
1575 for p in self._iter_cached_packs():
1576 try:
1577 for o in p.iterobjects_subset(todo, allow_missing=True):
1578 yield o
1579 todo.remove(o.id)
1580 except PackFileDisappeared as exc:
1581 self._evict_pack(exc.obj)
1582 # Maybe something else has added a pack with the object
1583 # in the mean time?
1584 for p in self._update_pack_cache():
1585 try:
1586 for o in p.iterobjects_subset(todo, allow_missing=True):
1587 yield o
1588 todo.remove(o.id)
1589 except PackFileDisappeared as exc:
1590 self._evict_pack(exc.obj)
1591 for alternate in self.alternates:
1592 for o in alternate.iterobjects_subset(todo, allow_missing=True):
1593 yield o
1594 todo.remove(o.id)
1595 for oid in todo:
1596 loose_obj: ShaFile | None = self._get_loose_object(oid)
1597 if loose_obj is not None:
1598 yield loose_obj
1599 elif not allow_missing:
1600 raise KeyError(oid)
1602 def get_unpacked_object(
1603 self, sha1: bytes, *, include_comp: bool = False
1604 ) -> UnpackedObject:
1605 """Obtain the unpacked object.
1607 Args:
1608 sha1: sha for the object.
1609 include_comp: Whether to include compression metadata.
1610 """
1611 if len(sha1) == self.object_format.hex_length:
1612 sha = hex_to_sha(cast(ObjectID, sha1))
1613 hexsha = cast(ObjectID, sha1)
1614 elif len(sha1) == self.object_format.oid_length:
1615 sha = cast(RawObjectID, sha1)
1616 hexsha = None
1617 else:
1618 raise AssertionError(f"Invalid object sha1 {sha1!r}")
1619 try:
1620 return self._lookup_in_packs(
1621 lambda p: p.get_unpacked_object(sha, include_comp=include_comp)
1622 )
1623 except KeyError:
1624 pass
1625 if hexsha is None:
1626 hexsha = sha_to_hex(sha)
1627 for alternate in self.alternates:
1628 assert isinstance(alternate, PackBasedObjectStore)
1629 try:
1630 return alternate.get_unpacked_object(hexsha, include_comp=include_comp)
1631 except KeyError:
1632 pass
1633 raise KeyError(hexsha)
1635 def add_objects(
1636 self,
1637 objects: Sequence[tuple[ShaFile, str | None]],
1638 progress: Callable[[str], None] | None = None,
1639 ) -> "Pack | None":
1640 """Add a set of objects to this object store.
1642 Args:
1643 objects: Iterable over (object, path) tuples, should support
1644 __len__.
1645 progress: Optional progress reporting function.
1646 Returns: Pack object of the objects written.
1647 """
1648 count = len(objects)
1649 record_iter = (full_unpacked_object(o) for (o, p) in objects)
1650 return self.add_pack_data(count, record_iter, progress=progress)
1653class DiskObjectStore(PackBasedObjectStore):
1654 """Git-style object store that exists on disk."""
1656 path: str | os.PathLike[str]
1657 pack_dir: str | os.PathLike[str]
1658 _alternates: "list[BaseObjectStore] | None"
1659 _commit_graph: "CommitGraph | None"
1661 def __init__(
1662 self,
1663 path: str | os.PathLike[str],
1664 *,
1665 loose_compression_level: int = -1,
1666 pack_compression_level: int = -1,
1667 pack_index_version: int | None = None,
1668 pack_delta_window_size: int | None = None,
1669 pack_window_memory: int | None = None,
1670 pack_delta_cache_size: int | None = None,
1671 pack_depth: int | None = None,
1672 pack_threads: int | None = None,
1673 pack_big_file_threshold: int | None = None,
1674 packed_git_limit: int | None = None,
1675 delta_base_cache_limit: int | None = None,
1676 fsync_object_files: bool = False,
1677 pack_write_bitmaps: bool = False,
1678 pack_write_bitmap_hash_cache: bool = True,
1679 pack_write_bitmap_lookup_table: bool = True,
1680 shared_perm: "SharedPerm | None" = None,
1681 object_format: "ObjectFormat | None" = None,
1682 loose_object_size_limit: int | None = None,
1683 alternates: "Iterable[str | os.PathLike[str]] | None" = None,
1684 ) -> None:
1685 """Open an object store.
1687 Args:
1688 path: Path of the object store.
1689 loose_compression_level: zlib compression level for loose objects
1690 pack_compression_level: zlib compression level for pack objects
1691 pack_index_version: pack index version to use (1, 2, or 3)
1692 pack_delta_window_size: sliding window size for delta compression
1693 pack_window_memory: memory limit for delta window operations
1694 pack_delta_cache_size: size of cache for delta operations
1695 pack_depth: maximum delta chain depth
1696 pack_threads: number of threads for pack operations
1697 pack_big_file_threshold: threshold for treating files as big
1698 packed_git_limit: maximum total bytes for mmapped pack files
1699 delta_base_cache_limit: maximum bytes for delta base object cache
1700 fsync_object_files: whether to fsync object files for durability
1701 pack_write_bitmaps: whether to write bitmap indexes for packs
1702 pack_write_bitmap_hash_cache: whether to include name-hash cache in bitmaps
1703 pack_write_bitmap_lookup_table: whether to include lookup table in bitmaps
1704 shared_perm: Shared repository permission setting
1705 object_format: Hash algorithm to use (SHA1 or SHA256)
1706 loose_object_size_limit: Maximum inflated size of a single loose
1707 object. Defaults to core.bigFileThreshold's Git default (512 MiB)
1708 via :data:`DEFAULT_LOOSE_OBJECT_SIZE_LIMIT` when None. Guards against
1709 decompression-bomb attacks.
1710 alternates: Extra alternate object directory paths to consult, in
1711 addition to those listed in ``objects/info/alternates``. Used to
1712 plumb ``GIT_ALTERNATE_OBJECT_DIRECTORIES`` through from porcelain.
1713 """
1714 # Import here to avoid circular dependency
1715 from .object_format import DEFAULT_OBJECT_FORMAT
1717 super().__init__(
1718 pack_compression_level=pack_compression_level,
1719 pack_index_version=pack_index_version,
1720 pack_delta_window_size=pack_delta_window_size,
1721 pack_window_memory=pack_window_memory,
1722 pack_delta_cache_size=pack_delta_cache_size,
1723 pack_depth=pack_depth,
1724 pack_threads=pack_threads,
1725 pack_big_file_threshold=pack_big_file_threshold,
1726 packed_git_limit=packed_git_limit,
1727 delta_base_cache_limit=delta_base_cache_limit,
1728 object_format=object_format if object_format else DEFAULT_OBJECT_FORMAT,
1729 )
1730 self.path = path
1731 self.pack_dir = os.path.join(self.path, PACKDIR)
1732 self._alternates = None
1733 self._extra_alternate_paths: list[str] = (
1734 [os.fsdecode(os.fspath(p)) for p in alternates]
1735 if alternates is not None
1736 else []
1737 )
1738 self.loose_compression_level = loose_compression_level
1739 self.pack_compression_level = pack_compression_level
1740 self.pack_index_version = pack_index_version
1741 self.fsync_object_files = fsync_object_files
1742 self.pack_write_bitmaps = pack_write_bitmaps
1743 self.pack_write_bitmap_hash_cache = pack_write_bitmap_hash_cache
1744 self.pack_write_bitmap_lookup_table = pack_write_bitmap_lookup_table
1745 self.shared_perm = shared_perm
1746 self.loose_object_size_limit = (
1747 loose_object_size_limit
1748 if loose_object_size_limit is not None
1749 else DEFAULT_LOOSE_OBJECT_SIZE_LIMIT
1750 )
1752 # Commit graph support - lazy loaded
1753 self._commit_graph = None
1754 self._use_commit_graph = True # Default to true
1756 # Multi-pack-index support - lazy loaded
1757 self._midx: MultiPackIndex | None = None
1758 self._use_midx = True # Default to true
1760 def __repr__(self) -> str:
1761 """Return string representation of DiskObjectStore.
1763 Returns:
1764 String representation including the store path
1765 """
1766 return f"<{self.__class__.__name__}({self.path!r})>"
1768 @classmethod
1769 def from_config(
1770 cls,
1771 path: str | os.PathLike[str],
1772 config: "Config",
1773 *,
1774 shared_perm: "SharedPerm | None" = None,
1775 alternates: "Iterable[str | os.PathLike[str]] | None" = None,
1776 ) -> "DiskObjectStore":
1777 """Create a DiskObjectStore from a configuration object.
1779 Args:
1780 path: Path to the object store directory
1781 config: Configuration object to read settings from
1782 shared_perm: Optional shared repository permission setting
1783 alternates: Extra alternate object directory paths to consult,
1784 appended to those listed in ``objects/info/alternates``.
1786 Returns:
1787 New DiskObjectStore instance configured according to config
1788 """
1789 try:
1790 default_compression_level = int(
1791 config.get((b"core",), b"compression").decode()
1792 )
1793 except KeyError:
1794 default_compression_level = -1
1795 try:
1796 loose_compression_level = int(
1797 config.get((b"core",), b"looseCompression").decode()
1798 )
1799 except KeyError:
1800 loose_compression_level = default_compression_level
1801 try:
1802 pack_compression_level = int(
1803 config.get((b"core",), "packCompression").decode()
1804 )
1805 except KeyError:
1806 pack_compression_level = default_compression_level
1807 try:
1808 pack_index_version = int(config.get((b"pack",), b"indexVersion").decode())
1809 except KeyError:
1810 pack_index_version = None
1812 # Read pack configuration options
1813 try:
1814 pack_delta_window_size = int(
1815 config.get((b"pack",), b"deltaWindowSize").decode()
1816 )
1817 except KeyError:
1818 pack_delta_window_size = None
1819 try:
1820 pack_window_memory = int(config.get((b"pack",), b"windowMemory").decode())
1821 except KeyError:
1822 pack_window_memory = None
1823 try:
1824 pack_delta_cache_size = int(
1825 config.get((b"pack",), b"deltaCacheSize").decode()
1826 )
1827 except KeyError:
1828 pack_delta_cache_size = None
1829 try:
1830 pack_depth = int(config.get((b"pack",), b"depth").decode())
1831 except KeyError:
1832 pack_depth = None
1833 try:
1834 pack_threads = int(config.get((b"pack",), b"threads").decode())
1835 except KeyError:
1836 pack_threads = None
1837 try:
1838 pack_big_file_threshold = int(
1839 config.get((b"pack",), b"bigFileThreshold").decode()
1840 )
1841 except KeyError:
1842 pack_big_file_threshold = None
1844 # Read core.packedGitLimit setting
1845 try:
1846 packed_git_limit = int(config.get((b"core",), b"packedGitLimit").decode())
1847 except KeyError:
1848 packed_git_limit = None
1850 # Read core.deltaBaseCacheLimit setting
1851 try:
1852 delta_base_cache_limit = int(
1853 config.get((b"core",), b"deltaBaseCacheLimit").decode()
1854 )
1855 except KeyError:
1856 delta_base_cache_limit = None
1858 # Read core.bigFileThreshold setting; used as the upper bound for
1859 # inflating a single loose object, guarding against decompression bombs.
1860 try:
1861 loose_object_size_limit: int | None = int(
1862 config.get((b"core",), b"bigFileThreshold").decode()
1863 )
1864 except KeyError:
1865 loose_object_size_limit = None
1867 # Read core.commitGraph setting
1868 use_commit_graph = config.get_boolean((b"core",), b"commitGraph", True)
1870 # Read core.multiPackIndex setting
1871 use_midx = config.get_boolean((b"core",), b"multiPackIndex", True)
1873 # Read core.fsyncObjectFiles setting
1874 fsync_object_files = config.get_boolean((b"core",), b"fsyncObjectFiles", False)
1876 # Read bitmap settings
1877 pack_write_bitmaps = config.get_boolean((b"pack",), b"writeBitmaps", False)
1878 pack_write_bitmap_hash_cache = config.get_boolean(
1879 (b"pack",), b"writeBitmapHashCache", True
1880 )
1881 pack_write_bitmap_lookup_table = config.get_boolean(
1882 (b"pack",), b"writeBitmapLookupTable", True
1883 )
1884 # Also check repack.writeBitmaps for backwards compatibility
1885 if not pack_write_bitmaps:
1886 pack_write_bitmaps = config.get_boolean(
1887 (b"repack",), b"writeBitmaps", False
1888 )
1890 # Get hash algorithm from config
1891 from .object_format import get_object_format
1893 object_format = None
1894 try:
1895 try:
1896 version = int(config.get((b"core",), b"repositoryformatversion"))
1897 except KeyError:
1898 version = 0
1899 if version == 1:
1900 try:
1901 object_format_name = config.get((b"extensions",), b"objectformat")
1902 except KeyError:
1903 object_format_name = b"sha1"
1904 object_format = get_object_format(object_format_name.decode("ascii"))
1905 except (KeyError, ValueError):
1906 pass
1908 instance = cls(
1909 path,
1910 loose_compression_level=loose_compression_level,
1911 pack_compression_level=pack_compression_level,
1912 pack_index_version=pack_index_version,
1913 pack_delta_window_size=pack_delta_window_size,
1914 pack_window_memory=pack_window_memory,
1915 pack_delta_cache_size=pack_delta_cache_size,
1916 pack_depth=pack_depth,
1917 pack_threads=pack_threads,
1918 pack_big_file_threshold=pack_big_file_threshold,
1919 packed_git_limit=packed_git_limit,
1920 delta_base_cache_limit=delta_base_cache_limit,
1921 fsync_object_files=fsync_object_files,
1922 pack_write_bitmaps=pack_write_bitmaps,
1923 pack_write_bitmap_hash_cache=pack_write_bitmap_hash_cache,
1924 pack_write_bitmap_lookup_table=pack_write_bitmap_lookup_table,
1925 shared_perm=shared_perm,
1926 object_format=object_format,
1927 loose_object_size_limit=loose_object_size_limit,
1928 alternates=alternates,
1929 )
1930 instance._use_commit_graph = use_commit_graph
1931 instance._use_midx = use_midx
1932 return instance
1934 @property
1935 def alternates(self) -> list["BaseObjectStore"]:
1936 """Get the list of alternate object stores.
1938 Reads from .git/objects/info/alternates if not already cached.
1940 Returns:
1941 List of DiskObjectStore instances for alternate object directories
1942 """
1943 if self._alternates is not None:
1944 return self._alternates
1945 self._alternates = []
1946 for path in self._read_alternate_paths():
1947 self._alternates.append(DiskObjectStore(path))
1948 for path in self._extra_alternate_paths:
1949 self._alternates.append(DiskObjectStore(path))
1950 return self._alternates
1952 def _read_alternate_paths(self) -> Iterator[str]:
1953 try:
1954 f = GitFile(os.path.join(self.path, INFODIR, "alternates"), "rb")
1955 except FileNotFoundError:
1956 return
1957 with f:
1958 for line in f.readlines():
1959 line = line.rstrip(b"\n")
1960 if line.startswith(b"#"):
1961 continue
1962 if os.path.isabs(line):
1963 yield os.fsdecode(line)
1964 else:
1965 yield os.fsdecode(os.path.join(os.fsencode(self.path), line))
1967 def add_alternate_path(self, path: str | os.PathLike[str]) -> None:
1968 """Add an alternate path to this object store."""
1969 info_dir = os.path.join(self.path, INFODIR)
1970 try:
1971 os.mkdir(info_dir)
1972 adjust_shared_perm(info_dir, self.shared_perm)
1973 except FileExistsError:
1974 pass
1975 alternates_path = os.path.join(self.path, INFODIR, "alternates")
1976 with GitFile(alternates_path, "wb", shared_perm=self.shared_perm) as f:
1977 try:
1978 orig_f = open(alternates_path, "rb")
1979 except FileNotFoundError:
1980 pass
1981 else:
1982 with orig_f:
1983 f.write(orig_f.read())
1984 f.write(os.fsencode(path) + b"\n")
1986 if not os.path.isabs(path):
1987 path = os.path.join(self.path, path)
1988 self.alternates.append(DiskObjectStore(path))
1990 def _update_pack_cache(self) -> list[Pack]:
1991 """Read and iterate over new pack files and cache them."""
1992 try:
1993 pack_dir_contents = set(os.listdir(self.pack_dir))
1994 except FileNotFoundError:
1995 return []
1996 pack_files = set()
1997 for name in pack_dir_contents:
1998 # Index any ".pack" file with a matching ".idx", not just
1999 # "pack-<hash>". ``git maintenance`` writes packs named
2000 # "loose-<hash>.pack"; these are ordinary packs and Git indexes
2001 # any .pack file present. The matching ".idx" also confirms the
2002 # pack is fully written.
2003 if name.endswith(".pack"):
2004 basename = name[: -len(".pack")]
2005 if basename + ".idx" in pack_dir_contents:
2006 pack_files.add(basename)
2008 # Open newly appeared pack files
2009 new_packs = []
2010 for basename in pack_files:
2011 if basename not in self._pack_cache:
2012 pack = Pack(
2013 os.path.join(self.pack_dir, basename),
2014 object_format=self.object_format,
2015 delta_window_size=self.pack_delta_window_size,
2016 window_memory=self.pack_window_memory,
2017 delta_cache_size=self.pack_delta_cache_size,
2018 depth=self.pack_depth,
2019 threads=self.pack_threads,
2020 big_file_threshold=self.pack_big_file_threshold,
2021 delta_base_cache_limit=self.delta_base_cache_limit,
2022 )
2023 new_packs.append(pack)
2024 self._pack_cache[basename] = pack
2025 self._mark_pack_used(basename)
2026 # Remove disappeared pack files
2027 for f in set(self._pack_cache) - pack_files:
2028 self._pack_cache.pop(f).close()
2029 try:
2030 self._pack_access_order.remove(f)
2031 except ValueError:
2032 pass
2033 self._enforce_packed_git_limit()
2034 return new_packs
2036 def _get_shafile_path(self, sha: ObjectID) -> str:
2037 # Reject anything that is neither a valid hex object id nor a raw
2038 # binary oid before building a path. Otherwise a malformed id (e.g.
2039 # one containing path separators) would be joined into a filename
2040 # that escapes the objects directory.
2041 if not valid_hexsha(sha):
2042 raise ValueError(f"Invalid object id {sha!r}")
2043 # Check from object dir
2044 return hex_to_filename(os.fspath(self.path), sha)
2046 def _iter_loose_objects(self) -> Iterator[ObjectID]:
2047 for base in os.listdir(self.path):
2048 if len(base) != 2:
2049 continue
2050 for rest in os.listdir(os.path.join(self.path, base)):
2051 sha = os.fsencode(base + rest)
2052 if not valid_hexsha(sha):
2053 continue
2054 yield ObjectID(sha)
2056 def count_loose_objects(self) -> int:
2057 """Count the number of loose objects in the object store.
2059 Returns:
2060 Number of loose objects
2061 """
2062 # Calculate expected filename length for loose
2063 # objects (excluding directory)
2064 fn_length = self.object_format.hex_length - 2
2065 count = 0
2066 if not os.path.exists(self.path):
2067 return 0
2069 for i in range(256):
2070 subdir = os.path.join(self.path, f"{i:02x}")
2071 try:
2072 count += len(
2073 [name for name in os.listdir(subdir) if len(name) == fn_length]
2074 )
2075 except FileNotFoundError:
2076 # Directory may have been removed or is inaccessible
2077 continue
2079 return count
2081 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
2082 try:
2083 # Load the object from path with SHA and hash algorithm from object store
2084 # Convert to hex ObjectID if needed
2085 if len(sha) == self.object_format.oid_length:
2086 hex_sha: ObjectID = sha_to_hex(RawObjectID(sha))
2087 else:
2088 hex_sha = ObjectID(sha)
2089 path = self._get_shafile_path(hex_sha)
2090 return ShaFile.from_path(
2091 path,
2092 hex_sha,
2093 object_format=self.object_format,
2094 max_size=self.loose_object_size_limit,
2095 )
2096 except FileNotFoundError:
2097 return None
2099 def delete_loose_object(self, sha: ObjectID) -> None:
2100 """Delete a loose object from disk.
2102 Args:
2103 sha: SHA1 of the object to delete
2105 Raises:
2106 FileNotFoundError: If the object file doesn't exist
2107 """
2108 _remove_readonly(self._get_shafile_path(sha))
2109 self._invalidate_peel_cache()
2111 def get_object_mtime(self, sha: ObjectID) -> float:
2112 """Get the modification time of an object.
2114 Args:
2115 sha: SHA1 of the object
2117 Returns:
2118 Modification time as seconds since epoch
2120 Raises:
2121 KeyError: if the object is not found
2122 """
2123 # First check if it's a loose object
2124 if self.contains_loose(sha):
2125 path = self._get_shafile_path(sha)
2126 try:
2127 return os.path.getmtime(path)
2128 except FileNotFoundError:
2129 pass
2131 # Check if it's in a pack file
2132 for pack in self.packs:
2133 try:
2134 if sha in pack:
2135 # Use the pack file's mtime for packed objects
2136 pack_path = pack._data_path
2137 try:
2138 return os.path.getmtime(pack_path)
2139 except (FileNotFoundError, AttributeError):
2140 pass
2141 except PackFileDisappeared:
2142 pass
2144 raise KeyError(sha)
2146 def _remove_pack(self, pack: Pack) -> None:
2147 self._invalidate_peel_cache()
2148 # _pack_cache is keyed by the full pack basename (e.g. "pack-<hash>"
2149 # or "loose-<hash>"), matching pack._basename.
2150 basename = os.path.basename(pack._basename)
2151 self._pack_cache.pop(basename, None)
2152 try:
2153 self._pack_access_order.remove(basename)
2154 except ValueError:
2155 pass
2156 # Store paths before closing to avoid re-opening files on Windows
2157 data_path = pack._data_path
2158 idx_path = pack._idx_path
2159 pack.close()
2160 _remove_readonly(data_path)
2161 if os.path.exists(idx_path):
2162 _remove_readonly(idx_path)
2164 def _get_pack_basepath(
2165 self, entries: Iterable[tuple[bytes, int, int | None]]
2166 ) -> str:
2167 suffix_bytes = iter_sha1(entry[0] for entry in entries)
2168 # TODO: Handle self.pack_dir being bytes
2169 suffix = suffix_bytes.decode("ascii")
2170 return os.path.join(self.pack_dir, "pack-" + suffix)
2172 def _index_pack(
2173 self,
2174 indexer: PackIndexer,
2175 num_objects: int,
2176 progress: Callable[..., None] | None = None,
2177 ) -> tuple[list[PackIndexEntry], set[RawObjectID]]:
2178 """Drain an indexer into index entries and the external refs it needs.
2180 Args:
2181 indexer: A PackIndexer over the pack being completed.
2182 num_objects: Number of objects in the pack, for progress reporting.
2183 progress: Optional progress reporting function.
2185 Returns: Tuple of (index entries, external refs). ext_refs() is only
2186 populated once the indexer has been drained.
2187 """
2188 entries = []
2189 for i, entry in enumerate(indexer):
2190 if progress is not None:
2191 progress(f"generating index: {i}/{num_objects}\r".encode("ascii"))
2192 entries.append(entry)
2193 return entries, set(indexer.ext_refs())
2195 def _complete_pack(
2196 self,
2197 f: BinaryIO,
2198 path: str,
2199 entries: list[PackIndexEntry],
2200 ext_refs: set[RawObjectID],
2201 progress: Callable[..., None] | None = None,
2202 refs: dict[Ref, ObjectID] | None = None,
2203 ) -> Pack:
2204 """Move a specific file containing a pack into the pack directory.
2206 Note: The file should be on the same file system as the
2207 packs directory.
2209 This takes ownership of ``f``: it appends any missing base objects,
2210 closes the file and renames it into place. Callers must have finished
2211 reading the pack (see :meth:`_index_pack`) before calling this; on
2212 Windows a mapping left over the file blocks both the write and the
2213 rename.
2215 Args:
2216 f: Open file object for the pack.
2217 path: Path to the pack file.
2218 entries: Index entries for the objects already in the pack.
2219 ext_refs: Objects the pack deltas against that it does not contain.
2220 progress: Optional progress reporting function.
2221 refs: Optional dictionary of refs for bitmap generation.
2222 """
2223 pack_sha, extra_entries = extend_pack(
2224 f,
2225 ext_refs,
2226 get_raw=self.get_raw,
2227 compression_level=self.pack_compression_level,
2228 progress=progress,
2229 object_format=self.object_format,
2230 )
2231 f.flush()
2232 if self.fsync_object_files:
2233 try:
2234 fileno = f.fileno()
2235 except AttributeError as e:
2236 raise OSError("fsync requested but file has no fileno()") from e
2237 else:
2238 os.fsync(fileno)
2239 f.close()
2241 entries.extend(extra_entries)
2243 # Move the pack in.
2244 entries.sort()
2245 pack_base_name = self._get_pack_basepath(entries)
2247 # A pack's identity is the SHA over its object SHAs, which is the
2248 # "<hash>" suffix _get_pack_basepath builds the name from. Compare by
2249 # that rather than by basename so an existing pack holding the same
2250 # objects under a different prefix (e.g. a "loose-<hash>" pack written
2251 # by git maintenance) is recognised and not duplicated.
2252 pack_name = os.path.basename(pack_base_name)[len("pack-") :].encode("ascii")
2253 for pack in self.packs:
2254 if pack.name() == pack_name:
2255 # The objects are already packed; drop the temporary pack we
2256 # were about to move in rather than leaking it into pack_dir.
2257 _remove_readonly(path)
2258 return pack
2260 target_pack_path = pack_base_name + ".pack"
2261 target_index_path = pack_base_name + ".idx"
2262 if sys.platform == "win32":
2263 # Windows might have the target pack file lingering. Attempt
2264 # removal, silently passing if the target does not exist.
2265 with suppress(FileNotFoundError):
2266 os.remove(target_pack_path)
2267 os.rename(path, target_pack_path)
2269 # Write the index.
2270 with GitFile(
2271 target_index_path,
2272 "wb",
2273 mask=PACK_MODE,
2274 fsync=self.fsync_object_files,
2275 shared_perm=self.shared_perm,
2276 ) as index_file:
2277 write_pack_index(
2278 index_file, entries, pack_sha, version=self.pack_index_version
2279 )
2281 # Generate bitmap if configured and refs are available
2282 if self.pack_write_bitmaps and refs:
2283 from .bitmap import generate_bitmap, write_bitmap
2284 from .pack import load_pack_index
2286 if progress:
2287 progress("Generating bitmap index\r".encode("ascii"))
2289 # Load the index we just wrote. load_pack_index keeps the file
2290 # open for the lifetime of the index (it mmaps it), so close it
2291 # once the bitmap is generated rather than leaving the .idx
2292 # mapped, which would lock it on Windows.
2293 with closing(
2294 load_pack_index(target_index_path, self.object_format)
2295 ) as pack_index:
2296 bitmap = generate_bitmap(
2297 pack_index=pack_index,
2298 object_store=self,
2299 refs=refs,
2300 pack_checksum=pack_sha,
2301 include_hash_cache=self.pack_write_bitmap_hash_cache,
2302 include_lookup_table=self.pack_write_bitmap_lookup_table,
2303 progress=lambda msg: (
2304 progress(msg.encode("ascii"))
2305 if progress and isinstance(msg, str)
2306 else None
2307 ),
2308 )
2310 # Write the bitmap
2311 target_bitmap_path = pack_base_name + ".bitmap"
2312 write_bitmap(target_bitmap_path, bitmap)
2314 if progress:
2315 progress("Bitmap index written\r".encode("ascii"))
2317 # Add the pack to the store and return it.
2318 final_pack = Pack(
2319 pack_base_name,
2320 object_format=self.object_format,
2321 delta_window_size=self.pack_delta_window_size,
2322 window_memory=self.pack_window_memory,
2323 delta_cache_size=self.pack_delta_cache_size,
2324 depth=self.pack_depth,
2325 threads=self.pack_threads,
2326 big_file_threshold=self.pack_big_file_threshold,
2327 delta_base_cache_limit=self.delta_base_cache_limit,
2328 )
2329 try:
2330 final_pack.check_length_and_checksum()
2331 # Materialise every object so payloads that fail to parse
2332 # (e.g. tree entries with garbage modes) are rejected rather
2333 # than silently landed on disk. MemoryObjectStore already
2334 # validates ingested objects this way via PackInflater; without
2335 # the same check DiskObjectStore was strictly weaker.
2336 for _obj in PackInflater.for_pack_data(
2337 final_pack.data, resolve_ext_ref=self.get_raw
2338 ):
2339 pass
2340 except BaseException:
2341 final_pack.close()
2342 with suppress(FileNotFoundError):
2343 os.remove(target_pack_path)
2344 with suppress(FileNotFoundError):
2345 os.remove(target_index_path)
2346 if self.pack_write_bitmaps and refs:
2347 with suppress(FileNotFoundError):
2348 os.remove(pack_base_name + ".bitmap")
2349 raise
2350 # _pack_cache is keyed by the full basename (/path/to/pack-HASH -> pack-HASH)
2351 self._add_cached_pack(os.path.basename(pack_base_name), final_pack)
2352 return final_pack
2354 def add_thin_pack(
2355 self,
2356 read_all: Callable[[int], bytes],
2357 read_some: Callable[[int], bytes] | None,
2358 progress: Callable[..., None] | None = None,
2359 *,
2360 max_input_size: int | None = None,
2361 ) -> "Pack":
2362 """Add a new thin pack to this object store.
2364 Thin packs are packs that contain deltas with parents that exist
2365 outside the pack. They should never be placed in the object store
2366 directly, and always indexed and completed as they are copied.
2368 Args:
2369 read_all: Read function that blocks until the number of
2370 requested bytes are read.
2371 read_some: Read function that returns at least one byte, but may
2372 not return the number of bytes requested.
2373 progress: Optional progress reporting function.
2374 max_input_size: Maximum number of bytes that may be read from
2375 the wire while ingesting this pack. Matches git's
2376 ``receive.maxInputSize`` / ``index-pack --max-input-size``
2377 semantics: ``None`` (the default) or ``0`` mean unlimited.
2378 Exceeding the cap raises ``PackInputTooLarge``.
2379 Returns: A Pack object pointing at the now-completed thin pack in the
2380 objects/pack directory.
2381 """
2382 import tempfile
2384 if max_input_size:
2385 read_all, read_some = _bound_read_callables(
2386 read_all, read_some, max_input_size
2387 )
2389 fd, path = tempfile.mkstemp(dir=self.path, prefix="tmp_pack_")
2390 with os.fdopen(fd, "w+b") as f:
2391 os.chmod(path, PACK_MODE)
2392 indexer = PackIndexer(
2393 f,
2394 self.object_format.hash_func,
2395 resolve_ext_ref=self.get_raw,
2396 )
2397 copier = PackStreamCopier(
2398 self.object_format.hash_func,
2399 read_all,
2400 read_some,
2401 f,
2402 delta_iter=indexer, # type: ignore[arg-type]
2403 )
2404 copier.verify(progress=progress)
2405 entries, ext_refs = self._index_pack(
2406 indexer, len(copier), progress=progress
2407 )
2408 return self._complete_pack(f, path, entries, ext_refs, progress=progress)
2410 def add_pack(
2411 self,
2412 ) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2413 """Add a new pack to this object store.
2415 Returns: Fileobject to write to, a commit function to
2416 call when the pack is finished and an abort
2417 function.
2418 """
2419 import tempfile
2421 fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
2422 f = os.fdopen(fd, "w+b")
2423 os.chmod(path, PACK_MODE)
2424 adjust_shared_perm(path, self.shared_perm)
2426 def commit() -> "Pack | None":
2427 if f.tell() > 0:
2428 f.seek(0)
2430 # Scope the mapping to indexing: _complete_pack writes to and
2431 # renames this same file, which a live mapping blocks on
2432 # Windows. PackData.close() leaves f open for it to finish.
2433 with PackData(path, file=f, object_format=self.object_format) as pd:
2434 indexer = PackIndexer.for_pack_data(
2435 pd,
2436 resolve_ext_ref=self.get_raw,
2437 )
2438 entries, ext_refs = self._index_pack(indexer, len(pd)) # type: ignore[arg-type]
2439 return self._complete_pack(f, path, entries, ext_refs)
2440 else:
2441 f.close()
2442 os.remove(path)
2443 return None
2445 def abort() -> None:
2446 f.close()
2447 os.remove(path)
2449 return f, commit, abort # type: ignore[return-value]
2451 def add_object(self, obj: ShaFile) -> None:
2452 """Add a single object to this object store.
2454 Args:
2455 obj: Object to add
2456 """
2457 # Use the correct hash algorithm for the object ID
2458 obj_id = ObjectID(obj.get_id(self.object_format))
2459 path = self._get_shafile_path(obj_id)
2460 dir = os.path.dirname(path)
2461 try:
2462 os.mkdir(dir)
2463 adjust_shared_perm(dir, self.shared_perm)
2464 except FileExistsError:
2465 pass
2466 try:
2467 # Refresh the mtime instead of just checking for existence. A
2468 # loose object with a stale mtime is a candidate for age-based
2469 # pruning, so a concurrent "git gc" could remove it before the
2470 # caller has created a reference to it.
2471 os.utime(path, None)
2472 except FileNotFoundError:
2473 pass # Not there after all, write it out below.
2474 except PermissionError:
2475 # Owned by another user in a shared repository. The mtime stays
2476 # stale, so write the object out to give it a fresh one.
2477 pass
2478 else:
2479 return # Already there and freshened, no need to write again
2480 with GitFile(
2481 path,
2482 "wb",
2483 mask=PACK_MODE,
2484 fsync=self.fsync_object_files,
2485 shared_perm=self.shared_perm,
2486 ) as f:
2487 f.write(
2488 obj.as_legacy_object(compression_level=self.loose_compression_level)
2489 )
2491 @classmethod
2492 def init(
2493 cls,
2494 path: str | os.PathLike[str],
2495 *,
2496 shared_perm: "SharedPerm | None" = None,
2497 object_format: "ObjectFormat | None" = None,
2498 ) -> "DiskObjectStore":
2499 """Initialize a new disk object store.
2501 Creates the necessary directory structure for a Git object store.
2503 Args:
2504 path: Path where the object store should be created
2505 shared_perm: Optional shared repository permission setting
2506 object_format: Hash algorithm to use (SHA1 or SHA256)
2508 Returns:
2509 New DiskObjectStore instance
2510 """
2511 try:
2512 os.mkdir(path)
2513 adjust_shared_perm(path, shared_perm)
2514 except FileExistsError:
2515 pass
2516 info_path = os.path.join(path, "info")
2517 pack_path = os.path.join(path, PACKDIR)
2518 os.mkdir(info_path)
2519 os.mkdir(pack_path)
2520 adjust_shared_perm(info_path, shared_perm)
2521 adjust_shared_perm(pack_path, shared_perm)
2522 return cls(path, shared_perm=shared_perm, object_format=object_format)
2524 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
2525 """Iterate over all object SHAs with the given prefix.
2527 Args:
2528 prefix: Hex prefix to search for (as bytes)
2530 Returns:
2531 Iterator of object SHAs (as ObjectID) matching the prefix
2532 """
2533 if len(prefix) < 2:
2534 yield from super().iter_prefix(prefix)
2535 return
2536 seen = set()
2537 dir = prefix[:2].decode()
2538 rest = prefix[2:].decode()
2539 try:
2540 for name in os.listdir(os.path.join(self.path, dir)):
2541 if name.startswith(rest):
2542 sha = ObjectID(os.fsencode(dir + name))
2543 if sha not in seen:
2544 seen.add(sha)
2545 yield sha
2546 except FileNotFoundError:
2547 pass
2549 for p in self.packs:
2550 bin_prefix = (
2551 binascii.unhexlify(prefix)
2552 if len(prefix) % 2 == 0
2553 else binascii.unhexlify(prefix[:-1])
2554 )
2555 for bin_sha in p.index.iter_prefix(bin_prefix):
2556 sha = sha_to_hex(bin_sha)
2557 if sha.startswith(prefix) and sha not in seen:
2558 seen.add(sha)
2559 yield sha
2560 for alternate in self.alternates:
2561 for sha in alternate.iter_prefix(prefix):
2562 if sha not in seen:
2563 seen.add(sha)
2564 yield sha
2566 def get_commit_graph(self) -> "CommitGraph | None":
2567 """Get the commit graph for this object store.
2569 Returns:
2570 CommitGraph object if available, None otherwise
2571 """
2572 if not self._use_commit_graph:
2573 return None
2575 if self._commit_graph is None:
2576 from .commit_graph import read_commit_graph
2578 # Look for commit graph in our objects directory
2579 graph_file = os.path.join(self.path, "info", "commit-graph")
2580 if os.path.exists(graph_file):
2581 self._commit_graph = read_commit_graph(graph_file)
2582 return self._commit_graph
2584 def get_midx(self) -> MultiPackIndex | None:
2585 """Get the multi-pack-index for this object store.
2587 Returns:
2588 MultiPackIndex object if available, None otherwise
2590 Raises:
2591 ValueError: If MIDX file is corrupt
2592 OSError: If MIDX file cannot be read
2593 """
2594 if not self._use_midx:
2595 return None
2597 if self._midx is None:
2598 # Look for MIDX in pack directory
2599 midx_file = os.path.join(self.pack_dir, "multi-pack-index")
2600 if os.path.exists(midx_file):
2601 self._midx = load_midx(midx_file)
2602 return self._midx
2604 def _get_pack_by_name(self, pack_name: str) -> Pack:
2605 """Get a pack referenced by a multi-pack-index entry.
2607 Args:
2608 pack_name: Pack index file name as stored in the MIDX. Usually
2609 ``pack-<hash>.idx``, but ``git maintenance`` writes packs
2610 named ``loose-<hash>.idx``, so any ``.idx`` basename is
2611 accepted.
2613 Returns:
2614 Pack object
2616 Raises:
2617 KeyError: If pack doesn't exist
2618 """
2619 if not pack_name.endswith(".idx"):
2620 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2621 # The name is joined under pack_dir below, so reject any path
2622 # separators that a corrupt or hostile MIDX could use to traverse
2623 # directories (backslash matters on Windows).
2624 if "/" in pack_name or "\\" in pack_name:
2625 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2626 basename = pack_name[: -len(".idx")]
2628 # _pack_cache is keyed by full basename and _update_pack_cache
2629 # discovers every "<name>-<hash>.pack" file (including the
2630 # "loose-<hash>" packs that git maintenance writes and the MIDX
2631 # references), so a referenced pack is normally already cached.
2632 try:
2633 return self._pack_cache[basename]
2634 except KeyError:
2635 pass
2637 pack_path = os.path.join(self.pack_dir, basename)
2638 if not os.path.exists(pack_path + ".pack"):
2639 raise KeyError(f"Pack {pack_name} not found")
2641 pack = Pack(
2642 pack_path,
2643 object_format=self.object_format,
2644 delta_window_size=self.pack_delta_window_size,
2645 window_memory=self.pack_window_memory,
2646 delta_cache_size=self.pack_delta_cache_size,
2647 depth=self.pack_depth,
2648 threads=self.pack_threads,
2649 big_file_threshold=self.pack_big_file_threshold,
2650 delta_base_cache_limit=self.delta_base_cache_limit,
2651 )
2652 self._pack_cache[basename] = pack
2653 self._mark_pack_used(basename)
2654 return pack
2656 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2657 """Check if a particular object is present by SHA1 and is packed.
2659 This checks the MIDX first if available, then falls back to checking
2660 individual pack indexes.
2662 Args:
2663 sha: SHA of the object (20/32 bytes binary or 40/64 bytes hex)
2665 Returns:
2666 True if the object is in a pack file
2667 """
2668 # Normalise to binary once: MIDX requires it, and passing binary to
2669 # the per-pack fallback avoids N redundant hex->binary conversions
2670 # inside PackIndex.object_offset. Mirrors ``get_raw`` below.
2671 if len(sha) == self.object_format.hex_length:
2672 sha = hex_to_sha(cast(ObjectID, sha))
2674 midx = self.get_midx()
2675 if midx is not None:
2676 result = midx.object_offset(sha)
2677 if result is not None:
2678 pack_name, _offset = result
2679 # A MIDX outlives the packs it names: repack and prune write
2680 # new packs and remove the old ones without rewriting it. Only
2681 # answer from the MIDX once the pack it names can still be
2682 # opened, as git does in fill_midx_entry, and otherwise fall
2683 # through to the per-pack scan.
2684 try:
2685 self._get_pack_by_name(pack_name)
2686 except (KeyError, PackFileDisappeared):
2687 pass
2688 else:
2689 return True
2691 # Fall back to checking individual packs
2692 return super().contains_packed(sha)
2694 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2695 """Obtain the raw fulltext for an object.
2697 This uses the MIDX if available for faster lookups.
2699 Args:
2700 name: SHA for the object (20 bytes binary or 40 bytes hex)
2702 Returns:
2703 Tuple with numeric type and object contents
2705 Raises:
2706 KeyError: If object not found
2707 """
2708 sha: RawObjectID
2709 if len(name) in (40, 64):
2710 # name is ObjectID (hex), convert to RawObjectID
2711 # Support both SHA1 (40) and SHA256 (64)
2712 sha = hex_to_sha(cast(ObjectID, name))
2713 elif len(name) in (20, 32):
2714 # name is already RawObjectID (binary)
2715 # Support both SHA1 (20) and SHA256 (32)
2716 sha = RawObjectID(name)
2717 else:
2718 raise AssertionError(f"Invalid object name {name!r}")
2720 # Try MIDX first for faster lookup
2721 midx = self.get_midx()
2722 if midx is not None:
2723 result = midx.object_offset(sha)
2724 if result is not None:
2725 pack_name, _offset = result
2726 try:
2727 pack = self._get_pack_by_name(pack_name)
2728 return pack.get_raw(sha)
2729 except (KeyError, PackFileDisappeared):
2730 # Pack disappeared or object not found, fall through to standard lookup
2731 pass
2733 # Fall back to the standard implementation
2734 return super().get_raw(name)
2736 def write_midx(self) -> bytes:
2737 """Write a multi-pack-index file for this object store.
2739 Creates a MIDX file that indexes all pack files in the pack directory.
2741 Returns:
2742 SHA-1 checksum of the written MIDX file
2744 Raises:
2745 OSError: If the pack directory doesn't exist or MIDX can't be written
2746 """
2747 from .midx import write_midx_file
2749 midx_path = os.path.join(self.pack_dir, "multi-pack-index")
2750 # Skip packs that vanish mid-collection (e.g. concurrent
2751 # ``git repack``); the survivors still produce a valid MIDX.
2752 pack_entries: list[tuple[str, list[tuple[RawObjectID, int, int | None]]]] = []
2753 for pack in self.packs:
2754 try:
2755 entries = list(pack.index.iterentries())
2756 except PackFileDisappeared as exc:
2757 self._evict_pack(exc.obj)
2758 continue
2759 pack_entries.append((os.path.basename(pack._basename) + ".idx", entries))
2760 if not pack_entries:
2761 return b"\x00" * 20
2762 return write_midx_file(midx_path, pack_entries)
2764 def write_commit_graph(
2765 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
2766 ) -> None:
2767 """Write a commit graph file for this object store.
2769 Args:
2770 refs: List of refs to include. If None, includes all refs from object store.
2771 reachable: If True, includes all commits reachable from refs.
2772 If False, only includes the direct ref targets.
2773 """
2774 from .commit_graph import get_reachable_commits
2776 if refs is None:
2777 # Get all commit objects from the object store
2778 all_refs = []
2779 # Iterate through all objects to find commits
2780 for sha in self:
2781 try:
2782 obj = self[sha]
2783 if obj.type_name == b"commit":
2784 all_refs.append(sha)
2785 except KeyError:
2786 continue
2787 else:
2788 # Use provided refs
2789 all_refs = list(refs)
2791 if not all_refs:
2792 return # No commits to include
2794 if reachable:
2795 # Get all reachable commits
2796 commit_ids = get_reachable_commits(self, all_refs)
2797 else:
2798 # Just use the direct ref targets - ensure they're hex ObjectIDs
2799 commit_ids = []
2800 for ref in all_refs:
2801 if isinstance(ref, bytes) and len(ref) == self.object_format.hex_length:
2802 # Already hex ObjectID
2803 commit_ids.append(ref)
2804 elif (
2805 isinstance(ref, bytes) and len(ref) == self.object_format.oid_length
2806 ):
2807 # Binary SHA, convert to hex ObjectID
2808 commit_ids.append(sha_to_hex(RawObjectID(ref)))
2809 else:
2810 # Assume it's already correct format
2811 commit_ids.append(ref)
2813 if commit_ids:
2814 # Write commit graph directly to our object store path
2815 # Generate the commit graph
2816 from .commit_graph import generate_commit_graph
2818 graph = generate_commit_graph(self, commit_ids)
2820 if graph.entries:
2821 # Ensure the info directory exists
2822 info_dir = os.path.join(self.path, "info")
2823 os.makedirs(info_dir, exist_ok=True)
2824 adjust_shared_perm(info_dir, self.shared_perm)
2826 # Write using GitFile for atomic operation
2827 graph_path = os.path.join(info_dir, "commit-graph")
2828 with GitFile(graph_path, "wb", shared_perm=self.shared_perm) as f:
2829 assert isinstance(
2830 f, _GitFile
2831 ) # GitFile in write mode always returns _GitFile
2832 graph.write_to_file(f)
2834 # Clear cached commit graph so it gets reloaded
2835 self._commit_graph = None
2837 def prune(self, grace_period: int | None = None) -> None:
2838 """Prune/clean up this object store.
2840 This removes temporary files that were left behind by interrupted
2841 pack operations. These are files that start with ``tmp_pack_`` in the
2842 repository directory or files with .pack extension but no corresponding
2843 .idx file in the pack directory.
2845 Args:
2846 grace_period: Grace period in seconds for removing temporary files.
2847 If None, uses DEFAULT_TEMPFILE_GRACE_PERIOD.
2848 """
2849 import glob
2851 if grace_period is None:
2852 grace_period = DEFAULT_TEMPFILE_GRACE_PERIOD
2854 # Clean up tmp_pack_* files in the repository directory
2855 for tmp_file in glob.glob(os.path.join(self.path, "tmp_pack_*")):
2856 # Check if file is old enough (more than grace period)
2857 mtime = os.path.getmtime(tmp_file)
2858 if time.time() - mtime > grace_period:
2859 os.remove(tmp_file)
2861 # Clean up orphaned .pack files without corresponding .idx files
2862 try:
2863 pack_dir_contents = os.listdir(self.pack_dir)
2864 except FileNotFoundError:
2865 return
2867 pack_files = {}
2868 idx_files = set()
2870 for name in pack_dir_contents:
2871 if name.endswith(".pack"):
2872 base_name = name[:-5] # Remove .pack extension
2873 pack_files[base_name] = name
2874 elif name.endswith(".idx"):
2875 base_name = name[:-4] # Remove .idx extension
2876 idx_files.add(base_name)
2878 # Remove .pack files without corresponding .idx files
2879 for base_name, pack_name in pack_files.items():
2880 if base_name not in idx_files:
2881 pack_path = os.path.join(self.pack_dir, pack_name)
2882 # Check if file is old enough (more than grace period)
2883 mtime = os.path.getmtime(pack_path)
2884 if time.time() - mtime > grace_period:
2885 os.remove(pack_path)
2887 def close(self) -> None:
2888 """Close the object store and release resources.
2890 This method closes all cached pack files, MIDX, and frees associated resources.
2891 Can be called multiple times safely.
2892 """
2893 # Close MIDX if it's loaded
2894 if self._midx is not None:
2895 self._midx.close()
2896 self._midx = None
2898 # Close alternates
2899 if self._alternates is not None:
2900 for alt in self._alternates:
2901 alt.close()
2902 self._alternates = None
2904 # Call parent class close to handle pack files
2905 super().close()
2908class MemoryObjectStore(PackCapableObjectStore):
2909 """Object store that keeps all objects in memory."""
2911 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
2912 """Initialize a MemoryObjectStore.
2914 Creates an empty in-memory object store.
2916 Args:
2917 object_format: Hash algorithm to use (defaults to SHA1)
2918 """
2919 super().__init__(object_format=object_format)
2920 self._data: dict[ObjectID, ShaFile] = {}
2921 self.pack_compression_level = -1
2923 def _to_hexsha(self, sha: ObjectID | RawObjectID) -> ObjectID:
2924 if len(sha) == self.object_format.hex_length:
2925 return cast(ObjectID, sha)
2926 elif len(sha) == self.object_format.oid_length:
2927 return sha_to_hex(cast(RawObjectID, sha))
2928 else:
2929 raise ValueError(f"Invalid sha {sha!r}")
2931 def contains_loose(self, sha: ObjectID) -> bool:
2932 """Check if a particular object is present by SHA1 and is loose."""
2933 return self._to_hexsha(sha) in self._data
2935 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2936 """Check if a particular object is present by SHA1 and is packed."""
2937 return False
2939 def __iter__(self) -> Iterator[ObjectID]:
2940 """Iterate over the SHAs that are present in this store."""
2941 return iter(self._data.keys())
2943 @property
2944 def packs(self) -> list[Pack]:
2945 """List with pack objects."""
2946 return []
2948 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2949 """Obtain the raw text for an object.
2951 Args:
2952 name: sha for the object.
2953 Returns: tuple with numeric type and object contents.
2954 """
2955 obj = self[self._to_hexsha(name)]
2956 return obj.type_num, obj.as_raw_string()
2958 def __getitem__(self, name: ObjectID | RawObjectID) -> ShaFile:
2959 """Retrieve an object by SHA.
2961 Args:
2962 name: SHA of the object (as hex string or bytes)
2964 Returns:
2965 Copy of the ShaFile object
2967 Raises:
2968 KeyError: If the object is not found
2969 """
2970 return self._data[self._to_hexsha(name)].copy()
2972 def __delitem__(self, name: ObjectID) -> None:
2973 """Delete an object from this store, for testing only."""
2974 del self._data[self._to_hexsha(name)]
2975 self._invalidate_peel_cache()
2977 def add_object(self, obj: ShaFile) -> None:
2978 """Add a single object to this object store."""
2979 self._data[obj.id] = obj.copy()
2981 def add_objects(
2982 self,
2983 objects: Iterable[tuple[ShaFile, str | None]],
2984 progress: Callable[[str], None] | None = None,
2985 ) -> None:
2986 """Add a set of objects to this object store.
2988 Args:
2989 objects: Iterable over a list of (object, path) tuples
2990 progress: Optional progress reporting function.
2991 """
2992 for obj, path in objects:
2993 self.add_object(obj)
2995 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2996 """Add a new pack to this object store.
2998 Because this object store doesn't support packs, we extract and add the
2999 individual objects.
3001 Returns: Fileobject to write to and a commit function to
3002 call when the pack is finished.
3003 """
3004 from tempfile import SpooledTemporaryFile
3006 f = SpooledTemporaryFile(max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-")
3008 def commit() -> None:
3009 size = f.tell()
3010 if size > 0:
3011 f.seek(0)
3013 p = PackData.from_file(f, self.object_format, size)
3014 try:
3015 # Verify the trailing pack checksum before extracting
3016 # objects. Without this, a fetch that delivered a
3017 # truncated pack would still be accepted: ``add_pack``
3018 # iterates objects by offset and never reaches the
3019 # trailing bytes, so a stream that lost the last few
3020 # bytes of its trailer slipped through silently.
3021 # ``add_thin_pack`` already validates via
3022 # ``PackStreamCopier.verify``; do the equivalent here.
3023 p.check()
3024 for obj in PackInflater.for_pack_data(p, self.get_raw):
3025 self.add_object(obj)
3026 finally:
3027 p.close()
3028 f.close()
3029 else:
3030 f.close()
3032 def abort() -> None:
3033 f.close()
3035 return f, commit, abort # type: ignore[return-value]
3037 def add_pack_data(
3038 self,
3039 count: int,
3040 unpacked_objects: Iterator[UnpackedObject],
3041 progress: Callable[[str], None] | None = None,
3042 ) -> None:
3043 """Add pack data to this object store.
3045 Args:
3046 count: Number of items to add
3047 unpacked_objects: Iterator of UnpackedObject instances
3048 progress: Optional progress reporting function.
3049 """
3050 if count == 0:
3051 return
3053 # Since MemoryObjectStore doesn't support pack files, we need to
3054 # extract individual objects. To handle deltas properly, we write
3055 # to a temporary pack and then use PackInflater to resolve them.
3056 f, commit, abort = self.add_pack()
3057 try:
3058 write_pack_data(
3059 f.write,
3060 unpacked_objects,
3061 num_records=count,
3062 progress=progress,
3063 object_format=self.object_format,
3064 )
3065 except BaseException:
3066 abort()
3067 raise
3068 else:
3069 commit()
3071 def add_thin_pack(
3072 self,
3073 read_all: Callable[[int], bytes],
3074 read_some: Callable[[int], bytes] | None,
3075 progress: Callable[[str], None] | None = None,
3076 ) -> None:
3077 """Add a new thin pack to this object store.
3079 Thin packs are packs that contain deltas with parents that exist
3080 outside the pack. Because this object store doesn't support packs, we
3081 extract and add the individual objects.
3083 Args:
3084 read_all: Read function that blocks until the number of
3085 requested bytes are read.
3086 read_some: Read function that returns at least one byte, but may
3087 not return the number of bytes requested.
3088 progress: Optional progress reporting function.
3089 """
3090 f, commit, abort = self.add_pack()
3091 try:
3092 copier = PackStreamCopier(
3093 self.object_format.hash_func,
3094 read_all,
3095 read_some,
3096 f,
3097 )
3098 copier.verify()
3099 except BaseException:
3100 abort()
3101 raise
3102 else:
3103 commit()
3106class ObjectIterator(Protocol):
3107 """Interface for iterating over objects."""
3109 def iterobjects(self) -> Iterator[ShaFile]:
3110 """Iterate over all objects.
3112 Returns:
3113 Iterator of ShaFile objects
3114 """
3115 raise NotImplementedError(self.iterobjects)
3118def tree_lookup_path(
3119 lookup_obj: Callable[[ObjectID | RawObjectID], ShaFile],
3120 root_sha: ObjectID | RawObjectID,
3121 path: bytes,
3122) -> tuple[int, ObjectID]:
3123 """Look up an object in a Git tree.
3125 Args:
3126 lookup_obj: Callback for retrieving object by SHA1
3127 root_sha: SHA1 of the root tree
3128 path: Path to lookup
3129 Returns: A tuple of (mode, SHA) of the resulting path.
3130 """
3131 tree = lookup_obj(root_sha)
3132 if not isinstance(tree, Tree):
3133 raise NotTreeError(root_sha)
3134 return tree.lookup_path(lookup_obj, path)
3137def _collect_filetree_revs(
3138 obj_store: ObjectContainer, tree_sha: ObjectID, kset: set[ObjectID]
3139) -> None:
3140 """Collect SHA1s of files and directories for specified tree.
3142 Args:
3143 obj_store: Object store to get objects by SHA from
3144 tree_sha: tree reference to walk
3145 kset: set to fill with references to files and directories
3146 """
3147 filetree = obj_store[tree_sha]
3148 assert isinstance(filetree, Tree)
3149 for name, mode, sha in filetree.iteritems():
3150 assert mode is not None
3151 assert sha is not None
3152 if not S_ISGITLINK(mode) and sha not in kset:
3153 kset.add(sha)
3154 if stat.S_ISDIR(mode):
3155 _collect_filetree_revs(obj_store, sha, kset)
3158def _split_commits_and_tags(
3159 obj_store: ObjectContainer,
3160 lst: Iterable[ObjectID],
3161 *,
3162 unknown: str = "error",
3163) -> tuple[set[ObjectID], set[ObjectID], set[ObjectID]]:
3164 """Split object id list into three lists with commit, tag, and other SHAs.
3166 Commits referenced by tags are included into commits
3167 list as well. Only SHA1s known in this repository will get
3168 through, controlled by the unknown parameter.
3170 Args:
3171 obj_store: Object store to get objects by SHA1 from
3172 lst: Collection of commit and tag SHAs
3173 unknown: How to handle unknown objects: "error", "warn", or "ignore"
3174 Returns: A tuple of (commits, tags, others) SHA1s
3175 """
3176 if unknown not in ("error", "warn", "ignore"):
3177 raise ValueError(
3178 f"unknown must be 'error', 'warn', or 'ignore', got {unknown!r}"
3179 )
3181 commits: set[ObjectID] = set()
3182 tags: set[ObjectID] = set()
3183 others: set[ObjectID] = set()
3184 for e in lst:
3185 try:
3186 o = obj_store[e]
3187 except KeyError:
3188 if unknown == "error":
3189 raise
3190 elif unknown == "warn":
3191 logger.warning("Object %s not found in object store", e.decode("ascii"))
3192 # else: ignore
3193 else:
3194 if isinstance(o, Commit):
3195 commits.add(e)
3196 elif isinstance(o, Tag):
3197 tags.add(e)
3198 tagged = o.object[1]
3199 c, t, os = _split_commits_and_tags(obj_store, [tagged], unknown=unknown)
3200 commits |= c
3201 tags |= t
3202 others |= os
3203 else:
3204 others.add(e)
3205 return (commits, tags, others)
3208class MissingObjectFinder:
3209 """Find the objects missing from another object store.
3211 Args:
3212 object_store: Object store containing at least all objects to be
3213 sent
3214 haves: SHA1s of commits not to send (already present in target)
3215 wants: SHA1s of commits to send
3216 progress: Optional function to report progress to.
3217 get_tagged: Function that returns a dict of pointed-to sha -> tag
3218 sha for including tags.
3219 get_parents: Optional function for getting the parents of a commit.
3220 """
3222 def __init__(
3223 self,
3224 object_store: BaseObjectStore,
3225 haves: Iterable[ObjectID],
3226 wants: Iterable[ObjectID],
3227 *,
3228 shallow: Set[ObjectID] | None = None,
3229 progress: Callable[[bytes], None] | None = None,
3230 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
3231 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3232 ) -> None:
3233 """Initialize a MissingObjectFinder.
3235 Args:
3236 object_store: Object store containing objects
3237 haves: SHA1s of objects already present in target
3238 wants: SHA1s of objects to send
3239 shallow: Set of shallow commit SHA1s
3240 progress: Optional progress reporting callback
3241 get_tagged: Function returning dict of pointed-to sha -> tag sha
3242 get_parents: Function for getting commit parents
3243 """
3244 self.object_store = object_store
3245 if shallow is None:
3246 shallow = set()
3247 self._get_parents = get_parents
3248 reachability = object_store.get_reachability_provider()
3249 # process Commits and Tags differently
3250 # haves may list commits/tags not available locally (silently ignore them).
3251 # wants should only contain valid SHAs (fail fast if not).
3252 have_commits, have_tags, have_others = _split_commits_and_tags(
3253 object_store, haves, unknown="ignore"
3254 )
3255 want_commits, want_tags, want_others = _split_commits_and_tags(
3256 object_store, wants, unknown="error"
3257 )
3258 # all_ancestors is a set of commits that shall not be sent
3259 # (complete repository up to 'haves')
3260 all_ancestors = reachability.get_reachable_commits(
3261 have_commits, exclude=None, shallow=shallow
3262 )
3263 # all_missing - complete set of commits between haves and wants
3264 # common_commits - boundary commits directly encountered when traversing wants
3265 # We use _collect_ancestors here because we need the exact boundary behavior:
3266 # commits that are in all_ancestors and directly reachable from wants,
3267 # but we don't traverse past them. This is hard to express with the
3268 # reachability abstraction alone.
3269 missing_commits, common_commits = _collect_ancestors(
3270 object_store,
3271 want_commits,
3272 frozenset(all_ancestors),
3273 shallow=frozenset(shallow),
3274 get_parents=self._get_parents,
3275 )
3277 self.remote_has: set[ObjectID] = set()
3278 # Now, fill sha_done with commits and revisions of
3279 # files and directories known to be both locally
3280 # and on target. Thus these commits and files
3281 # won't get selected for fetch
3282 for h in common_commits:
3283 self.remote_has.add(h)
3284 cmt = object_store[h]
3285 assert isinstance(cmt, Commit)
3286 # Get tree objects for this commit
3287 tree_objects = reachability.get_tree_objects([cmt.tree])
3288 self.remote_has.update(tree_objects)
3290 # record tags we have as visited, too
3291 for t in have_tags:
3292 self.remote_has.add(t)
3293 self.sha_done = set(self.remote_has)
3295 # in fact, what we 'want' is commits, tags, and others
3296 # we've found missing
3297 self.objects_to_send: set[tuple[ObjectID, bytes | None, int | None, bool]] = {
3298 (w, None, Commit.type_num, False) for w in missing_commits
3299 }
3300 missing_tags = want_tags.difference(have_tags)
3301 self.objects_to_send.update(
3302 {(w, None, Tag.type_num, False) for w in missing_tags}
3303 )
3304 missing_others = want_others.difference(have_others)
3305 self.objects_to_send.update({(w, None, None, False) for w in missing_others})
3307 if progress is None:
3308 self.progress: Callable[[bytes], None] = lambda x: None
3309 else:
3310 self.progress = progress
3311 self._tagged = (get_tagged and get_tagged()) or {}
3313 def get_remote_has(self) -> set[ObjectID]:
3314 """Get the set of SHAs the remote has.
3316 Returns:
3317 Set of SHA1s that the remote side already has
3318 """
3319 return self.remote_has
3321 def add_todo(
3322 self, entries: Iterable[tuple[ObjectID, bytes | None, int | None, bool]]
3323 ) -> None:
3324 """Add objects to the todo list.
3326 Args:
3327 entries: Iterable of tuples (sha, name, type_num, is_leaf)
3328 """
3329 self.objects_to_send.update([e for e in entries if e[0] not in self.sha_done])
3331 def __next__(self) -> tuple[ObjectID, PackHint | None]:
3332 """Get the next object to send.
3334 Returns:
3335 Tuple of (sha, pack_hint)
3337 Raises:
3338 StopIteration: When no more objects to send
3339 """
3340 while True:
3341 if not self.objects_to_send:
3342 self.progress(
3343 f"counting objects: {len(self.sha_done)}, done.\n".encode("ascii")
3344 )
3345 raise StopIteration
3346 (sha, name, type_num, leaf) = self.objects_to_send.pop()
3347 if sha not in self.sha_done:
3348 break
3349 if not leaf:
3350 o = self.object_store[sha]
3351 if isinstance(o, Commit):
3352 self.add_todo([(o.tree, b"", Tree.type_num, False)])
3353 elif isinstance(o, Tree):
3354 todos = []
3355 for n, m, s in o.iteritems():
3356 assert m is not None
3357 assert n is not None
3358 assert s is not None
3359 if not S_ISGITLINK(m):
3360 todos.append(
3361 (
3362 s,
3363 n,
3364 (Blob.type_num if stat.S_ISREG(m) else Tree.type_num),
3365 not stat.S_ISDIR(m),
3366 )
3367 )
3368 self.add_todo(todos)
3369 elif isinstance(o, Tag):
3370 self.add_todo([(o.object[1], None, o.object[0].type_num, False)])
3371 if sha in self._tagged:
3372 self.add_todo([(self._tagged[sha], None, None, True)])
3373 self.sha_done.add(sha)
3374 if len(self.sha_done) % 1000 == 0:
3375 self.progress(f"counting objects: {len(self.sha_done)}\r".encode("ascii"))
3376 if type_num is None:
3377 pack_hint = None
3378 else:
3379 pack_hint = (type_num, name)
3380 return (sha, pack_hint)
3382 def __iter__(self) -> Iterator[tuple[ObjectID, PackHint | None]]:
3383 """Return iterator over objects to send.
3385 Returns:
3386 Self (this class implements the iterator protocol)
3387 """
3388 return self
3391class ObjectStoreGraphWalker:
3392 """Graph walker that finds what commits are missing from an object store."""
3394 heads: set[ObjectID]
3395 """Revisions without descendants in the local repo."""
3397 get_parents: Callable[[ObjectID], list[ObjectID]]
3398 """Function to retrieve parents in the local repo."""
3400 shallow: set[ObjectID]
3402 def __init__(
3403 self,
3404 local_heads: Iterable[ObjectID],
3405 get_parents: Callable[[ObjectID], list[ObjectID]],
3406 shallow: set[ObjectID] | None = None,
3407 update_shallow: Callable[[set[ObjectID] | None, set[ObjectID] | None], None]
3408 | None = None,
3409 ) -> None:
3410 """Create a new instance.
3412 Args:
3413 local_heads: Heads to start search with
3414 get_parents: Function for finding the parents of a SHA1.
3415 shallow: Set of shallow commits.
3416 update_shallow: Function to update shallow commits.
3417 """
3418 self.heads = set(local_heads)
3419 self.get_parents = get_parents
3420 self.parents: dict[ObjectID, list[ObjectID] | None] = {}
3421 if shallow is None:
3422 shallow = set()
3423 self.shallow = shallow
3424 self.update_shallow = update_shallow
3426 def nak(self) -> None:
3427 """Nothing in common was found."""
3429 def ack(self, sha: ObjectID) -> None:
3430 """Ack that a revision and its ancestors are present in the source."""
3431 if len(sha) != 40:
3432 # TODO: support SHA256
3433 raise ValueError(f"unexpected sha {sha!r} received")
3434 ancestors = {sha}
3436 # stop if we run out of heads to remove
3437 while self.heads:
3438 for a in ancestors:
3439 if a in self.heads:
3440 self.heads.remove(a)
3442 # collect all ancestors
3443 new_ancestors = set()
3444 for a in ancestors:
3445 ps = self.parents.get(a)
3446 if ps is not None:
3447 new_ancestors.update(ps)
3448 self.parents[a] = None
3450 # no more ancestors; stop
3451 if not new_ancestors:
3452 break
3454 ancestors = new_ancestors
3456 def next(self) -> ObjectID | None:
3457 """Iterate over ancestors of heads in the target."""
3458 if self.heads:
3459 ret = self.heads.pop()
3460 try:
3461 ps = self.get_parents(ret)
3462 except KeyError:
3463 return None
3464 self.parents[ret] = ps
3465 self.heads.update([p for p in ps if p not in self.parents])
3466 return ret
3467 return None
3469 __next__ = next
3472def commit_tree_changes(
3473 object_store: BaseObjectStore,
3474 tree: ObjectID | Tree,
3475 changes: Sequence[tuple[bytes, int | None, ObjectID | None]],
3476) -> ObjectID:
3477 """Commit a specified set of changes to a tree structure.
3479 This will apply a set of changes on top of an existing tree, storing new
3480 objects in object_store.
3482 changes are a list of tuples with (path, mode, object_sha).
3483 Paths can be both blobs and trees. See the mode and
3484 object sha to None deletes the path.
3486 This method works especially well if there are only a small
3487 number of changes to a big tree. For a large number of changes
3488 to a large tree, use e.g. commit_tree.
3490 Args:
3491 object_store: Object store to store new objects in
3492 and retrieve old ones from.
3493 tree: Original tree root (SHA or Tree object)
3494 changes: changes to apply
3495 Returns: New tree root object
3496 """
3497 # TODO(jelmer): Save up the objects and add them using .add_objects
3498 # rather than with individual calls to .add_object.
3499 # Handle both Tree object and SHA
3500 if isinstance(tree, Tree):
3501 tree_obj: Tree = tree
3502 else:
3503 sha_obj = object_store[tree]
3504 assert isinstance(sha_obj, Tree)
3505 tree_obj = sha_obj
3506 nested_changes: dict[bytes, list[tuple[bytes, int | None, ObjectID | None]]] = {}
3507 for path, new_mode, new_sha in changes:
3508 try:
3509 (dirname, subpath) = path.split(b"/", 1)
3510 except ValueError:
3511 if new_sha is None:
3512 del tree_obj[path]
3513 else:
3514 assert new_mode is not None
3515 tree_obj[path] = (new_mode, new_sha)
3516 else:
3517 nested_changes.setdefault(dirname, []).append((subpath, new_mode, new_sha))
3518 for name, subchanges in nested_changes.items():
3519 try:
3520 orig_subtree_id: ObjectID | Tree = tree_obj[name][1]
3521 except KeyError:
3522 # For new directories, pass an empty Tree object
3523 orig_subtree_id = Tree()
3524 subtree_id = commit_tree_changes(object_store, orig_subtree_id, subchanges)
3525 subtree = object_store[subtree_id]
3526 assert isinstance(subtree, Tree)
3527 if len(subtree) == 0:
3528 del tree_obj[name]
3529 else:
3530 tree_obj[name] = (stat.S_IFDIR, subtree.id)
3531 object_store.add_object(tree_obj)
3532 return tree_obj.id
3535class OverlayObjectStore(BaseObjectStore):
3536 """Object store that can overlay multiple object stores."""
3538 def __init__(
3539 self,
3540 bases: list[BaseObjectStore],
3541 add_store: BaseObjectStore | None = None,
3542 ) -> None:
3543 """Initialize an OverlayObjectStore.
3545 Args:
3546 bases: List of base object stores to overlay
3547 add_store: Optional store to write new objects to
3549 Raises:
3550 ValueError: If stores have different hash algorithms
3551 """
3552 from .object_format import verify_same_object_format
3554 # Verify all stores use the same hash algorithm
3555 store_algorithms = [store.object_format for store in bases]
3556 if add_store:
3557 store_algorithms.append(add_store.object_format)
3559 object_format = verify_same_object_format(*store_algorithms)
3561 super().__init__(object_format=object_format)
3562 self.bases = bases
3563 self.add_store = add_store
3565 def add_object(self, obj: ShaFile) -> None:
3566 """Add a single object to the store.
3568 Args:
3569 obj: Object to add
3571 Raises:
3572 NotImplementedError: If no add_store was provided
3573 """
3574 if self.add_store is None:
3575 raise NotImplementedError(self.add_object)
3576 return self.add_store.add_object(obj)
3578 def add_objects(
3579 self,
3580 objects: Sequence[tuple[ShaFile, str | None]],
3581 progress: Callable[[str], None] | None = None,
3582 ) -> Pack | None:
3583 """Add multiple objects to the store.
3585 Args:
3586 objects: Iterator of objects to add
3587 progress: Optional progress reporting callback
3589 Raises:
3590 NotImplementedError: If no add_store was provided
3591 """
3592 if self.add_store is None:
3593 raise NotImplementedError(self.add_object)
3594 return self.add_store.add_objects(objects, progress)
3596 @property
3597 def packs(self) -> list[Pack]:
3598 """Get the list of packs from all overlaid stores.
3600 Returns:
3601 Combined list of packs from all base stores
3602 """
3603 ret = []
3604 for b in self.bases:
3605 ret.extend(b.packs)
3606 return ret
3608 def __iter__(self) -> Iterator[ObjectID]:
3609 """Iterate over all object SHAs in the overlaid stores.
3611 Returns:
3612 Iterator of object SHAs (deduped across stores)
3613 """
3614 done = set()
3615 for b in self.bases:
3616 for o_id in b:
3617 if o_id not in done:
3618 yield o_id
3619 done.add(o_id)
3621 def iterobjects_subset(
3622 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
3623 ) -> Iterator[ShaFile]:
3624 """Iterate over a subset of objects from the overlaid stores.
3626 Args:
3627 shas: Iterable of object SHAs to retrieve
3628 allow_missing: If True, skip missing objects; if False, raise KeyError
3630 Returns:
3631 Iterator of ShaFile objects
3633 Raises:
3634 KeyError: If an object is missing and allow_missing is False
3635 """
3636 todo = set(shas)
3637 found: set[ObjectID] = set()
3639 for b in self.bases:
3640 # Create a copy of todo for each base to avoid modifying
3641 # the set while iterating through it
3642 current_todo = todo - found
3643 for o in b.iterobjects_subset(current_todo, allow_missing=True):
3644 yield o
3645 found.add(o.id)
3647 # Check for any remaining objects not found
3648 missing = todo - found
3649 if missing and not allow_missing:
3650 raise KeyError(next(iter(missing)))
3652 def iter_unpacked_subset(
3653 self,
3654 shas: Iterable[ObjectID | RawObjectID],
3655 *,
3656 include_comp: bool = False,
3657 allow_missing: bool = False,
3658 convert_ofs_delta: bool = True,
3659 ) -> Iterator[UnpackedObject]:
3660 """Iterate over unpacked objects from the overlaid stores.
3662 Args:
3663 shas: Iterable of object SHAs to retrieve
3664 include_comp: Whether to include compressed data
3665 allow_missing: If True, skip missing objects; if False, raise KeyError
3666 convert_ofs_delta: Whether to convert OFS_DELTA objects
3668 Returns:
3669 Iterator of unpacked objects
3671 Raises:
3672 KeyError: If an object is missing and allow_missing is False
3673 """
3674 todo: set[ObjectID | RawObjectID] = set(shas)
3675 for b in self.bases:
3676 for o in b.iter_unpacked_subset(
3677 todo,
3678 include_comp=include_comp,
3679 allow_missing=True,
3680 convert_ofs_delta=convert_ofs_delta,
3681 ):
3682 yield o
3683 todo.remove(o.sha())
3684 if todo and not allow_missing:
3685 raise KeyError(next(iter(todo)))
3687 def get_raw(self, name: ObjectID | RawObjectID) -> tuple[int, bytes]:
3688 """Get the raw object data from the overlaid stores.
3690 Args:
3691 name: SHA of the object
3693 Returns:
3694 Tuple of (type_num, raw_data)
3696 Raises:
3697 KeyError: If object not found in any base store
3698 """
3699 for b in self.bases:
3700 try:
3701 return b.get_raw(name)
3702 except KeyError:
3703 pass
3704 raise KeyError(name)
3706 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
3707 """Check if an object is packed in any base store.
3709 Args:
3710 sha: SHA of the object
3712 Returns:
3713 True if object is packed in any base store
3714 """
3715 for b in self.bases:
3716 if b.contains_packed(sha):
3717 return True
3718 return False
3720 def contains_loose(self, sha: ObjectID) -> bool:
3721 """Check if an object is loose in any base store.
3723 Args:
3724 sha: SHA of the object
3726 Returns:
3727 True if object is loose in any base store
3728 """
3729 for b in self.bases:
3730 if b.contains_loose(sha):
3731 return True
3732 return False
3735def read_packs_file(f: BinaryIO) -> Iterator[str]:
3736 """Yield the packs listed in a packs file."""
3737 for line in f.read().splitlines():
3738 if not line:
3739 continue
3740 (kind, name) = line.split(b" ", 1)
3741 if kind != b"P":
3742 continue
3743 yield os.fsdecode(name)
3746class BucketBasedObjectStore(PackBasedObjectStore):
3747 """Object store implementation that uses a bucket store like S3 as backend."""
3749 def _iter_loose_objects(self) -> Iterator[ObjectID]:
3750 """Iterate over the SHAs of all loose objects."""
3751 return iter([])
3753 def _get_loose_object(self, sha: ObjectID) -> None:
3754 return None
3756 def delete_loose_object(self, sha: ObjectID) -> None:
3757 """Delete a loose object (no-op for bucket stores).
3759 Bucket-based stores don't have loose objects, so this is a no-op.
3761 Args:
3762 sha: SHA of the object to delete
3763 """
3764 # Doesn't exist..
3766 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
3767 """Pack loose objects. Returns number of objects packed.
3769 BucketBasedObjectStore doesn't support loose objects, so this is a no-op.
3771 Args:
3772 progress: Optional progress reporting callback (ignored)
3773 """
3774 return 0
3776 def _remove_pack_by_name(self, name: str) -> None:
3777 """Remove a pack by name. Subclasses should implement this."""
3778 raise NotImplementedError(self._remove_pack_by_name)
3780 def _iter_pack_names(self) -> Iterator[str]:
3781 raise NotImplementedError(self._iter_pack_names)
3783 def _get_pack(self, name: str) -> Pack:
3784 raise NotImplementedError(self._get_pack)
3786 def _update_pack_cache(self) -> list[Pack]:
3787 pack_files = set(self._iter_pack_names())
3789 # Open newly appeared pack files
3790 new_packs = []
3791 for f in pack_files:
3792 if f not in self._pack_cache:
3793 pack = self._get_pack(f)
3794 new_packs.append(pack)
3795 self._pack_cache[f] = pack
3796 # Remove disappeared pack files
3797 for f in set(self._pack_cache) - pack_files:
3798 self._pack_cache.pop(f).close()
3799 return new_packs
3801 def _upload_pack(
3802 self, basename: str, pack_file: BinaryIO, index_file: BinaryIO
3803 ) -> None:
3804 raise NotImplementedError
3806 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
3807 """Add a new pack to this object store.
3809 Returns: Fileobject to write to, a commit function to
3810 call when the pack is finished and an abort
3811 function.
3812 """
3813 import tempfile
3815 pf = tempfile.SpooledTemporaryFile(
3816 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3817 )
3819 def commit() -> Pack | None:
3820 if pf.tell() == 0:
3821 pf.close()
3822 return None
3824 pf.seek(0)
3826 p = PackData(pf.name, file=pf, object_format=self.object_format)
3827 entries = p.sorted_entries()
3828 basename = iter_sha1(entry[0] for entry in entries).decode("ascii")
3829 idxf = tempfile.SpooledTemporaryFile(
3830 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3831 )
3832 checksum = p.get_stored_checksum()
3833 write_pack_index(idxf, entries, checksum, version=self.pack_index_version)
3834 idxf.seek(0)
3835 idx = load_pack_index_file(basename + ".idx", idxf, self.object_format)
3836 for pack in self.packs:
3837 if pack.get_stored_checksum() == p.get_stored_checksum():
3838 p.close()
3839 idx.close()
3840 pf.close()
3841 idxf.close()
3842 return pack
3843 pf.seek(0)
3844 idxf.seek(0)
3845 self._upload_pack(basename, pf, idxf) # type: ignore[arg-type]
3846 final_pack = Pack.from_objects(p, idx)
3847 self._add_cached_pack(basename, final_pack)
3848 pf.close()
3849 idxf.close()
3850 return final_pack
3852 return pf, commit, pf.close # type: ignore[return-value]
3855def _collect_ancestors(
3856 store: ObjectContainer,
3857 heads: Iterable[ObjectID],
3858 common: frozenset[ObjectID] = frozenset(),
3859 shallow: frozenset[ObjectID] = frozenset(),
3860 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3861) -> tuple[set[ObjectID], set[ObjectID]]:
3862 """Collect all ancestors of heads up to (excluding) those in common.
3864 Args:
3865 store: Object store to get commits from
3866 heads: commits to start from
3867 common: commits to end at, or empty set to walk repository
3868 completely
3869 shallow: Set of shallow commits
3870 get_parents: Optional function for getting the parents of a
3871 commit.
3872 Returns: a tuple (A, B) where A - all commits reachable
3873 from heads but not present in common, B - common (shared) elements
3874 that are directly reachable from heads
3875 """
3876 bases = set()
3877 commits = set()
3878 queue: list[ObjectID] = []
3879 queue.extend(heads)
3881 # Try to use commit graph if available
3882 commit_graph = store.get_commit_graph()
3884 while queue:
3885 e = queue.pop(0)
3886 if e in common:
3887 bases.add(e)
3888 elif e not in commits:
3889 commits.add(e)
3890 if e in shallow:
3891 continue
3893 # Try to use commit graph for parent lookup
3894 parents = None
3895 if commit_graph:
3896 parents = commit_graph.get_parents(e)
3898 if parents is None:
3899 # Fall back to loading the object
3900 cmt = store[e]
3901 assert isinstance(cmt, Commit)
3902 parents = get_parents(cmt)
3904 queue.extend(parents)
3905 return (commits, bases)
3908def iter_tree_contents(
3909 store: ObjectContainer, tree_id: ObjectID | None, *, include_trees: bool = False
3910) -> Iterator[TreeEntry]:
3911 """Iterate the contents of a tree and all subtrees.
3913 Iteration is depth-first pre-order, as in e.g. os.walk.
3915 Args:
3916 store: Object store to get trees from
3917 tree_id: SHA1 of the tree.
3918 include_trees: If True, include tree objects in the iteration.
3920 Yields: TreeEntry namedtuples for all the objects in a tree.
3921 """
3922 if tree_id is None:
3923 return
3924 # This could be fairly easily generalized to >2 trees if we find a use
3925 # case.
3926 todo = [TreeEntry(b"", stat.S_IFDIR, tree_id)]
3927 while todo:
3928 entry = todo.pop()
3929 assert entry.mode is not None
3930 if stat.S_ISDIR(entry.mode):
3931 extra = []
3932 assert entry.sha is not None
3933 tree = store[entry.sha]
3934 assert isinstance(tree, Tree)
3935 for subentry in tree.iteritems(name_order=True):
3936 assert entry.path is not None
3937 extra.append(subentry.in_path(entry.path))
3938 todo.extend(reversed(extra))
3939 if not stat.S_ISDIR(entry.mode) or include_trees:
3940 yield entry
3943def iter_commit_contents(
3944 store: ObjectContainer,
3945 commit: Commit | ObjectID | RawObjectID,
3946 *,
3947 include: Sequence[str | bytes | Path] | None = None,
3948) -> Iterator[TreeEntry]:
3949 """Iterate the contents of the repository at the specified commit.
3951 This is a wrapper around iter_tree_contents() and
3952 tree_lookup_path() to simplify the common task of getting the
3953 contest of a repo at a particular commit. See also
3954 dulwich.index.build_file_from_blob() for writing individual files
3955 to disk.
3957 Args:
3958 store: Object store to get trees from
3959 commit: Commit object, or SHA1 of a commit
3960 include: if provided, only the entries whose paths are in the
3961 list, or whose parent tree is in the list, will be
3962 included. Note that duplicate or overlapping paths
3963 (e.g. ["foo", "foo/bar"]) may result in duplicate entries
3965 Yields: TreeEntry namedtuples for all matching files in a commit.
3966 """
3967 sha = commit.id if isinstance(commit, Commit) else commit
3968 if not isinstance(obj := store[sha], Commit):
3969 raise TypeError(
3970 f"{sha.decode('ascii')} should be ID of a Commit, but is {type(obj)}"
3971 )
3972 commit = obj
3973 encoding = commit.encoding or "utf-8"
3974 include_bytes: list[bytes] = (
3975 [
3976 path if isinstance(path, bytes) else str(path).encode(encoding)
3977 for path in include
3978 ]
3979 if include is not None
3980 else [b""]
3981 )
3983 for path in include_bytes:
3984 mode, obj_id = tree_lookup_path(store.__getitem__, commit.tree, path)
3985 # Iterate all contained files if path points to a dir, otherwise just get that
3986 # single file
3987 if isinstance(store[obj_id], Tree):
3988 for entry in iter_tree_contents(store, obj_id):
3989 yield entry.in_path(path)
3990 else:
3991 yield TreeEntry(path, mode, obj_id)
3994def peel_sha(
3995 store: ObjectContainer,
3996 sha: ObjectID | RawObjectID,
3997 cache: PeelCache | None = None,
3998) -> tuple[ShaFile, ShaFile]:
3999 """Peel all tags from a SHA.
4001 Args:
4002 store: Object store to get objects from
4003 sha: The object SHA to peel.
4004 cache: Optional mapping from tag id to fully-peeled id, updated as tags
4005 are peeled. Most callers want ObjectContainer.peel(), which supplies a
4006 cache kept on the store.
4007 Returns: The fully-peeled SHA1 of a tag object, after peeling all
4008 intermediate tags; if the original ref does not point to a tag,
4009 this will equal the original SHA1.
4010 """
4011 if cache is None:
4012 cache = {}
4013 unpeeled = obj = store[sha]
4014 seen: list[ObjectID] = []
4015 obj_class = object_class(obj.type_name)
4016 while obj_class is Tag:
4017 assert isinstance(obj, Tag)
4018 cached = cache.get(obj.id)
4019 if cached is not None:
4020 obj = store[cached]
4021 break
4022 seen.append(obj.id)
4023 obj_class, sha = obj.object
4024 obj = store[sha]
4025 for tag_id in seen:
4026 cache[tag_id] = obj.id
4027 return unpeeled, obj
4030class GraphTraversalReachability:
4031 """Naive graph traversal implementation of ObjectReachabilityProvider.
4033 This implementation wraps existing graph traversal functions
4034 (_collect_ancestors, _collect_filetree_revs) to provide the standard
4035 reachability interface without any performance optimizations.
4036 """
4038 def __init__(self, object_store: BaseObjectStore) -> None:
4039 """Initialize the graph traversal provider.
4041 Args:
4042 object_store: Object store to query
4043 """
4044 self.store = object_store
4046 def get_reachable_commits(
4047 self,
4048 heads: Iterable[ObjectID],
4049 exclude: Iterable[ObjectID] | None = None,
4050 shallow: Set[ObjectID] | None = None,
4051 ) -> set[ObjectID]:
4052 """Get all commits reachable from heads, excluding those in exclude.
4054 Uses _collect_ancestors for commit traversal.
4056 Args:
4057 heads: Starting commit SHAs
4058 exclude: Commit SHAs to exclude (and their ancestors)
4059 shallow: Set of shallow commit boundaries
4061 Returns:
4062 Set of commit SHAs reachable from heads but not from exclude
4063 """
4064 exclude_set = frozenset(exclude) if exclude else frozenset()
4065 shallow_set = frozenset(shallow) if shallow else frozenset()
4066 commits, _bases = _collect_ancestors(
4067 self.store, heads, exclude_set, shallow_set
4068 )
4069 return commits
4071 def get_tree_objects(
4072 self,
4073 tree_shas: Iterable[ObjectID],
4074 ) -> set[ObjectID]:
4075 """Get all trees and blobs reachable from the given trees.
4077 Uses _collect_filetree_revs for tree traversal.
4079 Args:
4080 tree_shas: Starting tree SHAs
4082 Returns:
4083 Set of tree and blob SHAs
4084 """
4085 result: set[ObjectID] = set()
4086 for tree_sha in tree_shas:
4087 _collect_filetree_revs(self.store, tree_sha, result)
4088 return result
4090 def get_reachable_objects(
4091 self,
4092 commits: Iterable[ObjectID],
4093 exclude_commits: Iterable[ObjectID] | None = None,
4094 ) -> set[ObjectID]:
4095 """Get all objects (commits + trees + blobs) reachable from commits.
4097 Args:
4098 commits: Starting commit SHAs
4099 exclude_commits: Commits whose objects should be excluded
4101 Returns:
4102 Set of all object SHAs (commits, trees, blobs)
4103 """
4104 commits_set = set(commits)
4105 result = set(commits_set)
4107 # Get trees for all commits
4108 tree_shas = []
4109 for commit_sha in commits_set:
4110 try:
4111 commit = self.store[commit_sha]
4112 if isinstance(commit, Commit):
4113 tree_shas.append(commit.tree)
4114 except KeyError:
4115 # Commit not in store, skip
4116 continue
4118 # Collect all tree/blob objects
4119 result.update(self.get_tree_objects(tree_shas))
4121 # Exclude objects from exclude_commits if needed
4122 if exclude_commits:
4123 exclude_objects = self.get_reachable_objects(exclude_commits, None)
4124 result -= exclude_objects
4126 return result
4129class BitmapReachability:
4130 """Bitmap-accelerated implementation of ObjectReachabilityProvider.
4132 This implementation uses packfile bitmap indexes where available to
4133 accelerate reachability queries. Falls back to graph traversal when
4134 bitmaps don't cover the requested commits.
4135 """
4137 def __init__(self, object_store: "PackBasedObjectStore") -> None:
4138 """Initialize the bitmap provider.
4140 Args:
4141 object_store: Pack-based object store with bitmap support
4142 """
4143 self.store = object_store
4144 # Fallback to graph traversal for operations not yet optimized
4145 self._fallback = GraphTraversalReachability(object_store)
4147 def _combine_commit_bitmaps(
4148 self,
4149 commit_shas: set[ObjectID],
4150 exclude_shas: set[ObjectID] | None = None,
4151 ) -> tuple["EWAHBitmap", "Pack"] | None:
4152 """Combine bitmaps for multiple commits using OR, with optional exclusion.
4154 Args:
4155 commit_shas: Set of commit SHAs to combine
4156 exclude_shas: Optional set of commit SHAs to exclude
4158 Returns:
4159 Tuple of (combined_bitmap, pack) or None if bitmaps unavailable
4160 """
4161 from .bitmap import find_commit_bitmaps
4163 # Find bitmaps for the commits
4164 commit_bitmaps = find_commit_bitmaps(commit_shas, self.store.packs)
4166 # If we can't find bitmaps for all commits, return None
4167 if len(commit_bitmaps) < len(commit_shas):
4168 return None
4170 # Combine bitmaps using OR
4171 combined_bitmap = None
4172 result_pack = None
4174 for commit_sha in commit_shas:
4175 pack, pack_bitmap, _sha_to_pos = commit_bitmaps[commit_sha]
4176 commit_bitmap = pack_bitmap.get_bitmap(commit_sha)
4178 if commit_bitmap is None:
4179 return None
4181 if combined_bitmap is None:
4182 combined_bitmap = commit_bitmap
4183 result_pack = pack
4184 elif pack == result_pack:
4185 # Same pack, can OR directly
4186 combined_bitmap = combined_bitmap | commit_bitmap
4187 else:
4188 # Different packs, can't combine
4189 return None
4191 # Handle exclusions if provided
4192 if exclude_shas and result_pack and combined_bitmap:
4193 exclude_bitmaps = find_commit_bitmaps(exclude_shas, [result_pack])
4195 if len(exclude_bitmaps) == len(exclude_shas):
4196 # All excludes have bitmaps, compute exclusion
4197 exclude_combined = None
4199 for commit_sha in exclude_shas:
4200 _pack, pack_bitmap, _sha_to_pos = exclude_bitmaps[commit_sha]
4201 exclude_bitmap = pack_bitmap.get_bitmap(commit_sha)
4203 if exclude_bitmap is None:
4204 break
4206 if exclude_combined is None:
4207 exclude_combined = exclude_bitmap
4208 else:
4209 exclude_combined = exclude_combined | exclude_bitmap
4211 # Subtract excludes using set difference
4212 if exclude_combined:
4213 combined_bitmap = combined_bitmap - exclude_combined
4215 if combined_bitmap and result_pack:
4216 return (combined_bitmap, result_pack)
4217 return None
4219 def get_reachable_commits(
4220 self,
4221 heads: Iterable[ObjectID],
4222 exclude: Iterable[ObjectID] | None = None,
4223 shallow: Set[ObjectID] | None = None,
4224 ) -> set[ObjectID]:
4225 """Get all commits reachable from heads using bitmaps where possible.
4227 Args:
4228 heads: Starting commit SHAs
4229 exclude: Commit SHAs to exclude (and their ancestors)
4230 shallow: Set of shallow commit boundaries
4232 Returns:
4233 Set of commit SHAs reachable from heads but not from exclude
4234 """
4235 from .bitmap import bitmap_to_object_shas
4237 # If shallow is specified, fall back to graph traversal
4238 # (bitmaps don't support shallow boundaries well)
4239 if shallow:
4240 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4242 heads_set = set(heads)
4243 exclude_set = set(exclude) if exclude else None
4245 # Try to combine bitmaps
4246 result = self._combine_commit_bitmaps(heads_set, exclude_set)
4247 if result is None:
4248 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4250 combined_bitmap, result_pack = result
4252 # Convert bitmap to commit SHAs, filtering for commits only
4253 pack_bitmap = result_pack.bitmap
4254 if pack_bitmap is None:
4255 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4256 commit_type_filter = pack_bitmap.commit_bitmap
4257 return bitmap_to_object_shas(
4258 combined_bitmap, result_pack.index, commit_type_filter
4259 )
4261 def get_tree_objects(
4262 self,
4263 tree_shas: Iterable[ObjectID],
4264 ) -> set[ObjectID]:
4265 """Get all trees and blobs reachable from the given trees.
4267 Args:
4268 tree_shas: Starting tree SHAs
4270 Returns:
4271 Set of tree and blob SHAs
4272 """
4273 # Tree traversal doesn't benefit much from bitmaps, use fallback
4274 return self._fallback.get_tree_objects(tree_shas)
4276 def get_reachable_objects(
4277 self,
4278 commits: Iterable[ObjectID],
4279 exclude_commits: Iterable[ObjectID] | None = None,
4280 ) -> set[ObjectID]:
4281 """Get all objects reachable from commits using bitmaps.
4283 Args:
4284 commits: Starting commit SHAs
4285 exclude_commits: Commits whose objects should be excluded
4287 Returns:
4288 Set of all object SHAs (commits, trees, blobs)
4289 """
4290 from .bitmap import bitmap_to_object_shas
4292 commits_set = set(commits)
4293 exclude_set = set(exclude_commits) if exclude_commits else None
4295 # Try to combine bitmaps
4296 result = self._combine_commit_bitmaps(commits_set, exclude_set)
4297 if result is None:
4298 return self._fallback.get_reachable_objects(commits, exclude_commits)
4300 combined_bitmap, result_pack = result
4302 # Convert bitmap to all object SHAs (no type filter)
4303 return bitmap_to_object_shas(combined_bitmap, result_pack.index, None)