Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/object_store.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# object_store.py -- Object store for git objects
2# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
3# and others
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
24"""Git object store interfaces and implementation."""
26__all__ = [
27 "DEFAULT_TEMPFILE_GRACE_PERIOD",
28 "INFODIR",
29 "PACKDIR",
30 "PACK_MODE",
31 "PEEL_CACHE_SIZE",
32 "BaseObjectStore",
33 "BitmapReachability",
34 "BucketBasedObjectStore",
35 "DiskObjectStore",
36 "GraphTraversalReachability",
37 "GraphWalker",
38 "MemoryObjectStore",
39 "MissingObjectFinder",
40 "ObjectIterator",
41 "ObjectReachabilityProvider",
42 "ObjectStoreGraphWalker",
43 "OverlayObjectStore",
44 "PackBasedObjectStore",
45 "PackCapableObjectStore",
46 "PackContainer",
47 "PackInputTooLarge",
48 "PeelCache",
49 "commit_tree_changes",
50 "find_shallow",
51 "get_depth",
52 "iter_commit_contents",
53 "iter_tree_contents",
54 "peel_sha",
55 "read_packs_file",
56 "tree_lookup_path",
57]
59import binascii
60import logging
61import os
62import stat
63import sys
64import time
65import warnings
66from collections import deque
67from collections.abc import (
68 Callable,
69 Iterable,
70 Iterator,
71 Mapping,
72 Sequence,
73 Set,
74)
75from contextlib import closing, suppress
76from io import BytesIO
77from pathlib import Path
78from typing import (
79 TYPE_CHECKING,
80 BinaryIO,
81 Protocol,
82 TypeVar,
83 cast,
84)
86if TYPE_CHECKING:
87 from .object_format import ObjectFormat
89from .errors import NotTreeError
90from .file import GitFile, SharedPerm, _GitFile, adjust_shared_perm
91from .lru_cache import LRUCache
92from .midx import MultiPackIndex, load_midx
93from .objects import (
94 DEFAULT_LOOSE_OBJECT_SIZE_LIMIT,
95 S_ISGITLINK,
96 Blob,
97 Commit,
98 ObjectID,
99 RawObjectID,
100 ShaFile,
101 Tag,
102 Tree,
103 TreeEntry,
104 hex_to_filename,
105 hex_to_sha,
106 object_class,
107 sha_to_hex,
108 valid_hexsha,
109)
110from .pack import (
111 PACK_SPOOL_FILE_MAX_SIZE,
112 ObjectContainer,
113 Pack,
114 PackData,
115 PackedObjectContainer,
116 PackFileDisappeared,
117 PackHint,
118 PackIndexEntry,
119 PackIndexer,
120 PackInflater,
121 PackStreamCopier,
122 UnpackedObject,
123 extend_pack,
124 full_unpacked_object,
125 generate_unpacked_objects,
126 iter_sha1,
127 load_pack_index_file,
128 pack_objects_to_data,
129 write_pack_data,
130 write_pack_index,
131)
132from .protocol import DEPTH_INFINITE, PEELED_TAG_SUFFIX
133from .refs import Ref
135if TYPE_CHECKING:
136 from .bitmap import EWAHBitmap
137 from .commit_graph import CommitGraph
138 from .config import Config
139 from .diff_tree import RenameDetector
140 from .pack import FilePackIndex, Pack
143logger = logging.getLogger(__name__)
145# Maximum number of times to rescan the pack directory after a pack file
146# disappears between snapshot and lazy open (e.g. concurrent repack).
147# Mirrors git's bounded reprepare_packed_git() retry.
148_MAX_PACK_RESCAN_ATTEMPTS = 3
150# Number of tag-peel results to memoize per object store. Entries are tiny
151# (two object ids), so this mainly bounds growth in long-lived processes.
152PEEL_CACHE_SIZE = 1000
155class PeelCache(Protocol):
156 """Minimal mapping interface peel_sha() needs from a cache."""
158 def get(self, key: ObjectID, default: None = None) -> ObjectID | None:
159 """Return the peeled id for key, or None."""
161 def __setitem__(self, key: ObjectID, value: ObjectID) -> None:
162 """Record the peeled id for key."""
165_T = TypeVar("_T")
168class GraphWalker(Protocol):
169 """Protocol for graph walker objects.
171 Implementations may also expose a ``shallow`` set, an ``unshallow`` set,
172 and an ``update_shallow`` callable for shallow-clone negotiation. These
173 are not part of the minimal protocol and callers must use ``hasattr`` or
174 ``getattr`` to access them.
175 """
177 def __next__(self) -> ObjectID | None:
178 """Return the next object SHA to visit."""
179 ...
181 def ack(self, sha: ObjectID, /) -> None:
182 """Acknowledge that an object has been received."""
183 ...
185 def nak(self) -> None:
186 """Nothing in common was found."""
187 ...
190class ObjectReachabilityProvider(Protocol):
191 """Protocol for computing object reachability queries.
193 This abstraction allows reachability computations to be backed by either
194 naive graph traversal or optimized bitmap indexes, with a consistent interface.
195 """
197 def get_reachable_commits(
198 self,
199 heads: Iterable[ObjectID],
200 exclude: Iterable[ObjectID] | None = None,
201 shallow: Set[ObjectID] | None = None,
202 ) -> set[ObjectID]:
203 """Get all commits reachable from heads, excluding those in exclude.
205 Args:
206 heads: Starting commit SHAs
207 exclude: Commit SHAs to exclude (and their ancestors)
208 shallow: Set of shallow commit boundaries (traversal stops here)
210 Returns:
211 Set of commit SHAs reachable from heads but not from exclude
212 """
213 ...
215 def get_reachable_objects(
216 self,
217 commits: Iterable[ObjectID],
218 exclude_commits: Iterable[ObjectID] | None = None,
219 ) -> set[ObjectID]:
220 """Get all objects (commits + trees + blobs) reachable from commits.
222 Args:
223 commits: Starting commit SHAs
224 exclude_commits: Commits whose objects should be excluded
226 Returns:
227 Set of all object SHAs (commits, trees, blobs, tags)
228 """
229 ...
231 def get_tree_objects(
232 self,
233 tree_shas: Iterable[ObjectID],
234 ) -> set[ObjectID]:
235 """Get all trees and blobs reachable from the given trees.
237 Args:
238 tree_shas: Starting tree SHAs
240 Returns:
241 Set of tree and blob SHAs
242 """
243 ...
246INFODIR = "info"
247PACKDIR = "pack"
249# use permissions consistent with Git; just readable by everyone
250# TODO: should packs also be non-writable on Windows? if so, that
251# would requite some rather significant adjustments to the test suite
252PACK_MODE = 0o444 if sys.platform != "win32" else 0o644
254# Grace period for cleaning up temporary pack files (in seconds)
255# Matches git's default of 2 weeks
256DEFAULT_TEMPFILE_GRACE_PERIOD = 14 * 24 * 60 * 60 # 2 weeks
259def _remove_readonly(path: str) -> None:
260 """Remove a file, clearing the read-only attribute first on Windows.
262 git stores pack files and loose objects read-only. Unix lets you unlink a
263 read-only file in a writable directory, but Windows refuses with
264 PermissionError, so clear the attribute and retry there.
265 """
266 try:
267 os.remove(path)
268 except PermissionError:
269 if sys.platform != "win32":
270 raise
271 os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
272 os.remove(path)
275class PackInputTooLarge(OSError):
276 """Raised when a received pack exceeds the configured input size cap.
278 Mirrors the failure mode of git's ``receive.maxInputSize`` /
279 ``git index-pack --max-input-size``.
280 """
283def _bound_read_callables(
284 read_all: Callable[[int], bytes],
285 read_some: Callable[[int], bytes] | None,
286 max_input_size: int,
287) -> tuple[Callable[[int], bytes], Callable[[int], bytes] | None]:
288 """Wrap pack-stream read callbacks so total bytes are capped.
290 When the cumulative number of bytes returned across ``read_all`` and
291 ``read_some`` exceeds ``max_input_size``, the next read raises
292 ``PackInputTooLarge``. This is the in-process analogue of
293 ``git index-pack --max-input-size``.
294 """
295 bytes_read = [0]
297 def _check(n: int) -> None:
298 bytes_read[0] += n
299 if bytes_read[0] > max_input_size:
300 raise PackInputTooLarge(
301 f"pack exceeds maximum input size of {max_input_size} bytes"
302 )
304 def wrapped_read_all(n: int) -> bytes:
305 data = read_all(n)
306 _check(len(data))
307 return data
309 if read_some is None:
310 return wrapped_read_all, None
312 def wrapped_read_some(n: int) -> bytes:
313 data = read_some(n)
314 _check(len(data))
315 return data
317 return wrapped_read_all, wrapped_read_some
320def find_shallow(
321 store: ObjectContainer, heads: Iterable[ObjectID], depth: int
322) -> tuple[set[ObjectID], set[ObjectID]]:
323 """Find shallow commits according to a given depth.
325 Args:
326 store: An ObjectStore for looking up objects.
327 heads: Iterable of head SHAs to start walking from.
328 depth: The depth of ancestors to include. A depth of one includes
329 only the heads themselves.
330 Returns: A tuple of (shallow, not_shallow), sets of SHAs that should be
331 considered shallow and unshallow according to the arguments. Note that
332 these sets may overlap if a commit is reachable along multiple paths.
333 """
334 parents: dict[ObjectID, list[ObjectID]] = {}
335 commit_graph = store.get_commit_graph()
337 def get_parents(sha: ObjectID) -> list[ObjectID]:
338 result = parents.get(sha, None)
339 if not result:
340 # Try to use commit graph first if available
341 if commit_graph:
342 graph_parents = commit_graph.get_parents(sha)
343 if graph_parents is not None:
344 result = graph_parents
345 parents[sha] = result
346 return result
347 # Fall back to loading the object
348 commit = store[sha]
349 assert isinstance(commit, Commit)
350 result = commit.parents
351 parents[sha] = result
352 return result
354 todo = [] # stack of (sha, depth)
355 for head_sha in heads:
356 obj = store[head_sha]
357 # Peel tags if necessary
358 while isinstance(obj, Tag):
359 _, sha = obj.object
360 obj = store[sha]
361 if isinstance(obj, Commit):
362 todo.append((obj.id, 1))
364 not_shallow = set()
365 shallow = set()
366 # A commit reachable along N distinct paths was popped and re-expanded N
367 # times, so a merge-heavy history walked in exponential time. Deduplicate
368 # on the (sha, depth) state: re-processing a state adds to the same set and
369 # pushes the same parents, so skipping repeats leaves the result unchanged.
370 seen: set[tuple[ObjectID, int]] = set()
371 while todo:
372 state = todo.pop()
373 if state in seen:
374 continue
375 seen.add(state)
376 sha, cur_depth = state
377 if cur_depth < depth:
378 not_shallow.add(sha)
379 new_depth = cur_depth + 1
380 todo.extend((p, new_depth) for p in get_parents(sha))
381 else:
382 shallow.add(sha)
384 return shallow, not_shallow
387def get_depth(
388 store: ObjectContainer,
389 head: ObjectID,
390 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
391 max_depth: int | None = None,
392) -> int:
393 """Return the current available depth for the given head.
395 For commits with multiple parents, the largest possible depth will be
396 returned.
398 Args:
399 store: Object store to search in
400 head: commit to start from
401 get_parents: optional function for getting the parents of a commit
402 max_depth: maximum depth to search
403 """
404 if head not in store:
405 return 0
406 current_depth = 1
407 queue = deque([(head, current_depth)])
408 commit_graph = store.get_commit_graph()
410 # Without deduplication a commit reachable along several paths is expanded
411 # once per path, so a merge-heavy history is walked in exponential time.
412 # Track the (sha, depth) states already queued; re-visiting one only
413 # recomputes the same max and re-queues the same parents. deque.popleft()
414 # keeps the O(1) breadth-first order that list.pop(0) made O(n).
415 seen: set[tuple[ObjectID, int]] = set()
416 while queue and (max_depth is None or current_depth < max_depth):
417 e, depth = queue.popleft()
418 if (e, depth) in seen:
419 continue
420 seen.add((e, depth))
421 current_depth = max(current_depth, depth)
423 # Try to use commit graph for parent lookup if available
424 parents = None
425 if commit_graph:
426 parents = commit_graph.get_parents(e)
428 if parents is None:
429 # Fall back to loading the object
430 cmt = store[e]
431 if isinstance(cmt, Tag):
432 _cls, sha = cmt.object
433 cmt = store[sha]
434 parents = get_parents(cmt)
436 queue.extend((parent, depth + 1) for parent in parents if parent in store)
437 return current_depth
440class PackContainer(Protocol):
441 """Protocol for containers that can accept pack files."""
443 def add_pack(self) -> tuple[BytesIO, Callable[[], None], Callable[[], None]]:
444 """Add a new pack."""
445 ...
448class BaseObjectStore:
449 """Object store interface."""
451 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
452 """Initialize object store.
454 Args:
455 object_format: Object format to use (defaults to DEFAULT_OBJECT_FORMAT)
456 """
457 from .object_format import DEFAULT_OBJECT_FORMAT
459 self.object_format = object_format if object_format else DEFAULT_OBJECT_FORMAT
461 def peel(self, sha: ObjectID | RawObjectID) -> tuple[ShaFile, ShaFile]:
462 """Peel all tags from a SHA, memoizing shared tag chains.
464 Peeling each ref separately is quadratic when many refs share one tag
465 chain, so results are kept in a bounded cache on the store. Git objects
466 are immutable, so an entry can only go stale if the objects behind it
467 are deleted; the deletion paths call _invalidate_peel_cache().
469 Args:
470 sha: The object SHA to peel.
471 Returns: Tuple of (unpeeled, peeled) objects.
472 """
473 return peel_sha(self, sha, cache=self._peel_cache)
475 @property
476 def _peel_cache(self) -> "LRUCache[ObjectID, ObjectID]":
477 cache: LRUCache[ObjectID, ObjectID] | None = getattr(
478 self, "_BaseObjectStore__peel_cache", None
479 )
480 if cache is None:
481 cache = LRUCache(max_cache=PEEL_CACHE_SIZE)
482 self.__peel_cache = cache
483 return cache
485 def _invalidate_peel_cache(self) -> None:
486 """Drop memoized peel results, after objects may have been removed."""
487 self._peel_cache.clear()
489 def determine_wants_all(
490 self, refs: Mapping[Ref, ObjectID], depth: int | None = None
491 ) -> list[ObjectID]:
492 """Determine which objects are wanted based on refs."""
494 def _want_deepen(sha: ObjectID) -> bool:
495 if not depth:
496 return False
497 if depth == DEPTH_INFINITE:
498 return True
499 return depth > self._get_depth(sha)
501 return [
502 sha
503 for (ref, sha) in refs.items()
504 if (sha not in self or _want_deepen(sha))
505 and not ref.endswith(PEELED_TAG_SUFFIX)
506 ]
508 def contains_loose(self, sha: ObjectID) -> bool:
509 """Check if a particular object is present by SHA1 and is loose."""
510 raise NotImplementedError(self.contains_loose)
512 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
513 """Check if a particular object is present by SHA1 and is packed."""
514 return False # Default implementation for stores that don't support packing
516 def __contains__(self, sha1: ObjectID) -> bool:
517 """Check if a particular object is present by SHA1.
519 This method makes no distinction between loose and packed objects.
520 """
521 return self.contains_loose(sha1)
523 @property
524 def packs(self) -> list[Pack]:
525 """Iterable of pack objects."""
526 raise NotImplementedError
528 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
529 """Obtain the raw text for an object.
531 Args:
532 name: sha for the object.
533 Returns: tuple with numeric type and object contents.
534 """
535 raise NotImplementedError(self.get_raw)
537 def __getitem__(self, sha1: ObjectID | RawObjectID) -> ShaFile:
538 """Obtain an object by SHA1.
540 Raises:
541 ChecksumMismatch: if the stored contents do not hash to the
542 requested object id.
543 """
544 if len(sha1) == self.object_format.oid_length:
545 hexsha = sha_to_hex(RawObjectID(sha1))
546 else:
547 hexsha = ObjectID(sha1)
548 type_num, uncomp = self.get_raw(sha1)
549 return ShaFile.from_raw_string(
550 type_num, uncomp, verify_sha=hexsha, object_format=self.object_format
551 )
553 def __iter__(self) -> Iterator[ObjectID]:
554 """Iterate over the SHAs that are present in this store."""
555 raise NotImplementedError(self.__iter__)
557 def add_object(self, obj: ShaFile) -> None:
558 """Add a single object to this object store."""
559 raise NotImplementedError(self.add_object)
561 def add_objects(
562 self,
563 objects: Sequence[tuple[ShaFile, str | None]],
564 progress: Callable[..., None] | None = None,
565 ) -> "Pack | None":
566 """Add a set of objects to this object store.
568 Args:
569 objects: Iterable over a list of (object, path) tuples
570 progress: Optional progress callback
571 """
572 raise NotImplementedError(self.add_objects)
574 def get_reachability_provider(
575 self, prefer_bitmaps: bool = True
576 ) -> ObjectReachabilityProvider:
577 """Get a reachability provider for this object store.
579 Returns an ObjectReachabilityProvider that can efficiently compute
580 object reachability queries. Subclasses can override this to provide
581 optimized implementations (e.g., using bitmap indexes).
583 Args:
584 prefer_bitmaps: Whether to prefer bitmap-based reachability if
585 available.
587 Returns:
588 ObjectReachabilityProvider instance
589 """
590 return GraphTraversalReachability(self)
592 def tree_changes(
593 self,
594 source: ObjectID | None,
595 target: ObjectID | None,
596 want_unchanged: bool = False,
597 include_trees: bool = False,
598 change_type_same: bool = False,
599 rename_detector: "RenameDetector | None" = None,
600 paths: Sequence[bytes] | None = None,
601 ) -> Iterator[
602 tuple[
603 tuple[bytes | None, bytes | None],
604 tuple[int | None, int | None],
605 tuple[ObjectID | None, ObjectID | None],
606 ]
607 ]:
608 """Find the differences between the contents of two trees.
610 Args:
611 source: SHA1 of the source tree
612 target: SHA1 of the target tree
613 want_unchanged: Whether unchanged files should be reported
614 include_trees: Whether to include trees
615 change_type_same: Whether to report files changing
616 type in the same entry.
617 rename_detector: RenameDetector object for detecting renames.
618 paths: Optional list of paths to filter to (as bytes).
619 Returns: Iterator over tuples with
620 (oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
621 """
622 from .diff_tree import tree_changes
624 for change in tree_changes(
625 self,
626 source,
627 target,
628 want_unchanged=want_unchanged,
629 include_trees=include_trees,
630 change_type_same=change_type_same,
631 rename_detector=rename_detector,
632 paths=paths,
633 ):
634 old_path = change.old.path if change.old is not None else None
635 new_path = change.new.path if change.new is not None else None
636 old_mode = change.old.mode if change.old is not None else None
637 new_mode = change.new.mode if change.new is not None else None
638 old_sha = change.old.sha if change.old is not None else None
639 new_sha = change.new.sha if change.new is not None else None
640 yield (
641 (old_path, new_path),
642 (old_mode, new_mode),
643 (old_sha, new_sha),
644 )
646 def iter_tree_contents(
647 self, tree_id: ObjectID, include_trees: bool = False
648 ) -> Iterator[TreeEntry]:
649 """Iterate the contents of a tree and all subtrees.
651 Iteration is depth-first pre-order, as in e.g. os.walk.
653 Args:
654 tree_id: SHA1 of the tree.
655 include_trees: If True, include tree objects in the iteration.
656 Returns: Iterator over TreeEntry namedtuples for all the objects in a
657 tree.
658 """
659 warnings.warn(
660 "Please use dulwich.object_store.iter_tree_contents",
661 DeprecationWarning,
662 stacklevel=2,
663 )
664 return iter_tree_contents(self, tree_id, include_trees=include_trees)
666 def iterobjects_subset(
667 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
668 ) -> Iterator[ShaFile]:
669 """Iterate over a subset of objects in the store.
671 Args:
672 shas: Iterable of object SHAs to retrieve
673 allow_missing: If True, skip missing objects; if False, raise KeyError
675 Returns:
676 Iterator of ShaFile objects
678 Raises:
679 KeyError: If an object is missing and allow_missing is False
680 """
681 for sha in shas:
682 try:
683 yield self[sha]
684 except KeyError:
685 if not allow_missing:
686 raise
688 def iter_unpacked_subset(
689 self,
690 shas: Iterable[ObjectID | RawObjectID],
691 *,
692 include_comp: bool = False,
693 allow_missing: bool = False,
694 convert_ofs_delta: bool = True,
695 ) -> "Iterator[UnpackedObject]":
696 """Iterate over unpacked objects for a subset of SHAs.
698 Default implementation that converts ShaFile objects to UnpackedObject.
699 Subclasses may override for more efficient unpacked access.
701 Args:
702 shas: Iterable of object SHAs to retrieve
703 include_comp: Whether to include compressed data (ignored in base
704 implementation)
705 allow_missing: If True, skip missing objects; if False, raise
706 KeyError
707 convert_ofs_delta: Whether to convert OFS_DELTA objects (ignored in
708 base implementation)
710 Returns:
711 Iterator of UnpackedObject instances
713 Raises:
714 KeyError: If an object is missing and allow_missing is False
715 """
716 from .pack import UnpackedObject
718 for sha in shas:
719 try:
720 obj = self[sha]
721 # Convert ShaFile to UnpackedObject
722 unpacked = UnpackedObject(
723 obj.type_num, decomp_chunks=obj.as_raw_chunks(), sha=obj.id
724 )
725 yield unpacked
726 except KeyError:
727 if not allow_missing:
728 raise
730 def find_missing_objects(
731 self,
732 haves: Iterable[ObjectID],
733 wants: Iterable[ObjectID],
734 shallow: Set[ObjectID] | None = None,
735 progress: Callable[..., None] | None = None,
736 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
737 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
738 ) -> Iterator[tuple[ObjectID, PackHint | None]]:
739 """Find the missing objects required for a set of revisions.
741 Args:
742 haves: Iterable over SHAs already in common.
743 wants: Iterable over SHAs of objects to fetch.
744 shallow: Set of shallow commit SHA1s to skip
745 progress: Simple progress function that will be called with
746 updated progress strings.
747 get_tagged: Function that returns a dict of pointed-to sha ->
748 tag sha for including tags.
749 get_parents: Optional function for getting the parents of a
750 commit.
751 Returns: Iterator over (sha, path) pairs.
752 """
753 warnings.warn("Please use MissingObjectFinder(store)", DeprecationWarning)
754 finder = MissingObjectFinder(
755 self,
756 haves=haves,
757 wants=wants,
758 shallow=shallow,
759 progress=progress,
760 get_tagged=get_tagged,
761 get_parents=get_parents,
762 )
763 return iter(finder)
765 def find_common_revisions(self, graphwalker: GraphWalker) -> list[ObjectID]:
766 """Find which revisions this store has in common using graphwalker.
768 Args:
769 graphwalker: A graphwalker object.
770 Returns: List of SHAs that are in common
771 """
772 haves = []
773 sha = next(graphwalker)
774 while sha:
775 if sha in self:
776 haves.append(sha)
777 graphwalker.ack(sha)
778 sha = next(graphwalker)
779 return haves
781 def generate_pack_data(
782 self,
783 have: Iterable[ObjectID],
784 want: Iterable[ObjectID],
785 *,
786 shallow: Set[ObjectID] | None = None,
787 progress: Callable[..., None] | None = None,
788 ofs_delta: bool = True,
789 ) -> tuple[int, Iterator[UnpackedObject]]:
790 """Generate pack data objects for a set of wants/haves.
792 Args:
793 have: List of SHA1s of objects that should not be sent
794 want: List of SHA1s of objects that should be sent
795 shallow: Set of shallow commit SHA1s to skip
796 ofs_delta: Whether OFS deltas can be included
797 progress: Optional progress reporting method
798 """
799 # Note that the pack-specific implementation below is more efficient,
800 # as it reuses deltas
801 missing_objects = MissingObjectFinder(
802 self, haves=have, wants=want, shallow=shallow, progress=progress
803 )
804 object_ids = list(missing_objects)
805 return pack_objects_to_data(
806 [(self[oid], path) for oid, path in object_ids],
807 ofs_delta=ofs_delta,
808 progress=progress,
809 )
811 def peel_sha(self, sha: ObjectID | RawObjectID) -> ObjectID:
812 """Peel all tags from a SHA.
814 Args:
815 sha: The object SHA to peel.
816 Returns: The fully-peeled SHA1 of a tag object, after peeling all
817 intermediate tags; if the original ref does not point to a tag,
818 this will equal the original SHA1.
819 """
820 warnings.warn(
821 "Please use dulwich.object_store.peel_sha()",
822 DeprecationWarning,
823 stacklevel=2,
824 )
825 return peel_sha(self, sha)[1].id
827 def _get_depth(
828 self,
829 head: ObjectID,
830 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
831 max_depth: int | None = None,
832 ) -> int:
833 """Return the current available depth for the given head.
835 For commits with multiple parents, the largest possible depth will be
836 returned.
838 Args:
839 head: commit to start from
840 get_parents: optional function for getting the parents of a commit
841 max_depth: maximum depth to search
842 """
843 return get_depth(self, head, get_parents=get_parents, max_depth=max_depth)
845 def close(self) -> None:
846 """Close any files opened by this object store."""
847 # Default implementation is a NO-OP
849 def prune(self, grace_period: int | None = None) -> None:
850 """Prune/clean up this object store.
852 This includes removing orphaned temporary files and other
853 housekeeping tasks. Default implementation is a NO-OP.
855 Args:
856 grace_period: Grace period in seconds for removing temporary files.
857 If None, uses the default grace period.
858 """
859 # Default implementation is a NO-OP
861 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
862 """Iterate over all SHA1s that start with a given prefix.
864 The default implementation is a naive iteration over all objects.
865 However, subclasses may override this method with more efficient
866 implementations.
867 """
868 for sha in self:
869 if sha.startswith(prefix):
870 yield sha
872 def get_commit_graph(self) -> "CommitGraph | None":
873 """Get the commit graph for this object store.
875 Returns:
876 CommitGraph object if available, None otherwise
877 """
878 return None
880 def write_commit_graph(
881 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
882 ) -> None:
883 """Write a commit graph file for this object store.
885 Args:
886 refs: List of refs to include. If None, includes all refs from object store.
887 reachable: If True, includes all commits reachable from refs.
888 If False, only includes the direct ref targets.
890 Note:
891 Default implementation does nothing. Subclasses should override
892 this method to provide commit graph writing functionality.
893 """
894 raise NotImplementedError(self.write_commit_graph)
896 def get_object_mtime(self, sha: ObjectID) -> float:
897 """Get the modification time of an object.
899 Args:
900 sha: SHA1 of the object
902 Returns:
903 Modification time as seconds since epoch
905 Raises:
906 KeyError: if the object is not found
907 """
908 # Default implementation raises KeyError
909 # Subclasses should override to provide actual mtime
910 raise KeyError(sha)
913class PackCapableObjectStore(BaseObjectStore, PackedObjectContainer):
914 """Object store that supports pack operations.
916 This is a base class for object stores that can handle pack files,
917 including both disk-based and memory-based stores.
918 """
920 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
921 """Add a new pack to this object store.
923 Returns: Tuple of (file, commit_func, abort_func)
924 """
925 raise NotImplementedError(self.add_pack)
927 def add_pack_data(
928 self,
929 count: int,
930 unpacked_objects: Iterator["UnpackedObject"],
931 progress: Callable[..., None] | None = None,
932 ) -> "Pack | None":
933 """Add pack data to this object store.
935 Args:
936 count: Number of objects
937 unpacked_objects: Iterator over unpacked objects
938 progress: Optional progress callback
939 """
940 raise NotImplementedError(self.add_pack_data)
942 def get_unpacked_object(
943 self, sha1: ObjectID | RawObjectID, *, include_comp: bool = False
944 ) -> "UnpackedObject":
945 """Get a raw unresolved object.
947 Args:
948 sha1: SHA-1 hash of the object
949 include_comp: Whether to include compressed data
951 Returns:
952 UnpackedObject instance
953 """
954 from .pack import UnpackedObject
956 obj = self[sha1]
957 return UnpackedObject(obj.type_num, sha=sha1, decomp_chunks=obj.as_raw_chunks())
959 def iterobjects_subset(
960 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
961 ) -> Iterator[ShaFile]:
962 """Iterate over a subset of objects.
964 Args:
965 shas: Iterable of object SHAs to retrieve
966 allow_missing: If True, skip missing objects
968 Returns:
969 Iterator of ShaFile objects
970 """
971 for sha in shas:
972 try:
973 yield self[sha]
974 except KeyError:
975 if not allow_missing:
976 raise
979class PackBasedObjectStore(PackCapableObjectStore, PackedObjectContainer):
980 """Object store that uses pack files for storage.
982 This class provides a base implementation for object stores that use
983 Git pack files as their primary storage mechanism. It handles caching
984 of open pack files and provides configuration for pack file operations.
985 """
987 def __init__(
988 self,
989 pack_compression_level: int = -1,
990 pack_index_version: int | None = None,
991 pack_delta_window_size: int | None = None,
992 pack_window_memory: int | None = None,
993 pack_delta_cache_size: int | None = None,
994 pack_depth: int | None = None,
995 pack_threads: int | None = None,
996 pack_big_file_threshold: int | None = None,
997 *,
998 packed_git_limit: int | None = None,
999 delta_base_cache_limit: int | None = None,
1000 object_format: "ObjectFormat | None" = None,
1001 ) -> None:
1002 """Initialize a PackBasedObjectStore.
1004 Args:
1005 pack_compression_level: Compression level for pack files (-1 to 9)
1006 pack_index_version: Pack index version to use
1007 pack_delta_window_size: Window size for delta compression
1008 pack_window_memory: Maximum memory to use for delta window
1009 pack_delta_cache_size: Cache size for delta operations
1010 pack_depth: Maximum depth for pack deltas
1011 pack_threads: Number of threads to use for packing
1012 pack_big_file_threshold: Threshold for treating files as "big"
1013 packed_git_limit: Maximum total bytes for mmapped pack files.
1014 When exceeded, least-recently-used packs are closed to free memory.
1015 delta_base_cache_limit: Maximum bytes for caching delta base objects.
1016 Controls memory used to cache resolved base objects during delta
1017 unpacking, corresponding to Git's core.deltaBaseCacheLimit.
1018 object_format: Hash algorithm to use
1019 """
1020 super().__init__(object_format=object_format)
1021 self._pack_cache: dict[str, Pack] = {}
1022 self._pack_access_order: list[str] = []
1023 self.packed_git_limit = packed_git_limit
1024 self.delta_base_cache_limit = delta_base_cache_limit
1025 self.pack_compression_level = pack_compression_level
1026 self.pack_index_version = pack_index_version
1027 self.pack_delta_window_size = pack_delta_window_size
1028 self.pack_window_memory = pack_window_memory
1029 self.pack_delta_cache_size = pack_delta_cache_size
1030 self.pack_depth = pack_depth
1031 self.pack_threads = pack_threads
1032 self.pack_big_file_threshold = pack_big_file_threshold
1034 def get_reachability_provider(
1035 self,
1036 prefer_bitmaps: bool = True,
1037 ) -> ObjectReachabilityProvider:
1038 """Get the best reachability provider for the object store.
1040 Args:
1041 prefer_bitmaps: Whether to use bitmaps if available
1043 Returns:
1044 ObjectReachabilityProvider implementation (either bitmap-accelerated
1045 or graph traversal)
1046 """
1047 if prefer_bitmaps:
1048 # Check if any packs have bitmaps. ``self.packs`` rescans the pack
1049 # directory, so a pack removed by a concurrent repack is dropped
1050 # from the cache and its replacement, if any, is probed here too.
1051 for pack in self.packs:
1052 try:
1053 if pack.bitmap is not None:
1054 return BitmapReachability(self)
1055 except FileNotFoundError:
1056 # Bitmap file doesn't exist for this pack
1057 continue
1058 except PackFileDisappeared as exc:
1059 # The pack vanished between the scan and the bitmap probe.
1060 self._evict_pack(exc.obj)
1061 continue
1063 # Fall back to graph traversal
1064 return GraphTraversalReachability(self)
1066 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
1067 """Add a new pack to this object store."""
1068 raise NotImplementedError(self.add_pack)
1070 def add_pack_data(
1071 self,
1072 count: int,
1073 unpacked_objects: Iterator[UnpackedObject],
1074 progress: Callable[..., None] | None = None,
1075 ) -> "Pack | None":
1076 """Add pack data to this object store.
1078 Args:
1079 count: Number of items to add
1080 unpacked_objects: Iterator of UnpackedObject instances
1081 progress: Optional progress callback
1082 """
1083 if count == 0:
1084 # Don't bother writing an empty pack file
1085 return None
1086 f, commit, abort = self.add_pack()
1087 try:
1088 write_pack_data(
1089 f.write,
1090 unpacked_objects,
1091 num_records=count,
1092 progress=progress,
1093 compression_level=self.pack_compression_level,
1094 object_format=self.object_format,
1095 )
1096 except BaseException:
1097 abort()
1098 raise
1099 else:
1100 return commit()
1102 @property
1103 def alternates(self) -> list["BaseObjectStore"]:
1104 """Return list of alternate object stores."""
1105 return []
1107 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
1108 """Check if a particular object is present by SHA1 and is packed.
1110 This does not check alternates.
1111 """
1113 def lookup(p: "Pack") -> bool:
1114 if sha in p:
1115 return True
1116 raise KeyError
1118 try:
1119 return self._lookup_in_packs(lookup)
1120 except KeyError:
1121 return False
1123 def __contains__(self, sha: ObjectID) -> bool:
1124 """Check if a particular object is present by SHA1.
1126 This method makes no distinction between loose and packed objects.
1127 """
1128 if self.contains_packed(sha) or self.contains_loose(sha):
1129 return True
1130 for alternate in self.alternates:
1131 if sha in alternate:
1132 return True
1133 return False
1135 def _add_cached_pack(self, base_name: str, pack: Pack) -> None:
1136 """Add a newly appeared pack to the cache by path."""
1137 prev_pack = self._pack_cache.get(base_name)
1138 if prev_pack is not pack:
1139 self._pack_cache[base_name] = pack
1140 if prev_pack:
1141 prev_pack.close()
1142 self._mark_pack_used(base_name)
1143 self._enforce_packed_git_limit()
1145 def generate_pack_data(
1146 self,
1147 have: Iterable[ObjectID],
1148 want: Iterable[ObjectID],
1149 *,
1150 shallow: Set[ObjectID] | None = None,
1151 progress: Callable[..., None] | None = None,
1152 ofs_delta: bool = True,
1153 ) -> tuple[int, Iterator[UnpackedObject]]:
1154 """Generate pack data objects for a set of wants/haves.
1156 Args:
1157 have: List of SHA1s of objects that should not be sent
1158 want: List of SHA1s of objects that should be sent
1159 shallow: Set of shallow commit SHA1s to skip
1160 ofs_delta: Whether OFS deltas can be included
1161 progress: Optional progress reporting method
1162 """
1163 missing_objects = MissingObjectFinder(
1164 self, haves=have, wants=want, shallow=shallow, progress=progress
1165 )
1166 remote_has = missing_objects.get_remote_has()
1167 object_ids = list(missing_objects)
1168 return len(object_ids), generate_unpacked_objects(
1169 self,
1170 object_ids,
1171 progress=progress,
1172 ofs_delta=ofs_delta,
1173 other_haves=remote_has,
1174 )
1176 def _clear_cached_packs(self) -> None:
1177 pack_cache = self._pack_cache
1178 self._pack_cache = {}
1179 self._pack_access_order = []
1180 while pack_cache:
1181 (_name, pack) = pack_cache.popitem()
1182 pack.close()
1184 def _total_pack_mmap_size(self) -> int:
1185 """Return the total mmapped memory across all cached packs."""
1186 return sum(pack.mmap_size for pack in self._pack_cache.values())
1188 def _mark_pack_used(self, pack_hash: str) -> None:
1189 """Mark a pack as recently used for LRU tracking."""
1190 try:
1191 self._pack_access_order.remove(pack_hash)
1192 except ValueError:
1193 pass
1194 self._pack_access_order.append(pack_hash)
1196 def _enforce_packed_git_limit(self) -> None:
1197 """Evict least-recently-used packs if the memory limit is exceeded."""
1198 if self.packed_git_limit is None:
1199 return
1200 while (
1201 self._pack_access_order
1202 and self._total_pack_mmap_size() > self.packed_git_limit
1203 ):
1204 oldest = self._pack_access_order.pop(0)
1205 pack = self._pack_cache.get(oldest)
1206 if pack is not None:
1207 pack.close()
1208 del self._pack_cache[oldest]
1210 def _iter_cached_packs(self) -> Iterator[Pack]:
1211 return iter(list(self._pack_cache.values()))
1213 def _evict_pack(self, pack: "Pack | FilePackIndex") -> None:
1214 """Evict a pack from the cache after its backing file disappeared.
1216 ``pack`` may be a :class:`Pack` or a :class:`FilePackIndex`; in the
1217 latter case the index's owning ``Pack`` is matched via the cached
1218 pack's ``_idx`` reference.
1219 """
1220 for key, cached in list(self._pack_cache.items()):
1221 if cached is pack or cached._idx is pack:
1222 del self._pack_cache[key]
1223 try:
1224 self._pack_access_order.remove(key)
1225 except ValueError:
1226 pass
1227 try:
1228 cached.close()
1229 except OSError:
1230 pass
1231 break
1233 def _lookup_in_packs(self, lookup: "Callable[[Pack], _T]") -> "_T":
1234 """Run ``lookup(pack)`` against each cached pack and return the first hit.
1236 ``lookup`` should raise ``KeyError`` if the pack does not contain the
1237 target. ``PackFileDisappeared`` from a concurrent ``git repack`` /
1238 ``gc --auto`` is caught: the stale pack is evicted, the pack
1239 directory is rescanned, and the search retries — bounded, mirroring
1240 git's ``reprepare_packed_git()``. If no cached pack has the object
1241 the pack directory is rescanned once to pick up any newly-arrived
1242 packs (e.g. another writer just landed one). ``KeyError`` is raised
1243 if no pack — old or new — has the object.
1244 """
1245 rescanned = False
1246 for _attempt in range(_MAX_PACK_RESCAN_ATTEMPTS):
1247 disappeared = False
1248 for pack_hash, pack in list(self._pack_cache.items()):
1249 try:
1250 result = lookup(pack)
1251 except KeyError:
1252 continue
1253 except PackFileDisappeared as exc:
1254 self._evict_pack(exc.obj)
1255 disappeared = True
1256 continue
1257 self._mark_pack_used(pack_hash)
1258 self._enforce_packed_git_limit()
1259 return result
1260 if disappeared:
1261 self._update_pack_cache()
1262 rescanned = True
1263 continue
1264 if not rescanned:
1265 # Maybe another process just landed a pack with the object.
1266 if self._update_pack_cache():
1267 rescanned = True
1268 continue
1269 break
1270 raise KeyError
1272 def _update_pack_cache(self) -> list[Pack]:
1273 raise NotImplementedError(self._update_pack_cache)
1275 def close(self) -> None:
1276 """Close the object store and release resources.
1278 This method closes all cached pack files and frees associated resources.
1279 Can be called multiple times safely.
1280 """
1281 self._clear_cached_packs()
1283 def __del__(self) -> None:
1284 """Warn if the object store is being deleted with unclosed packs."""
1285 if self._pack_cache:
1286 import warnings
1288 warnings.warn(
1289 f"ObjectStore {self!r} was destroyed with {len(self._pack_cache)} "
1290 "unclosed pack(s). Please call close() explicitly.",
1291 ResourceWarning,
1292 stacklevel=2,
1293 )
1294 self.close()
1296 @property
1297 def packs(self) -> list[Pack]:
1298 """List with pack objects."""
1299 return list(self._iter_cached_packs()) + list(self._update_pack_cache())
1301 def count_pack_files(self) -> int:
1302 """Count the number of pack files.
1304 Returns:
1305 Number of pack files (excluding those with .keep files)
1306 """
1307 count = 0
1308 for pack in self.packs:
1309 # Check if there's a .keep file for this pack
1310 keep_path = pack._basename + ".keep"
1311 if not os.path.exists(keep_path):
1312 count += 1
1313 return count
1315 def _iter_alternate_objects(self) -> Iterator[ObjectID]:
1316 """Iterate over the SHAs of all the objects in alternate stores."""
1317 for alternate in self.alternates:
1318 yield from alternate
1320 def _iter_loose_objects(self) -> Iterator[ObjectID]:
1321 """Iterate over the SHAs of all loose objects."""
1322 raise NotImplementedError(self._iter_loose_objects)
1324 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
1325 raise NotImplementedError(self._get_loose_object)
1327 def delete_loose_object(self, sha: ObjectID) -> None:
1328 """Delete a loose object.
1330 This method only handles loose objects. For packed objects,
1331 use repack(exclude=...) to exclude them during repacking.
1332 """
1333 raise NotImplementedError(self.delete_loose_object)
1335 def _remove_pack(self, pack: "Pack") -> None:
1336 raise NotImplementedError(self._remove_pack)
1338 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
1339 """Pack loose objects.
1341 Args:
1342 progress: Optional progress reporting callback
1344 Returns: Number of objects packed
1345 """
1346 objects: list[tuple[ShaFile, None]] = []
1347 for sha in self._iter_loose_objects():
1348 obj = self._get_loose_object(sha)
1349 if obj is not None:
1350 objects.append((obj, None))
1351 self.add_objects(objects, progress=progress)
1352 for obj, path in objects:
1353 self.delete_loose_object(obj.id)
1354 return len(objects)
1356 def repack(
1357 self,
1358 exclude: Set[bytes] | None = None,
1359 progress: Callable[[str], None] | None = None,
1360 ) -> int:
1361 """Repack the packs in this repository.
1363 Note that this implementation is fairly naive and currently keeps all
1364 objects in memory while it repacks.
1366 Args:
1367 exclude: Optional set of object SHAs to exclude from repacking
1368 progress: Optional progress reporting callback
1369 """
1370 if exclude is None:
1371 exclude = set()
1373 loose_objects = set()
1374 excluded_loose_objects = set()
1375 for sha in self._iter_loose_objects():
1376 if sha not in exclude:
1377 obj = self._get_loose_object(sha)
1378 if obj is not None:
1379 loose_objects.add(obj)
1380 else:
1381 excluded_loose_objects.add(sha)
1383 objects: set[tuple[ShaFile, None]] = {(obj, None) for obj in loose_objects}
1384 old_packs = {p.name(): p for p in self.packs}
1385 for name, pack in old_packs.items():
1386 objects.update(
1387 (obj, None) for obj in pack.iterobjects() if obj.id not in exclude
1388 )
1390 # Only create a new pack if there are objects to pack
1391 if objects:
1392 # The name of the consolidated pack might match the name of a
1393 # pre-existing pack. Take care not to remove the newly created
1394 # consolidated pack.
1395 consolidated = self.add_objects(list(objects), progress=progress)
1396 if consolidated is not None:
1397 old_packs.pop(consolidated.name(), None)
1399 # Delete loose objects that were packed
1400 for obj in loose_objects:
1401 if obj is not None:
1402 self.delete_loose_object(obj.id)
1403 # Delete excluded loose objects
1404 for sha in excluded_loose_objects:
1405 self.delete_loose_object(sha)
1406 for name, pack in old_packs.items():
1407 self._remove_pack(pack)
1408 self._update_pack_cache()
1409 return len(objects)
1411 def generate_pack_bitmaps(
1412 self,
1413 refs: dict[Ref, ObjectID],
1414 *,
1415 commit_interval: int | None = None,
1416 progress: Callable[[str], None] | None = None,
1417 ) -> int:
1418 """Generate bitmap indexes for all packs that don't have them.
1420 This generates .bitmap files for packfiles, enabling fast reachability
1421 queries. Equivalent to the bitmap generation part of 'git repack -b'.
1423 Args:
1424 refs: Dictionary of ref names to commit SHAs
1425 commit_interval: Include every Nth commit in bitmap index (None for default)
1426 progress: Optional progress reporting callback
1428 Returns:
1429 Number of bitmaps generated
1430 """
1431 count = 0
1432 for pack in self.packs:
1433 pack.ensure_bitmap(
1434 self, refs, commit_interval=commit_interval, progress=progress
1435 )
1436 count += 1
1438 # Update cache to pick up new bitmaps
1439 self._update_pack_cache()
1441 return count
1443 def __iter__(self) -> Iterator[ObjectID]:
1444 """Iterate over the SHAs that are present in this store."""
1445 self._update_pack_cache()
1446 for pack in self._iter_cached_packs():
1447 try:
1448 yield from pack
1449 except PackFileDisappeared as exc:
1450 self._evict_pack(exc.obj)
1451 yield from self._iter_loose_objects()
1452 yield from self._iter_alternate_objects()
1454 def contains_loose(self, sha: ObjectID) -> bool:
1455 """Check if a particular object is present by SHA1 and is loose.
1457 This does not check alternates.
1458 """
1459 return self._get_loose_object(sha) is not None
1461 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
1462 """Obtain the raw fulltext for an object.
1464 Args:
1465 name: sha for the object.
1466 Returns: tuple with numeric type and object contents.
1467 """
1468 sha: RawObjectID
1469 hexsha: ObjectID | None
1470 if len(name) == self.object_format.hex_length:
1471 sha = hex_to_sha(ObjectID(name))
1472 hexsha = cast(ObjectID, name)
1473 elif len(name) == self.object_format.oid_length:
1474 sha = RawObjectID(name)
1475 hexsha = None
1476 else:
1477 raise AssertionError(f"Invalid object name {name!r}")
1478 try:
1479 return self._lookup_in_packs(lambda p: p.get_raw(sha))
1480 except KeyError:
1481 pass
1482 if hexsha is None:
1483 hexsha = sha_to_hex(sha)
1484 ret = self._get_loose_object(hexsha)
1485 if ret is not None:
1486 return ret.type_num, ret.as_raw_string()
1487 for alternate in self.alternates:
1488 try:
1489 return alternate.get_raw(hexsha)
1490 except KeyError:
1491 pass
1492 raise KeyError(hexsha)
1494 def iter_unpacked_subset(
1495 self,
1496 shas: Iterable[ObjectID | RawObjectID],
1497 *,
1498 include_comp: bool = False,
1499 allow_missing: bool = False,
1500 convert_ofs_delta: bool = True,
1501 ) -> Iterator[UnpackedObject]:
1502 """Iterate over a subset of objects, yielding UnpackedObject instances.
1504 Args:
1505 shas: Set of object SHAs to retrieve
1506 include_comp: Whether to include compressed data
1507 allow_missing: If True, skip missing objects; if False, raise KeyError
1508 convert_ofs_delta: Whether to convert OFS_DELTA objects
1510 Returns:
1511 Iterator of UnpackedObject instances
1513 Raises:
1514 KeyError: If an object is missing and allow_missing is False
1515 """
1516 todo: set[ObjectID | RawObjectID] = set(shas)
1517 for p in self._iter_cached_packs():
1518 try:
1519 for unpacked in p.iter_unpacked_subset(
1520 todo,
1521 include_comp=include_comp,
1522 allow_missing=True,
1523 convert_ofs_delta=convert_ofs_delta,
1524 ):
1525 yield unpacked
1526 hexsha = sha_to_hex(unpacked.sha())
1527 todo.remove(hexsha)
1528 except PackFileDisappeared as exc:
1529 self._evict_pack(exc.obj)
1530 # Maybe something else has added a pack with the object
1531 # in the mean time?
1532 for p in self._update_pack_cache():
1533 try:
1534 for unpacked in p.iter_unpacked_subset(
1535 todo,
1536 include_comp=include_comp,
1537 allow_missing=True,
1538 convert_ofs_delta=convert_ofs_delta,
1539 ):
1540 yield unpacked
1541 hexsha = sha_to_hex(unpacked.sha())
1542 todo.remove(hexsha)
1543 except PackFileDisappeared as exc:
1544 self._evict_pack(exc.obj)
1545 for alternate in self.alternates:
1546 assert isinstance(alternate, PackBasedObjectStore)
1547 for unpacked in alternate.iter_unpacked_subset(
1548 todo,
1549 include_comp=include_comp,
1550 allow_missing=True,
1551 convert_ofs_delta=convert_ofs_delta,
1552 ):
1553 yield unpacked
1554 hexsha = sha_to_hex(unpacked.sha())
1555 todo.remove(hexsha)
1557 def iterobjects_subset(
1558 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
1559 ) -> Iterator[ShaFile]:
1560 """Iterate over a subset of objects in the store.
1562 This method searches for objects in pack files, alternates, and loose storage.
1564 Args:
1565 shas: Iterable of object SHAs to retrieve
1566 allow_missing: If True, skip missing objects; if False, raise KeyError
1568 Returns:
1569 Iterator of ShaFile objects
1571 Raises:
1572 KeyError: If an object is missing and allow_missing is False
1573 """
1574 todo: set[ObjectID] = set(shas)
1575 for p in self._iter_cached_packs():
1576 try:
1577 for o in p.iterobjects_subset(todo, allow_missing=True):
1578 yield o
1579 todo.remove(o.id)
1580 except PackFileDisappeared as exc:
1581 self._evict_pack(exc.obj)
1582 # Maybe something else has added a pack with the object
1583 # in the mean time?
1584 for p in self._update_pack_cache():
1585 try:
1586 for o in p.iterobjects_subset(todo, allow_missing=True):
1587 yield o
1588 todo.remove(o.id)
1589 except PackFileDisappeared as exc:
1590 self._evict_pack(exc.obj)
1591 for alternate in self.alternates:
1592 for o in alternate.iterobjects_subset(todo, allow_missing=True):
1593 yield o
1594 todo.remove(o.id)
1595 for oid in todo:
1596 loose_obj: ShaFile | None = self._get_loose_object(oid)
1597 if loose_obj is not None:
1598 yield loose_obj
1599 elif not allow_missing:
1600 raise KeyError(oid)
1602 def get_unpacked_object(
1603 self, sha1: bytes, *, include_comp: bool = False
1604 ) -> UnpackedObject:
1605 """Obtain the unpacked object.
1607 Args:
1608 sha1: sha for the object.
1609 include_comp: Whether to include compression metadata.
1610 """
1611 if len(sha1) == self.object_format.hex_length:
1612 sha = hex_to_sha(cast(ObjectID, sha1))
1613 hexsha = cast(ObjectID, sha1)
1614 elif len(sha1) == self.object_format.oid_length:
1615 sha = cast(RawObjectID, sha1)
1616 hexsha = None
1617 else:
1618 raise AssertionError(f"Invalid object sha1 {sha1!r}")
1619 try:
1620 return self._lookup_in_packs(
1621 lambda p: p.get_unpacked_object(sha, include_comp=include_comp)
1622 )
1623 except KeyError:
1624 pass
1625 if hexsha is None:
1626 hexsha = sha_to_hex(sha)
1627 for alternate in self.alternates:
1628 assert isinstance(alternate, PackBasedObjectStore)
1629 try:
1630 return alternate.get_unpacked_object(hexsha, include_comp=include_comp)
1631 except KeyError:
1632 pass
1633 raise KeyError(hexsha)
1635 def add_objects(
1636 self,
1637 objects: Sequence[tuple[ShaFile, str | None]],
1638 progress: Callable[[str], None] | None = None,
1639 ) -> "Pack | None":
1640 """Add a set of objects to this object store.
1642 Args:
1643 objects: Iterable over (object, path) tuples, should support
1644 __len__.
1645 progress: Optional progress reporting function.
1646 Returns: Pack object of the objects written.
1647 """
1648 count = len(objects)
1649 record_iter = (full_unpacked_object(o) for (o, p) in objects)
1650 return self.add_pack_data(count, record_iter, progress=progress)
1653class DiskObjectStore(PackBasedObjectStore):
1654 """Git-style object store that exists on disk."""
1656 path: str | os.PathLike[str]
1657 pack_dir: str | os.PathLike[str]
1658 _alternates: "list[BaseObjectStore] | None"
1659 _commit_graph: "CommitGraph | None"
1661 def __init__(
1662 self,
1663 path: str | os.PathLike[str],
1664 *,
1665 loose_compression_level: int = -1,
1666 pack_compression_level: int = -1,
1667 pack_index_version: int | None = None,
1668 pack_delta_window_size: int | None = None,
1669 pack_window_memory: int | None = None,
1670 pack_delta_cache_size: int | None = None,
1671 pack_depth: int | None = None,
1672 pack_threads: int | None = None,
1673 pack_big_file_threshold: int | None = None,
1674 packed_git_limit: int | None = None,
1675 delta_base_cache_limit: int | None = None,
1676 fsync_object_files: bool = False,
1677 pack_write_bitmaps: bool = False,
1678 pack_write_bitmap_hash_cache: bool = True,
1679 pack_write_bitmap_lookup_table: bool = True,
1680 shared_perm: "SharedPerm | None" = None,
1681 object_format: "ObjectFormat | None" = None,
1682 loose_object_size_limit: int | None = None,
1683 alternates: "Iterable[str | os.PathLike[str]] | None" = None,
1684 ) -> None:
1685 """Open an object store.
1687 Args:
1688 path: Path of the object store.
1689 loose_compression_level: zlib compression level for loose objects
1690 pack_compression_level: zlib compression level for pack objects
1691 pack_index_version: pack index version to use (1, 2, or 3)
1692 pack_delta_window_size: sliding window size for delta compression
1693 pack_window_memory: memory limit for delta window operations
1694 pack_delta_cache_size: size of cache for delta operations
1695 pack_depth: maximum delta chain depth
1696 pack_threads: number of threads for pack operations
1697 pack_big_file_threshold: threshold for treating files as big
1698 packed_git_limit: maximum total bytes for mmapped pack files
1699 delta_base_cache_limit: maximum bytes for delta base object cache
1700 fsync_object_files: whether to fsync object files for durability
1701 pack_write_bitmaps: whether to write bitmap indexes for packs
1702 pack_write_bitmap_hash_cache: whether to include name-hash cache in bitmaps
1703 pack_write_bitmap_lookup_table: whether to include lookup table in bitmaps
1704 shared_perm: Shared repository permission setting
1705 object_format: Hash algorithm to use (SHA1 or SHA256)
1706 loose_object_size_limit: Maximum inflated size of a single loose
1707 object. Defaults to core.bigFileThreshold's Git default (512 MiB)
1708 via :data:`DEFAULT_LOOSE_OBJECT_SIZE_LIMIT` when None. Guards against
1709 decompression-bomb attacks.
1710 alternates: Extra alternate object directory paths to consult, in
1711 addition to those listed in ``objects/info/alternates``. Used to
1712 plumb ``GIT_ALTERNATE_OBJECT_DIRECTORIES`` through from porcelain.
1713 """
1714 # Import here to avoid circular dependency
1715 from .object_format import DEFAULT_OBJECT_FORMAT
1717 super().__init__(
1718 pack_compression_level=pack_compression_level,
1719 pack_index_version=pack_index_version,
1720 pack_delta_window_size=pack_delta_window_size,
1721 pack_window_memory=pack_window_memory,
1722 pack_delta_cache_size=pack_delta_cache_size,
1723 pack_depth=pack_depth,
1724 pack_threads=pack_threads,
1725 pack_big_file_threshold=pack_big_file_threshold,
1726 packed_git_limit=packed_git_limit,
1727 delta_base_cache_limit=delta_base_cache_limit,
1728 object_format=object_format if object_format else DEFAULT_OBJECT_FORMAT,
1729 )
1730 self.path = path
1731 self.pack_dir = os.path.join(self.path, PACKDIR)
1732 self._alternates = None
1733 self._extra_alternate_paths: list[str] = (
1734 [os.fsdecode(os.fspath(p)) for p in alternates]
1735 if alternates is not None
1736 else []
1737 )
1738 self.loose_compression_level = loose_compression_level
1739 self.pack_compression_level = pack_compression_level
1740 self.pack_index_version = pack_index_version
1741 self.fsync_object_files = fsync_object_files
1742 self.pack_write_bitmaps = pack_write_bitmaps
1743 self.pack_write_bitmap_hash_cache = pack_write_bitmap_hash_cache
1744 self.pack_write_bitmap_lookup_table = pack_write_bitmap_lookup_table
1745 self.shared_perm = shared_perm
1746 self.loose_object_size_limit = (
1747 loose_object_size_limit
1748 if loose_object_size_limit is not None
1749 else DEFAULT_LOOSE_OBJECT_SIZE_LIMIT
1750 )
1752 # Commit graph support - lazy loaded
1753 self._commit_graph = None
1754 self._use_commit_graph = True # Default to true
1756 # Multi-pack-index support - lazy loaded
1757 self._midx: MultiPackIndex | None = None
1758 self._use_midx = True # Default to true
1760 def __repr__(self) -> str:
1761 """Return string representation of DiskObjectStore.
1763 Returns:
1764 String representation including the store path
1765 """
1766 return f"<{self.__class__.__name__}({self.path!r})>"
1768 @classmethod
1769 def from_config(
1770 cls,
1771 path: str | os.PathLike[str],
1772 config: "Config",
1773 *,
1774 shared_perm: "SharedPerm | None" = None,
1775 alternates: "Iterable[str | os.PathLike[str]] | None" = None,
1776 ) -> "DiskObjectStore":
1777 """Create a DiskObjectStore from a configuration object.
1779 Args:
1780 path: Path to the object store directory
1781 config: Configuration object to read settings from
1782 shared_perm: Optional shared repository permission setting
1783 alternates: Extra alternate object directory paths to consult,
1784 appended to those listed in ``objects/info/alternates``.
1786 Returns:
1787 New DiskObjectStore instance configured according to config
1788 """
1789 try:
1790 default_compression_level = int(
1791 config.get((b"core",), b"compression").decode()
1792 )
1793 except KeyError:
1794 default_compression_level = -1
1795 try:
1796 loose_compression_level = int(
1797 config.get((b"core",), b"looseCompression").decode()
1798 )
1799 except KeyError:
1800 loose_compression_level = default_compression_level
1801 try:
1802 pack_compression_level = int(
1803 config.get((b"core",), "packCompression").decode()
1804 )
1805 except KeyError:
1806 pack_compression_level = default_compression_level
1807 try:
1808 pack_index_version = int(config.get((b"pack",), b"indexVersion").decode())
1809 except KeyError:
1810 pack_index_version = None
1812 # Read pack configuration options
1813 try:
1814 pack_delta_window_size = int(
1815 config.get((b"pack",), b"deltaWindowSize").decode()
1816 )
1817 except KeyError:
1818 pack_delta_window_size = None
1819 try:
1820 pack_window_memory = int(config.get((b"pack",), b"windowMemory").decode())
1821 except KeyError:
1822 pack_window_memory = None
1823 try:
1824 pack_delta_cache_size = int(
1825 config.get((b"pack",), b"deltaCacheSize").decode()
1826 )
1827 except KeyError:
1828 pack_delta_cache_size = None
1829 try:
1830 pack_depth = int(config.get((b"pack",), b"depth").decode())
1831 except KeyError:
1832 pack_depth = None
1833 try:
1834 pack_threads = int(config.get((b"pack",), b"threads").decode())
1835 except KeyError:
1836 pack_threads = None
1837 try:
1838 pack_big_file_threshold = int(
1839 config.get((b"pack",), b"bigFileThreshold").decode()
1840 )
1841 except KeyError:
1842 pack_big_file_threshold = None
1844 # Read core.packedGitLimit setting
1845 try:
1846 packed_git_limit = int(config.get((b"core",), b"packedGitLimit").decode())
1847 except KeyError:
1848 packed_git_limit = None
1850 # Read core.deltaBaseCacheLimit setting
1851 try:
1852 delta_base_cache_limit = int(
1853 config.get((b"core",), b"deltaBaseCacheLimit").decode()
1854 )
1855 except KeyError:
1856 delta_base_cache_limit = None
1858 # Read core.bigFileThreshold setting; used as the upper bound for
1859 # inflating a single loose object, guarding against decompression bombs.
1860 try:
1861 loose_object_size_limit: int | None = int(
1862 config.get((b"core",), b"bigFileThreshold").decode()
1863 )
1864 except KeyError:
1865 loose_object_size_limit = None
1867 # Read core.commitGraph setting
1868 use_commit_graph = config.get_boolean((b"core",), b"commitGraph", True)
1870 # Read core.multiPackIndex setting
1871 use_midx = config.get_boolean((b"core",), b"multiPackIndex", True)
1873 # Read core.fsyncObjectFiles setting
1874 fsync_object_files = config.get_boolean((b"core",), b"fsyncObjectFiles", False)
1876 # Read bitmap settings
1877 pack_write_bitmaps = config.get_boolean((b"pack",), b"writeBitmaps", False)
1878 pack_write_bitmap_hash_cache = config.get_boolean(
1879 (b"pack",), b"writeBitmapHashCache", True
1880 )
1881 pack_write_bitmap_lookup_table = config.get_boolean(
1882 (b"pack",), b"writeBitmapLookupTable", True
1883 )
1884 # Also check repack.writeBitmaps for backwards compatibility
1885 if not pack_write_bitmaps:
1886 pack_write_bitmaps = config.get_boolean(
1887 (b"repack",), b"writeBitmaps", False
1888 )
1890 # Get hash algorithm from config
1891 from .object_format import get_object_format
1893 object_format = None
1894 try:
1895 try:
1896 version = int(config.get((b"core",), b"repositoryformatversion"))
1897 except KeyError:
1898 version = 0
1899 if version == 1:
1900 try:
1901 object_format_name = config.get((b"extensions",), b"objectformat")
1902 except KeyError:
1903 object_format_name = b"sha1"
1904 object_format = get_object_format(object_format_name.decode("ascii"))
1905 except (KeyError, ValueError):
1906 pass
1908 instance = cls(
1909 path,
1910 loose_compression_level=loose_compression_level,
1911 pack_compression_level=pack_compression_level,
1912 pack_index_version=pack_index_version,
1913 pack_delta_window_size=pack_delta_window_size,
1914 pack_window_memory=pack_window_memory,
1915 pack_delta_cache_size=pack_delta_cache_size,
1916 pack_depth=pack_depth,
1917 pack_threads=pack_threads,
1918 pack_big_file_threshold=pack_big_file_threshold,
1919 packed_git_limit=packed_git_limit,
1920 delta_base_cache_limit=delta_base_cache_limit,
1921 fsync_object_files=fsync_object_files,
1922 pack_write_bitmaps=pack_write_bitmaps,
1923 pack_write_bitmap_hash_cache=pack_write_bitmap_hash_cache,
1924 pack_write_bitmap_lookup_table=pack_write_bitmap_lookup_table,
1925 shared_perm=shared_perm,
1926 object_format=object_format,
1927 loose_object_size_limit=loose_object_size_limit,
1928 alternates=alternates,
1929 )
1930 instance._use_commit_graph = use_commit_graph
1931 instance._use_midx = use_midx
1932 return instance
1934 @property
1935 def alternates(self) -> list["BaseObjectStore"]:
1936 """Get the list of alternate object stores.
1938 Reads from .git/objects/info/alternates if not already cached.
1940 Returns:
1941 List of DiskObjectStore instances for alternate object directories
1942 """
1943 if self._alternates is not None:
1944 return self._alternates
1945 self._alternates = []
1946 for path in self._read_alternate_paths():
1947 self._alternates.append(DiskObjectStore(path))
1948 for path in self._extra_alternate_paths:
1949 self._alternates.append(DiskObjectStore(path))
1950 return self._alternates
1952 def _read_alternate_paths(self) -> Iterator[str]:
1953 try:
1954 f = GitFile(os.path.join(self.path, INFODIR, "alternates"), "rb")
1955 except FileNotFoundError:
1956 return
1957 with f:
1958 for line in f.readlines():
1959 line = line.rstrip(b"\n")
1960 if line.startswith(b"#"):
1961 continue
1962 if os.path.isabs(line):
1963 yield os.fsdecode(line)
1964 else:
1965 yield os.fsdecode(os.path.join(os.fsencode(self.path), line))
1967 def add_alternate_path(self, path: str | os.PathLike[str]) -> None:
1968 """Add an alternate path to this object store."""
1969 info_dir = os.path.join(self.path, INFODIR)
1970 try:
1971 os.mkdir(info_dir)
1972 adjust_shared_perm(info_dir, self.shared_perm)
1973 except FileExistsError:
1974 pass
1975 alternates_path = os.path.join(self.path, INFODIR, "alternates")
1976 with GitFile(alternates_path, "wb", shared_perm=self.shared_perm) as f:
1977 try:
1978 orig_f = open(alternates_path, "rb")
1979 except FileNotFoundError:
1980 pass
1981 else:
1982 with orig_f:
1983 f.write(orig_f.read())
1984 f.write(os.fsencode(path) + b"\n")
1986 if not os.path.isabs(path):
1987 path = os.path.join(self.path, path)
1988 self.alternates.append(DiskObjectStore(path))
1990 def _update_pack_cache(self) -> list[Pack]:
1991 """Read and iterate over new pack files and cache them."""
1992 try:
1993 pack_dir_contents = set(os.listdir(self.pack_dir))
1994 except FileNotFoundError:
1995 return []
1996 pack_files = set()
1997 for name in pack_dir_contents:
1998 # Index any ".pack" file with a matching ".idx", not just
1999 # "pack-<hash>". ``git maintenance`` writes packs named
2000 # "loose-<hash>.pack"; these are ordinary packs and Git indexes
2001 # any .pack file present. The matching ".idx" also confirms the
2002 # pack is fully written.
2003 if name.endswith(".pack"):
2004 basename = name[: -len(".pack")]
2005 if basename + ".idx" in pack_dir_contents:
2006 pack_files.add(basename)
2008 # Open newly appeared pack files
2009 new_packs = []
2010 for basename in pack_files:
2011 if basename not in self._pack_cache:
2012 pack = Pack(
2013 os.path.join(self.pack_dir, basename),
2014 object_format=self.object_format,
2015 delta_window_size=self.pack_delta_window_size,
2016 window_memory=self.pack_window_memory,
2017 delta_cache_size=self.pack_delta_cache_size,
2018 depth=self.pack_depth,
2019 threads=self.pack_threads,
2020 big_file_threshold=self.pack_big_file_threshold,
2021 delta_base_cache_limit=self.delta_base_cache_limit,
2022 )
2023 new_packs.append(pack)
2024 self._pack_cache[basename] = pack
2025 self._mark_pack_used(basename)
2026 # Remove disappeared pack files
2027 for f in set(self._pack_cache) - pack_files:
2028 self._pack_cache.pop(f).close()
2029 try:
2030 self._pack_access_order.remove(f)
2031 except ValueError:
2032 pass
2033 self._enforce_packed_git_limit()
2034 return new_packs
2036 def _get_shafile_path(self, sha: ObjectID) -> str:
2037 # Reject anything that is neither a valid hex object id nor a raw
2038 # binary oid before building a path. Otherwise a malformed id (e.g.
2039 # one containing path separators) would be joined into a filename
2040 # that escapes the objects directory.
2041 if not valid_hexsha(sha):
2042 raise ValueError(f"Invalid object id {sha!r}")
2043 # Check from object dir
2044 return hex_to_filename(os.fspath(self.path), sha)
2046 def _iter_loose_objects(self) -> Iterator[ObjectID]:
2047 for base in os.listdir(self.path):
2048 if len(base) != 2:
2049 continue
2050 for rest in os.listdir(os.path.join(self.path, base)):
2051 sha = os.fsencode(base + rest)
2052 if not valid_hexsha(sha):
2053 continue
2054 yield ObjectID(sha)
2056 def count_loose_objects(self) -> int:
2057 """Count the number of loose objects in the object store.
2059 Returns:
2060 Number of loose objects
2061 """
2062 # Calculate expected filename length for loose
2063 # objects (excluding directory)
2064 fn_length = self.object_format.hex_length - 2
2065 count = 0
2066 if not os.path.exists(self.path):
2067 return 0
2069 for i in range(256):
2070 subdir = os.path.join(self.path, f"{i:02x}")
2071 try:
2072 count += len(
2073 [name for name in os.listdir(subdir) if len(name) == fn_length]
2074 )
2075 except FileNotFoundError:
2076 # Directory may have been removed or is inaccessible
2077 continue
2079 return count
2081 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
2082 try:
2083 # Load the object from path with SHA and hash algorithm from object store
2084 # Convert to hex ObjectID if needed
2085 if len(sha) == self.object_format.oid_length:
2086 hex_sha: ObjectID = sha_to_hex(RawObjectID(sha))
2087 else:
2088 hex_sha = ObjectID(sha)
2089 path = self._get_shafile_path(hex_sha)
2090 return ShaFile.from_path(
2091 path,
2092 hex_sha,
2093 object_format=self.object_format,
2094 max_size=self.loose_object_size_limit,
2095 )
2096 except FileNotFoundError:
2097 return None
2099 def delete_loose_object(self, sha: ObjectID) -> None:
2100 """Delete a loose object from disk.
2102 Args:
2103 sha: SHA1 of the object to delete
2105 Raises:
2106 FileNotFoundError: If the object file doesn't exist
2107 """
2108 _remove_readonly(self._get_shafile_path(sha))
2109 self._invalidate_peel_cache()
2111 def get_object_mtime(self, sha: ObjectID) -> float:
2112 """Get the modification time of an object.
2114 Args:
2115 sha: SHA1 of the object
2117 Returns:
2118 Modification time as seconds since epoch
2120 Raises:
2121 KeyError: if the object is not found
2122 """
2123 # First check if it's a loose object
2124 if self.contains_loose(sha):
2125 path = self._get_shafile_path(sha)
2126 try:
2127 return os.path.getmtime(path)
2128 except FileNotFoundError:
2129 pass
2131 # Check if it's in a pack file
2132 for pack in self.packs:
2133 try:
2134 if sha in pack:
2135 # Use the pack file's mtime for packed objects
2136 pack_path = pack._data_path
2137 try:
2138 return os.path.getmtime(pack_path)
2139 except (FileNotFoundError, AttributeError):
2140 pass
2141 except PackFileDisappeared:
2142 pass
2144 raise KeyError(sha)
2146 def _remove_pack(self, pack: Pack) -> None:
2147 self._invalidate_peel_cache()
2148 # _pack_cache is keyed by the full pack basename (e.g. "pack-<hash>"
2149 # or "loose-<hash>"), matching pack._basename.
2150 basename = os.path.basename(pack._basename)
2151 self._pack_cache.pop(basename, None)
2152 try:
2153 self._pack_access_order.remove(basename)
2154 except ValueError:
2155 pass
2156 # Store paths before closing to avoid re-opening files on Windows
2157 data_path = pack._data_path
2158 idx_path = pack._idx_path
2159 pack.close()
2160 _remove_readonly(data_path)
2161 if os.path.exists(idx_path):
2162 _remove_readonly(idx_path)
2164 def _get_pack_basepath(
2165 self, entries: Iterable[tuple[bytes, int, int | None]]
2166 ) -> str:
2167 suffix_bytes = iter_sha1(entry[0] for entry in entries)
2168 # TODO: Handle self.pack_dir being bytes
2169 suffix = suffix_bytes.decode("ascii")
2170 return os.path.join(self.pack_dir, "pack-" + suffix)
2172 def _index_pack(
2173 self,
2174 indexer: PackIndexer,
2175 num_objects: int,
2176 progress: Callable[..., None] | None = None,
2177 ) -> tuple[list[PackIndexEntry], set[RawObjectID]]:
2178 """Drain an indexer into index entries and the external refs it needs.
2180 Args:
2181 indexer: A PackIndexer over the pack being completed.
2182 num_objects: Number of objects in the pack, for progress reporting.
2183 progress: Optional progress reporting function.
2185 Returns: Tuple of (index entries, external refs). ext_refs() is only
2186 populated once the indexer has been drained.
2187 """
2188 entries = []
2189 for i, entry in enumerate(indexer):
2190 if progress is not None:
2191 progress(f"generating index: {i}/{num_objects}\r".encode("ascii"))
2192 entries.append(entry)
2193 return entries, set(indexer.ext_refs())
2195 def _complete_pack(
2196 self,
2197 f: BinaryIO,
2198 path: str,
2199 entries: list[PackIndexEntry],
2200 ext_refs: set[RawObjectID],
2201 progress: Callable[..., None] | None = None,
2202 refs: dict[Ref, ObjectID] | None = None,
2203 ) -> Pack:
2204 """Move a specific file containing a pack into the pack directory.
2206 Note: The file should be on the same file system as the
2207 packs directory.
2209 This takes ownership of ``f``: it appends any missing base objects,
2210 closes the file and renames it into place. Callers must have finished
2211 reading the pack (see :meth:`_index_pack`) before calling this; on
2212 Windows a mapping left over the file blocks both the write and the
2213 rename.
2215 Args:
2216 f: Open file object for the pack.
2217 path: Path to the pack file.
2218 entries: Index entries for the objects already in the pack.
2219 ext_refs: Objects the pack deltas against that it does not contain.
2220 progress: Optional progress reporting function.
2221 refs: Optional dictionary of refs for bitmap generation.
2222 """
2223 pack_sha, extra_entries = extend_pack(
2224 f,
2225 ext_refs,
2226 get_raw=self.get_raw,
2227 compression_level=self.pack_compression_level,
2228 progress=progress,
2229 object_format=self.object_format,
2230 )
2231 f.flush()
2232 if self.fsync_object_files:
2233 try:
2234 fileno = f.fileno()
2235 except AttributeError as e:
2236 raise OSError("fsync requested but file has no fileno()") from e
2237 else:
2238 os.fsync(fileno)
2239 f.close()
2241 entries.extend(extra_entries)
2243 # Move the pack in.
2244 entries.sort()
2245 pack_base_name = self._get_pack_basepath(entries)
2247 # A pack's identity is the SHA over its object SHAs, which is the
2248 # "<hash>" suffix _get_pack_basepath builds the name from. Compare by
2249 # that rather than by basename so an existing pack holding the same
2250 # objects under a different prefix (e.g. a "loose-<hash>" pack written
2251 # by git maintenance) is recognised and not duplicated.
2252 pack_name = os.path.basename(pack_base_name)[len("pack-") :].encode("ascii")
2253 for pack in self.packs:
2254 if pack.name() == pack_name:
2255 # The objects are already packed; drop the temporary pack we
2256 # were about to move in rather than leaking it into pack_dir.
2257 _remove_readonly(path)
2258 return pack
2260 target_pack_path = pack_base_name + ".pack"
2261 target_index_path = pack_base_name + ".idx"
2262 if sys.platform == "win32":
2263 # Windows might have the target pack file lingering. Attempt
2264 # removal, silently passing if the target does not exist.
2265 with suppress(FileNotFoundError):
2266 os.remove(target_pack_path)
2267 os.rename(path, target_pack_path)
2269 # Write the index.
2270 with GitFile(
2271 target_index_path,
2272 "wb",
2273 mask=PACK_MODE,
2274 fsync=self.fsync_object_files,
2275 shared_perm=self.shared_perm,
2276 ) as index_file:
2277 write_pack_index(
2278 index_file, entries, pack_sha, version=self.pack_index_version
2279 )
2281 # Generate bitmap if configured and refs are available
2282 if self.pack_write_bitmaps and refs:
2283 from .bitmap import generate_bitmap, write_bitmap
2284 from .pack import load_pack_index
2286 if progress:
2287 progress("Generating bitmap index\r".encode("ascii"))
2289 # Load the index we just wrote. load_pack_index keeps the file
2290 # open for the lifetime of the index (it mmaps it), so close it
2291 # once the bitmap is generated rather than leaving the .idx
2292 # mapped, which would lock it on Windows.
2293 with closing(
2294 load_pack_index(target_index_path, self.object_format)
2295 ) as pack_index:
2296 bitmap = generate_bitmap(
2297 pack_index=pack_index,
2298 object_store=self,
2299 refs=refs,
2300 pack_checksum=pack_sha,
2301 include_hash_cache=self.pack_write_bitmap_hash_cache,
2302 include_lookup_table=self.pack_write_bitmap_lookup_table,
2303 progress=lambda msg: (
2304 progress(msg.encode("ascii"))
2305 if progress and isinstance(msg, str)
2306 else None
2307 ),
2308 )
2310 # Write the bitmap
2311 target_bitmap_path = pack_base_name + ".bitmap"
2312 write_bitmap(target_bitmap_path, bitmap)
2314 if progress:
2315 progress("Bitmap index written\r".encode("ascii"))
2317 # Add the pack to the store and return it.
2318 final_pack = Pack(
2319 pack_base_name,
2320 object_format=self.object_format,
2321 delta_window_size=self.pack_delta_window_size,
2322 window_memory=self.pack_window_memory,
2323 delta_cache_size=self.pack_delta_cache_size,
2324 depth=self.pack_depth,
2325 threads=self.pack_threads,
2326 big_file_threshold=self.pack_big_file_threshold,
2327 delta_base_cache_limit=self.delta_base_cache_limit,
2328 )
2329 try:
2330 final_pack.check_length_and_checksum()
2331 # Materialise every object so payloads that fail to parse
2332 # (e.g. tree entries with garbage modes) are rejected rather
2333 # than silently landed on disk. MemoryObjectStore already
2334 # validates ingested objects this way via PackInflater; without
2335 # the same check DiskObjectStore was strictly weaker.
2336 for _obj in PackInflater.for_pack_data(
2337 final_pack.data, resolve_ext_ref=self.get_raw
2338 ):
2339 pass
2340 except BaseException:
2341 final_pack.close()
2342 with suppress(FileNotFoundError):
2343 os.remove(target_pack_path)
2344 with suppress(FileNotFoundError):
2345 os.remove(target_index_path)
2346 if self.pack_write_bitmaps and refs:
2347 with suppress(FileNotFoundError):
2348 os.remove(pack_base_name + ".bitmap")
2349 raise
2350 # _pack_cache is keyed by the full basename (/path/to/pack-HASH -> pack-HASH)
2351 self._add_cached_pack(os.path.basename(pack_base_name), final_pack)
2352 return final_pack
2354 def add_thin_pack(
2355 self,
2356 read_all: Callable[[int], bytes],
2357 read_some: Callable[[int], bytes] | None,
2358 progress: Callable[..., None] | None = None,
2359 *,
2360 max_input_size: int | None = None,
2361 ) -> "Pack":
2362 """Add a new thin pack to this object store.
2364 Thin packs are packs that contain deltas with parents that exist
2365 outside the pack. They should never be placed in the object store
2366 directly, and always indexed and completed as they are copied.
2368 Args:
2369 read_all: Read function that blocks until the number of
2370 requested bytes are read.
2371 read_some: Read function that returns at least one byte, but may
2372 not return the number of bytes requested.
2373 progress: Optional progress reporting function.
2374 max_input_size: Maximum number of bytes that may be read from
2375 the wire while ingesting this pack. Matches git's
2376 ``receive.maxInputSize`` / ``index-pack --max-input-size``
2377 semantics: ``None`` (the default) or ``0`` mean unlimited.
2378 Exceeding the cap raises ``PackInputTooLarge``.
2379 Returns: A Pack object pointing at the now-completed thin pack in the
2380 objects/pack directory.
2381 """
2382 import tempfile
2384 if max_input_size:
2385 read_all, read_some = _bound_read_callables(
2386 read_all, read_some, max_input_size
2387 )
2389 fd, path = tempfile.mkstemp(dir=self.path, prefix="tmp_pack_")
2390 with os.fdopen(fd, "w+b") as f:
2391 os.chmod(path, PACK_MODE)
2392 indexer = PackIndexer(
2393 f,
2394 self.object_format.hash_func,
2395 resolve_ext_ref=self.get_raw,
2396 )
2397 copier = PackStreamCopier(
2398 self.object_format.hash_func,
2399 read_all,
2400 read_some,
2401 f,
2402 delta_iter=indexer, # type: ignore[arg-type]
2403 )
2404 copier.verify(progress=progress)
2405 f.flush()
2406 f.seek(0)
2407 # Index from a mapping as add_pack does. Unmapped before completion.
2408 with PackData(path, file=f, object_format=self.object_format) as pd:
2409 indexer.set_pack_data(pd)
2410 entries, ext_refs = self._index_pack(
2411 indexer, len(copier), progress=progress
2412 )
2413 return self._complete_pack(f, path, entries, ext_refs, progress=progress)
2415 def add_pack(
2416 self,
2417 ) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2418 """Add a new pack to this object store.
2420 Returns: Fileobject to write to, a commit function to
2421 call when the pack is finished and an abort
2422 function.
2423 """
2424 import tempfile
2426 fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
2427 f = os.fdopen(fd, "w+b")
2428 os.chmod(path, PACK_MODE)
2429 adjust_shared_perm(path, self.shared_perm)
2431 def commit() -> "Pack | None":
2432 if f.tell() > 0:
2433 f.seek(0)
2435 # Scope the mapping to indexing: _complete_pack writes to and
2436 # renames this same file, which a live mapping blocks on
2437 # Windows. PackData.close() leaves f open for it to finish.
2438 with PackData(path, file=f, object_format=self.object_format) as pd:
2439 indexer = PackIndexer.for_pack_data(
2440 pd,
2441 resolve_ext_ref=self.get_raw,
2442 )
2443 entries, ext_refs = self._index_pack(indexer, len(pd)) # type: ignore[arg-type]
2444 return self._complete_pack(f, path, entries, ext_refs)
2445 else:
2446 f.close()
2447 os.remove(path)
2448 return None
2450 def abort() -> None:
2451 f.close()
2452 os.remove(path)
2454 return f, commit, abort # type: ignore[return-value]
2456 def add_object(self, obj: ShaFile) -> None:
2457 """Add a single object to this object store.
2459 Args:
2460 obj: Object to add
2461 """
2462 # Use the correct hash algorithm for the object ID
2463 obj_id = ObjectID(obj.get_id(self.object_format))
2464 path = self._get_shafile_path(obj_id)
2465 dir = os.path.dirname(path)
2466 try:
2467 os.mkdir(dir)
2468 adjust_shared_perm(dir, self.shared_perm)
2469 except FileExistsError:
2470 pass
2471 try:
2472 # Refresh the mtime instead of just checking for existence. A
2473 # loose object with a stale mtime is a candidate for age-based
2474 # pruning, so a concurrent "git gc" could remove it before the
2475 # caller has created a reference to it.
2476 os.utime(path, None)
2477 except FileNotFoundError:
2478 pass # Not there after all, write it out below.
2479 except PermissionError:
2480 # Owned by another user in a shared repository. The mtime stays
2481 # stale, so write the object out to give it a fresh one.
2482 pass
2483 else:
2484 return # Already there and freshened, no need to write again
2485 with GitFile(
2486 path,
2487 "wb",
2488 mask=PACK_MODE,
2489 fsync=self.fsync_object_files,
2490 shared_perm=self.shared_perm,
2491 ) as f:
2492 f.write(
2493 obj.as_legacy_object(compression_level=self.loose_compression_level)
2494 )
2496 @classmethod
2497 def init(
2498 cls,
2499 path: str | os.PathLike[str],
2500 *,
2501 shared_perm: "SharedPerm | None" = None,
2502 object_format: "ObjectFormat | None" = None,
2503 ) -> "DiskObjectStore":
2504 """Initialize a new disk object store.
2506 Creates the necessary directory structure for a Git object store.
2508 Args:
2509 path: Path where the object store should be created
2510 shared_perm: Optional shared repository permission setting
2511 object_format: Hash algorithm to use (SHA1 or SHA256)
2513 Returns:
2514 New DiskObjectStore instance
2515 """
2516 try:
2517 os.mkdir(path)
2518 adjust_shared_perm(path, shared_perm)
2519 except FileExistsError:
2520 pass
2521 info_path = os.path.join(path, "info")
2522 pack_path = os.path.join(path, PACKDIR)
2523 os.mkdir(info_path)
2524 os.mkdir(pack_path)
2525 adjust_shared_perm(info_path, shared_perm)
2526 adjust_shared_perm(pack_path, shared_perm)
2527 return cls(path, shared_perm=shared_perm, object_format=object_format)
2529 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
2530 """Iterate over all object SHAs with the given prefix.
2532 Args:
2533 prefix: Hex prefix to search for (as bytes)
2535 Returns:
2536 Iterator of object SHAs (as ObjectID) matching the prefix
2537 """
2538 if len(prefix) < 2:
2539 yield from super().iter_prefix(prefix)
2540 return
2541 seen = set()
2542 dir = prefix[:2].decode()
2543 rest = prefix[2:].decode()
2544 try:
2545 for name in os.listdir(os.path.join(self.path, dir)):
2546 if name.startswith(rest):
2547 sha = ObjectID(os.fsencode(dir + name))
2548 if sha not in seen:
2549 seen.add(sha)
2550 yield sha
2551 except FileNotFoundError:
2552 pass
2554 for p in self.packs:
2555 bin_prefix = (
2556 binascii.unhexlify(prefix)
2557 if len(prefix) % 2 == 0
2558 else binascii.unhexlify(prefix[:-1])
2559 )
2560 for bin_sha in p.index.iter_prefix(bin_prefix):
2561 sha = sha_to_hex(bin_sha)
2562 if sha.startswith(prefix) and sha not in seen:
2563 seen.add(sha)
2564 yield sha
2565 for alternate in self.alternates:
2566 for sha in alternate.iter_prefix(prefix):
2567 if sha not in seen:
2568 seen.add(sha)
2569 yield sha
2571 def get_commit_graph(self) -> "CommitGraph | None":
2572 """Get the commit graph for this object store.
2574 Returns:
2575 CommitGraph object if available, None otherwise
2576 """
2577 if not self._use_commit_graph:
2578 return None
2580 if self._commit_graph is None:
2581 from .commit_graph import read_commit_graph
2583 # Look for commit graph in our objects directory
2584 graph_file = os.path.join(self.path, "info", "commit-graph")
2585 if os.path.exists(graph_file):
2586 self._commit_graph = read_commit_graph(graph_file)
2587 return self._commit_graph
2589 def get_midx(self) -> MultiPackIndex | None:
2590 """Get the multi-pack-index for this object store.
2592 Returns:
2593 MultiPackIndex object if available, None otherwise
2595 Raises:
2596 ValueError: If MIDX file is corrupt
2597 OSError: If MIDX file cannot be read
2598 """
2599 if not self._use_midx:
2600 return None
2602 if self._midx is None:
2603 # Look for MIDX in pack directory
2604 midx_file = os.path.join(self.pack_dir, "multi-pack-index")
2605 if os.path.exists(midx_file):
2606 self._midx = load_midx(midx_file)
2607 return self._midx
2609 def _get_pack_by_name(self, pack_name: str) -> Pack:
2610 """Get a pack referenced by a multi-pack-index entry.
2612 Args:
2613 pack_name: Pack index file name as stored in the MIDX. Usually
2614 ``pack-<hash>.idx``, but ``git maintenance`` writes packs
2615 named ``loose-<hash>.idx``, so any ``.idx`` basename is
2616 accepted.
2618 Returns:
2619 Pack object
2621 Raises:
2622 KeyError: If pack doesn't exist
2623 """
2624 if not pack_name.endswith(".idx"):
2625 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2626 # The name is joined under pack_dir below, so reject any path
2627 # separators that a corrupt or hostile MIDX could use to traverse
2628 # directories (backslash matters on Windows).
2629 if "/" in pack_name or "\\" in pack_name:
2630 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2631 basename = pack_name[: -len(".idx")]
2633 # _pack_cache is keyed by full basename and _update_pack_cache
2634 # discovers every "<name>-<hash>.pack" file (including the
2635 # "loose-<hash>" packs that git maintenance writes and the MIDX
2636 # references), so a referenced pack is normally already cached.
2637 try:
2638 return self._pack_cache[basename]
2639 except KeyError:
2640 pass
2642 pack_path = os.path.join(self.pack_dir, basename)
2643 if not os.path.exists(pack_path + ".pack"):
2644 raise KeyError(f"Pack {pack_name} not found")
2646 pack = Pack(
2647 pack_path,
2648 object_format=self.object_format,
2649 delta_window_size=self.pack_delta_window_size,
2650 window_memory=self.pack_window_memory,
2651 delta_cache_size=self.pack_delta_cache_size,
2652 depth=self.pack_depth,
2653 threads=self.pack_threads,
2654 big_file_threshold=self.pack_big_file_threshold,
2655 delta_base_cache_limit=self.delta_base_cache_limit,
2656 )
2657 self._pack_cache[basename] = pack
2658 self._mark_pack_used(basename)
2659 return pack
2661 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2662 """Check if a particular object is present by SHA1 and is packed.
2664 This checks the MIDX first if available, then falls back to checking
2665 individual pack indexes.
2667 Args:
2668 sha: SHA of the object (20/32 bytes binary or 40/64 bytes hex)
2670 Returns:
2671 True if the object is in a pack file
2672 """
2673 # Normalise to binary once: MIDX requires it, and passing binary to
2674 # the per-pack fallback avoids N redundant hex->binary conversions
2675 # inside PackIndex.object_offset. Mirrors ``get_raw`` below.
2676 if len(sha) == self.object_format.hex_length:
2677 sha = hex_to_sha(cast(ObjectID, sha))
2679 midx = self.get_midx()
2680 if midx is not None:
2681 result = midx.object_offset(sha)
2682 if result is not None:
2683 pack_name, _offset = result
2684 # A MIDX outlives the packs it names: repack and prune write
2685 # new packs and remove the old ones without rewriting it. Only
2686 # answer from the MIDX once the pack it names can still be
2687 # opened, as git does in fill_midx_entry, and otherwise fall
2688 # through to the per-pack scan.
2689 try:
2690 self._get_pack_by_name(pack_name)
2691 except (KeyError, PackFileDisappeared):
2692 pass
2693 else:
2694 return True
2696 # Fall back to checking individual packs
2697 return super().contains_packed(sha)
2699 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2700 """Obtain the raw fulltext for an object.
2702 This uses the MIDX if available for faster lookups.
2704 Args:
2705 name: SHA for the object (20 bytes binary or 40 bytes hex)
2707 Returns:
2708 Tuple with numeric type and object contents
2710 Raises:
2711 KeyError: If object not found
2712 """
2713 sha: RawObjectID
2714 if len(name) in (40, 64):
2715 # name is ObjectID (hex), convert to RawObjectID
2716 # Support both SHA1 (40) and SHA256 (64)
2717 sha = hex_to_sha(cast(ObjectID, name))
2718 elif len(name) in (20, 32):
2719 # name is already RawObjectID (binary)
2720 # Support both SHA1 (20) and SHA256 (32)
2721 sha = RawObjectID(name)
2722 else:
2723 raise AssertionError(f"Invalid object name {name!r}")
2725 # Try MIDX first for faster lookup
2726 midx = self.get_midx()
2727 if midx is not None:
2728 result = midx.object_offset(sha)
2729 if result is not None:
2730 pack_name, _offset = result
2731 try:
2732 pack = self._get_pack_by_name(pack_name)
2733 return pack.get_raw(sha)
2734 except (KeyError, PackFileDisappeared):
2735 # Pack disappeared or object not found, fall through to standard lookup
2736 pass
2738 # Fall back to the standard implementation
2739 return super().get_raw(name)
2741 def write_midx(self) -> bytes:
2742 """Write a multi-pack-index file for this object store.
2744 Creates a MIDX file that indexes all pack files in the pack directory.
2746 Returns:
2747 SHA-1 checksum of the written MIDX file
2749 Raises:
2750 OSError: If the pack directory doesn't exist or MIDX can't be written
2751 """
2752 from .midx import write_midx_file
2754 midx_path = os.path.join(self.pack_dir, "multi-pack-index")
2755 # Skip packs that vanish mid-collection (e.g. concurrent
2756 # ``git repack``); the survivors still produce a valid MIDX.
2757 pack_entries: list[tuple[str, list[tuple[RawObjectID, int, int | None]]]] = []
2758 for pack in self.packs:
2759 try:
2760 entries = list(pack.index.iterentries())
2761 except PackFileDisappeared as exc:
2762 self._evict_pack(exc.obj)
2763 continue
2764 pack_entries.append((os.path.basename(pack._basename) + ".idx", entries))
2765 if not pack_entries:
2766 return b"\x00" * 20
2767 return write_midx_file(midx_path, pack_entries)
2769 def write_commit_graph(
2770 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
2771 ) -> None:
2772 """Write a commit graph file for this object store.
2774 Args:
2775 refs: List of refs to include. If None, includes all refs from object store.
2776 reachable: If True, includes all commits reachable from refs.
2777 If False, only includes the direct ref targets.
2778 """
2779 from .commit_graph import get_reachable_commits
2781 if refs is None:
2782 # Get all commit objects from the object store
2783 all_refs = []
2784 # Iterate through all objects to find commits
2785 for sha in self:
2786 try:
2787 obj = self[sha]
2788 if obj.type_name == b"commit":
2789 all_refs.append(sha)
2790 except KeyError:
2791 continue
2792 else:
2793 # Use provided refs
2794 all_refs = list(refs)
2796 if not all_refs:
2797 return # No commits to include
2799 if reachable:
2800 # Get all reachable commits
2801 commit_ids = get_reachable_commits(self, all_refs)
2802 else:
2803 # Just use the direct ref targets - ensure they're hex ObjectIDs
2804 commit_ids = []
2805 for ref in all_refs:
2806 if isinstance(ref, bytes) and len(ref) == self.object_format.hex_length:
2807 # Already hex ObjectID
2808 commit_ids.append(ref)
2809 elif (
2810 isinstance(ref, bytes) and len(ref) == self.object_format.oid_length
2811 ):
2812 # Binary SHA, convert to hex ObjectID
2813 commit_ids.append(sha_to_hex(RawObjectID(ref)))
2814 else:
2815 # Assume it's already correct format
2816 commit_ids.append(ref)
2818 if commit_ids:
2819 # Write commit graph directly to our object store path
2820 # Generate the commit graph
2821 from .commit_graph import generate_commit_graph
2823 graph = generate_commit_graph(self, commit_ids)
2825 if graph.entries:
2826 # Ensure the info directory exists
2827 info_dir = os.path.join(self.path, "info")
2828 os.makedirs(info_dir, exist_ok=True)
2829 adjust_shared_perm(info_dir, self.shared_perm)
2831 # Write using GitFile for atomic operation
2832 graph_path = os.path.join(info_dir, "commit-graph")
2833 with GitFile(graph_path, "wb", shared_perm=self.shared_perm) as f:
2834 assert isinstance(
2835 f, _GitFile
2836 ) # GitFile in write mode always returns _GitFile
2837 graph.write_to_file(f)
2839 # Clear cached commit graph so it gets reloaded
2840 self._commit_graph = None
2842 def prune(self, grace_period: int | None = None) -> None:
2843 """Prune/clean up this object store.
2845 This removes temporary files that were left behind by interrupted
2846 pack operations. These are files that start with ``tmp_pack_`` in the
2847 repository directory or files with .pack extension but no corresponding
2848 .idx file in the pack directory.
2850 Args:
2851 grace_period: Grace period in seconds for removing temporary files.
2852 If None, uses DEFAULT_TEMPFILE_GRACE_PERIOD.
2853 """
2854 import glob
2856 if grace_period is None:
2857 grace_period = DEFAULT_TEMPFILE_GRACE_PERIOD
2859 # Clean up tmp_pack_* files in the repository directory
2860 for tmp_file in glob.glob(os.path.join(self.path, "tmp_pack_*")):
2861 # Check if file is old enough (more than grace period)
2862 mtime = os.path.getmtime(tmp_file)
2863 if time.time() - mtime > grace_period:
2864 os.remove(tmp_file)
2866 # Clean up orphaned .pack files without corresponding .idx files
2867 try:
2868 pack_dir_contents = os.listdir(self.pack_dir)
2869 except FileNotFoundError:
2870 return
2872 pack_files = {}
2873 idx_files = set()
2875 for name in pack_dir_contents:
2876 if name.endswith(".pack"):
2877 base_name = name[:-5] # Remove .pack extension
2878 pack_files[base_name] = name
2879 elif name.endswith(".idx"):
2880 base_name = name[:-4] # Remove .idx extension
2881 idx_files.add(base_name)
2883 # Remove .pack files without corresponding .idx files
2884 for base_name, pack_name in pack_files.items():
2885 if base_name not in idx_files:
2886 pack_path = os.path.join(self.pack_dir, pack_name)
2887 # Check if file is old enough (more than grace period)
2888 mtime = os.path.getmtime(pack_path)
2889 if time.time() - mtime > grace_period:
2890 os.remove(pack_path)
2892 def close(self) -> None:
2893 """Close the object store and release resources.
2895 This method closes all cached pack files, MIDX, and frees associated resources.
2896 Can be called multiple times safely.
2897 """
2898 # Close MIDX if it's loaded
2899 if self._midx is not None:
2900 self._midx.close()
2901 self._midx = None
2903 # Close alternates
2904 if self._alternates is not None:
2905 for alt in self._alternates:
2906 alt.close()
2907 self._alternates = None
2909 # Call parent class close to handle pack files
2910 super().close()
2913class MemoryObjectStore(PackCapableObjectStore):
2914 """Object store that keeps all objects in memory."""
2916 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
2917 """Initialize a MemoryObjectStore.
2919 Creates an empty in-memory object store.
2921 Args:
2922 object_format: Hash algorithm to use (defaults to SHA1)
2923 """
2924 super().__init__(object_format=object_format)
2925 self._data: dict[ObjectID, ShaFile] = {}
2926 self.pack_compression_level = -1
2928 def _to_hexsha(self, sha: ObjectID | RawObjectID) -> ObjectID:
2929 if len(sha) == self.object_format.hex_length:
2930 return cast(ObjectID, sha)
2931 elif len(sha) == self.object_format.oid_length:
2932 return sha_to_hex(cast(RawObjectID, sha))
2933 else:
2934 raise ValueError(f"Invalid sha {sha!r}")
2936 def contains_loose(self, sha: ObjectID) -> bool:
2937 """Check if a particular object is present by SHA1 and is loose."""
2938 return self._to_hexsha(sha) in self._data
2940 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2941 """Check if a particular object is present by SHA1 and is packed."""
2942 return False
2944 def __iter__(self) -> Iterator[ObjectID]:
2945 """Iterate over the SHAs that are present in this store."""
2946 return iter(self._data.keys())
2948 @property
2949 def packs(self) -> list[Pack]:
2950 """List with pack objects."""
2951 return []
2953 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2954 """Obtain the raw text for an object.
2956 Args:
2957 name: sha for the object.
2958 Returns: tuple with numeric type and object contents.
2959 """
2960 obj = self[self._to_hexsha(name)]
2961 return obj.type_num, obj.as_raw_string()
2963 def __getitem__(self, name: ObjectID | RawObjectID) -> ShaFile:
2964 """Retrieve an object by SHA.
2966 Args:
2967 name: SHA of the object (as hex string or bytes)
2969 Returns:
2970 Copy of the ShaFile object
2972 Raises:
2973 KeyError: If the object is not found
2974 """
2975 return self._data[self._to_hexsha(name)].copy()
2977 def __delitem__(self, name: ObjectID) -> None:
2978 """Delete an object from this store, for testing only."""
2979 del self._data[self._to_hexsha(name)]
2980 self._invalidate_peel_cache()
2982 def add_object(self, obj: ShaFile) -> None:
2983 """Add a single object to this object store."""
2984 self._data[obj.id] = obj.copy()
2986 def add_objects(
2987 self,
2988 objects: Iterable[tuple[ShaFile, str | None]],
2989 progress: Callable[[str], None] | None = None,
2990 ) -> None:
2991 """Add a set of objects to this object store.
2993 Args:
2994 objects: Iterable over a list of (object, path) tuples
2995 progress: Optional progress reporting function.
2996 """
2997 for obj, path in objects:
2998 self.add_object(obj)
3000 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
3001 """Add a new pack to this object store.
3003 Because this object store doesn't support packs, we extract and add the
3004 individual objects.
3006 Returns: Fileobject to write to and a commit function to
3007 call when the pack is finished.
3008 """
3009 from tempfile import SpooledTemporaryFile
3011 f = SpooledTemporaryFile(max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-")
3013 def commit() -> None:
3014 size = f.tell()
3015 if size > 0:
3016 f.seek(0)
3018 p = PackData.from_file(f, self.object_format, size)
3019 try:
3020 # Verify the trailing pack checksum before extracting
3021 # objects. Without this, a fetch that delivered a
3022 # truncated pack would still be accepted: ``add_pack``
3023 # iterates objects by offset and never reaches the
3024 # trailing bytes, so a stream that lost the last few
3025 # bytes of its trailer slipped through silently.
3026 # ``add_thin_pack`` already validates via
3027 # ``PackStreamCopier.verify``; do the equivalent here.
3028 p.check()
3029 for obj in PackInflater.for_pack_data(p, self.get_raw):
3030 self.add_object(obj)
3031 finally:
3032 p.close()
3033 f.close()
3034 else:
3035 f.close()
3037 def abort() -> None:
3038 f.close()
3040 return f, commit, abort # type: ignore[return-value]
3042 def add_pack_data(
3043 self,
3044 count: int,
3045 unpacked_objects: Iterator[UnpackedObject],
3046 progress: Callable[[str], None] | None = None,
3047 ) -> None:
3048 """Add pack data to this object store.
3050 Args:
3051 count: Number of items to add
3052 unpacked_objects: Iterator of UnpackedObject instances
3053 progress: Optional progress reporting function.
3054 """
3055 if count == 0:
3056 return
3058 # Since MemoryObjectStore doesn't support pack files, we need to
3059 # extract individual objects. To handle deltas properly, we write
3060 # to a temporary pack and then use PackInflater to resolve them.
3061 f, commit, abort = self.add_pack()
3062 try:
3063 write_pack_data(
3064 f.write,
3065 unpacked_objects,
3066 num_records=count,
3067 progress=progress,
3068 object_format=self.object_format,
3069 )
3070 except BaseException:
3071 abort()
3072 raise
3073 else:
3074 commit()
3076 def add_thin_pack(
3077 self,
3078 read_all: Callable[[int], bytes],
3079 read_some: Callable[[int], bytes] | None,
3080 progress: Callable[[str], None] | None = None,
3081 *,
3082 max_input_size: int | None = None,
3083 ) -> None:
3084 """Add a new thin pack to this object store.
3086 Thin packs are packs that contain deltas with parents that exist
3087 outside the pack. Because this object store doesn't support packs, we
3088 extract and add the individual objects.
3090 Args:
3091 read_all: Read function that blocks until the number of
3092 requested bytes are read.
3093 read_some: Read function that returns at least one byte, but may
3094 not return the number of bytes requested.
3095 progress: Optional progress reporting function.
3096 max_input_size: Maximum number of bytes that may be read from
3097 the wire while ingesting this pack. ``None`` (the default) or
3098 ``0`` mean unlimited. Exceeding the cap raises
3099 ``PackInputTooLarge``.
3100 """
3101 if max_input_size:
3102 read_all, read_some = _bound_read_callables(
3103 read_all, read_some, max_input_size
3104 )
3106 f, commit, abort = self.add_pack()
3107 try:
3108 copier = PackStreamCopier(
3109 self.object_format.hash_func,
3110 read_all,
3111 read_some,
3112 f,
3113 )
3114 copier.verify()
3115 except BaseException:
3116 abort()
3117 raise
3118 else:
3119 commit()
3122class ObjectIterator(Protocol):
3123 """Interface for iterating over objects."""
3125 def iterobjects(self) -> Iterator[ShaFile]:
3126 """Iterate over all objects.
3128 Returns:
3129 Iterator of ShaFile objects
3130 """
3131 raise NotImplementedError(self.iterobjects)
3134def tree_lookup_path(
3135 lookup_obj: Callable[[ObjectID | RawObjectID], ShaFile],
3136 root_sha: ObjectID | RawObjectID,
3137 path: bytes,
3138) -> tuple[int, ObjectID]:
3139 """Look up an object in a Git tree.
3141 Args:
3142 lookup_obj: Callback for retrieving object by SHA1
3143 root_sha: SHA1 of the root tree
3144 path: Path to lookup
3145 Returns: A tuple of (mode, SHA) of the resulting path.
3146 """
3147 tree = lookup_obj(root_sha)
3148 if not isinstance(tree, Tree):
3149 raise NotTreeError(root_sha)
3150 return tree.lookup_path(lookup_obj, path)
3153def _collect_filetree_revs(
3154 obj_store: ObjectContainer, tree_sha: ObjectID, kset: set[ObjectID]
3155) -> None:
3156 """Collect SHA1s of files and directories for specified tree.
3158 Args:
3159 obj_store: Object store to get objects by SHA from
3160 tree_sha: tree reference to walk
3161 kset: set to fill with references to files and directories
3162 """
3163 filetree = obj_store[tree_sha]
3164 assert isinstance(filetree, Tree)
3165 for name, mode, sha in filetree.iteritems():
3166 assert mode is not None
3167 assert sha is not None
3168 if not S_ISGITLINK(mode) and sha not in kset:
3169 kset.add(sha)
3170 if stat.S_ISDIR(mode):
3171 _collect_filetree_revs(obj_store, sha, kset)
3174def _split_commits_and_tags(
3175 obj_store: ObjectContainer,
3176 lst: Iterable[ObjectID],
3177 *,
3178 unknown: str = "error",
3179) -> tuple[set[ObjectID], set[ObjectID], set[ObjectID]]:
3180 """Split object id list into three lists with commit, tag, and other SHAs.
3182 Commits referenced by tags are included into commits
3183 list as well. Only SHA1s known in this repository will get
3184 through, controlled by the unknown parameter.
3186 Args:
3187 obj_store: Object store to get objects by SHA1 from
3188 lst: Collection of commit and tag SHAs
3189 unknown: How to handle unknown objects: "error", "warn", or "ignore"
3190 Returns: A tuple of (commits, tags, others) SHA1s
3191 """
3192 if unknown not in ("error", "warn", "ignore"):
3193 raise ValueError(
3194 f"unknown must be 'error', 'warn', or 'ignore', got {unknown!r}"
3195 )
3197 commits: set[ObjectID] = set()
3198 tags: set[ObjectID] = set()
3199 others: set[ObjectID] = set()
3200 for e in lst:
3201 try:
3202 o = obj_store[e]
3203 except KeyError:
3204 if unknown == "error":
3205 raise
3206 elif unknown == "warn":
3207 logger.warning("Object %s not found in object store", e.decode("ascii"))
3208 # else: ignore
3209 else:
3210 if isinstance(o, Commit):
3211 commits.add(e)
3212 elif isinstance(o, Tag):
3213 tags.add(e)
3214 tagged = o.object[1]
3215 c, t, os = _split_commits_and_tags(obj_store, [tagged], unknown=unknown)
3216 commits |= c
3217 tags |= t
3218 others |= os
3219 else:
3220 others.add(e)
3221 return (commits, tags, others)
3224class MissingObjectFinder:
3225 """Find the objects missing from another object store.
3227 Args:
3228 object_store: Object store containing at least all objects to be
3229 sent
3230 haves: SHA1s of commits not to send (already present in target)
3231 wants: SHA1s of commits to send
3232 progress: Optional function to report progress to.
3233 get_tagged: Function that returns a dict of pointed-to sha -> tag
3234 sha for including tags.
3235 get_parents: Optional function for getting the parents of a commit.
3236 """
3238 def __init__(
3239 self,
3240 object_store: BaseObjectStore,
3241 haves: Iterable[ObjectID],
3242 wants: Iterable[ObjectID],
3243 *,
3244 shallow: Set[ObjectID] | None = None,
3245 progress: Callable[[bytes], None] | None = None,
3246 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
3247 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3248 ) -> None:
3249 """Initialize a MissingObjectFinder.
3251 Args:
3252 object_store: Object store containing objects
3253 haves: SHA1s of objects already present in target
3254 wants: SHA1s of objects to send
3255 shallow: Set of shallow commit SHA1s
3256 progress: Optional progress reporting callback
3257 get_tagged: Function returning dict of pointed-to sha -> tag sha
3258 get_parents: Function for getting commit parents
3259 """
3260 self.object_store = object_store
3261 if shallow is None:
3262 shallow = set()
3263 self._get_parents = get_parents
3264 reachability = object_store.get_reachability_provider()
3265 # process Commits and Tags differently
3266 # haves may list commits/tags not available locally (silently ignore them).
3267 # wants should only contain valid SHAs (fail fast if not).
3268 have_commits, have_tags, have_others = _split_commits_and_tags(
3269 object_store, haves, unknown="ignore"
3270 )
3271 want_commits, want_tags, want_others = _split_commits_and_tags(
3272 object_store, wants, unknown="error"
3273 )
3274 # all_ancestors is a set of commits that shall not be sent
3275 # (complete repository up to 'haves')
3276 all_ancestors = reachability.get_reachable_commits(
3277 have_commits, exclude=None, shallow=shallow
3278 )
3279 # all_missing - complete set of commits between haves and wants
3280 # common_commits - boundary commits directly encountered when traversing wants
3281 # We use _collect_ancestors here because we need the exact boundary behavior:
3282 # commits that are in all_ancestors and directly reachable from wants,
3283 # but we don't traverse past them. This is hard to express with the
3284 # reachability abstraction alone.
3285 missing_commits, common_commits = _collect_ancestors(
3286 object_store,
3287 want_commits,
3288 frozenset(all_ancestors),
3289 shallow=frozenset(shallow),
3290 get_parents=self._get_parents,
3291 )
3293 self.remote_has: set[ObjectID] = set()
3294 # Now, fill sha_done with commits and revisions of
3295 # files and directories known to be both locally
3296 # and on target. Thus these commits and files
3297 # won't get selected for fetch
3298 for h in common_commits:
3299 self.remote_has.add(h)
3300 cmt = object_store[h]
3301 assert isinstance(cmt, Commit)
3302 # Get tree objects for this commit
3303 tree_objects = reachability.get_tree_objects([cmt.tree])
3304 self.remote_has.update(tree_objects)
3306 # record tags we have as visited, too
3307 for t in have_tags:
3308 self.remote_has.add(t)
3309 self.sha_done = set(self.remote_has)
3311 # in fact, what we 'want' is commits, tags, and others
3312 # we've found missing
3313 self.objects_to_send: set[tuple[ObjectID, bytes | None, int | None, bool]] = {
3314 (w, None, Commit.type_num, False) for w in missing_commits
3315 }
3316 missing_tags = want_tags.difference(have_tags)
3317 self.objects_to_send.update(
3318 {(w, None, Tag.type_num, False) for w in missing_tags}
3319 )
3320 missing_others = want_others.difference(have_others)
3321 self.objects_to_send.update({(w, None, None, False) for w in missing_others})
3323 if progress is None:
3324 self.progress: Callable[[bytes], None] = lambda x: None
3325 else:
3326 self.progress = progress
3327 self._tagged = (get_tagged and get_tagged()) or {}
3329 def get_remote_has(self) -> set[ObjectID]:
3330 """Get the set of SHAs the remote has.
3332 Returns:
3333 Set of SHA1s that the remote side already has
3334 """
3335 return self.remote_has
3337 def add_todo(
3338 self, entries: Iterable[tuple[ObjectID, bytes | None, int | None, bool]]
3339 ) -> None:
3340 """Add objects to the todo list.
3342 Args:
3343 entries: Iterable of tuples (sha, name, type_num, is_leaf)
3344 """
3345 self.objects_to_send.update([e for e in entries if e[0] not in self.sha_done])
3347 def __next__(self) -> tuple[ObjectID, PackHint | None]:
3348 """Get the next object to send.
3350 Returns:
3351 Tuple of (sha, pack_hint)
3353 Raises:
3354 StopIteration: When no more objects to send
3355 """
3356 while True:
3357 if not self.objects_to_send:
3358 self.progress(
3359 f"counting objects: {len(self.sha_done)}, done.\n".encode("ascii")
3360 )
3361 raise StopIteration
3362 (sha, name, type_num, leaf) = self.objects_to_send.pop()
3363 if sha not in self.sha_done:
3364 break
3365 if not leaf:
3366 o = self.object_store[sha]
3367 match o:
3368 case Commit():
3369 self.add_todo([(o.tree, b"", Tree.type_num, False)])
3370 case Tree():
3371 todos = []
3372 for n, m, s in o.iteritems():
3373 assert m is not None
3374 assert n is not None
3375 assert s is not None
3376 if not S_ISGITLINK(m):
3377 todos.append(
3378 (
3379 s,
3380 n,
3381 (
3382 Blob.type_num
3383 if stat.S_ISREG(m)
3384 else Tree.type_num
3385 ),
3386 not stat.S_ISDIR(m),
3387 )
3388 )
3389 self.add_todo(todos)
3390 case Tag():
3391 self.add_todo([(o.object[1], None, o.object[0].type_num, False)])
3392 if sha in self._tagged:
3393 self.add_todo([(self._tagged[sha], None, None, True)])
3394 self.sha_done.add(sha)
3395 if len(self.sha_done) % 1000 == 0:
3396 self.progress(f"counting objects: {len(self.sha_done)}\r".encode("ascii"))
3397 if type_num is None:
3398 pack_hint = None
3399 else:
3400 pack_hint = (type_num, name)
3401 return (sha, pack_hint)
3403 def __iter__(self) -> Iterator[tuple[ObjectID, PackHint | None]]:
3404 """Return iterator over objects to send.
3406 Returns:
3407 Self (this class implements the iterator protocol)
3408 """
3409 return self
3412class ObjectStoreGraphWalker:
3413 """Graph walker that finds what commits are missing from an object store."""
3415 heads: set[ObjectID]
3416 """Revisions without descendants in the local repo."""
3418 get_parents: Callable[[ObjectID], list[ObjectID]]
3419 """Function to retrieve parents in the local repo."""
3421 shallow: set[ObjectID]
3423 def __init__(
3424 self,
3425 local_heads: Iterable[ObjectID],
3426 get_parents: Callable[[ObjectID], list[ObjectID]],
3427 shallow: set[ObjectID] | None = None,
3428 update_shallow: Callable[[set[ObjectID] | None, set[ObjectID] | None], None]
3429 | None = None,
3430 ) -> None:
3431 """Create a new instance.
3433 Args:
3434 local_heads: Heads to start search with
3435 get_parents: Function for finding the parents of a SHA1.
3436 shallow: Set of shallow commits.
3437 update_shallow: Function to update shallow commits.
3438 """
3439 self.heads = set(local_heads)
3440 self.get_parents = get_parents
3441 self.parents: dict[ObjectID, list[ObjectID] | None] = {}
3442 if shallow is None:
3443 shallow = set()
3444 self.shallow = shallow
3445 self.update_shallow = update_shallow
3447 def nak(self) -> None:
3448 """Nothing in common was found."""
3450 def ack(self, sha: ObjectID) -> None:
3451 """Ack that a revision and its ancestors are present in the source."""
3452 if len(sha) != 40:
3453 # TODO: support SHA256
3454 raise ValueError(f"unexpected sha {sha!r} received")
3455 ancestors = {sha}
3457 # stop if we run out of heads to remove
3458 while self.heads:
3459 for a in ancestors:
3460 if a in self.heads:
3461 self.heads.remove(a)
3463 # collect all ancestors
3464 new_ancestors = set()
3465 for a in ancestors:
3466 ps = self.parents.get(a)
3467 if ps is not None:
3468 new_ancestors.update(ps)
3469 self.parents[a] = None
3471 # no more ancestors; stop
3472 if not new_ancestors:
3473 break
3475 ancestors = new_ancestors
3477 def next(self) -> ObjectID | None:
3478 """Iterate over ancestors of heads in the target."""
3479 if self.heads:
3480 ret = self.heads.pop()
3481 try:
3482 ps = self.get_parents(ret)
3483 except KeyError:
3484 return None
3485 self.parents[ret] = ps
3486 self.heads.update([p for p in ps if p not in self.parents])
3487 return ret
3488 return None
3490 __next__ = next
3493def commit_tree_changes(
3494 object_store: BaseObjectStore,
3495 tree: ObjectID | Tree,
3496 changes: Sequence[tuple[bytes, int | None, ObjectID | None]],
3497) -> ObjectID:
3498 """Commit a specified set of changes to a tree structure.
3500 This will apply a set of changes on top of an existing tree, storing new
3501 objects in object_store.
3503 changes are a list of tuples with (path, mode, object_sha).
3504 Paths can be both blobs and trees. See the mode and
3505 object sha to None deletes the path.
3507 This method works especially well if there are only a small
3508 number of changes to a big tree. For a large number of changes
3509 to a large tree, use e.g. commit_tree.
3511 Args:
3512 object_store: Object store to store new objects in
3513 and retrieve old ones from.
3514 tree: Original tree root (SHA or Tree object)
3515 changes: changes to apply
3516 Returns: New tree root object
3517 """
3518 # TODO(jelmer): Save up the objects and add them using .add_objects
3519 # rather than with individual calls to .add_object.
3520 # Handle both Tree object and SHA
3521 if isinstance(tree, Tree):
3522 tree_obj: Tree = tree
3523 else:
3524 sha_obj = object_store[tree]
3525 assert isinstance(sha_obj, Tree)
3526 tree_obj = sha_obj
3527 nested_changes: dict[bytes, list[tuple[bytes, int | None, ObjectID | None]]] = {}
3528 for path, new_mode, new_sha in changes:
3529 try:
3530 (dirname, subpath) = path.split(b"/", 1)
3531 except ValueError:
3532 if new_sha is None:
3533 del tree_obj[path]
3534 else:
3535 assert new_mode is not None
3536 tree_obj[path] = (new_mode, new_sha)
3537 else:
3538 nested_changes.setdefault(dirname, []).append((subpath, new_mode, new_sha))
3539 for name, subchanges in nested_changes.items():
3540 try:
3541 orig_subtree_id: ObjectID | Tree = tree_obj[name][1]
3542 except KeyError:
3543 # For new directories, pass an empty Tree object
3544 orig_subtree_id = Tree()
3545 subtree_id = commit_tree_changes(object_store, orig_subtree_id, subchanges)
3546 subtree = object_store[subtree_id]
3547 assert isinstance(subtree, Tree)
3548 if len(subtree) == 0:
3549 del tree_obj[name]
3550 else:
3551 tree_obj[name] = (stat.S_IFDIR, subtree.id)
3552 object_store.add_object(tree_obj)
3553 return tree_obj.id
3556class OverlayObjectStore(BaseObjectStore):
3557 """Object store that can overlay multiple object stores."""
3559 def __init__(
3560 self,
3561 bases: list[BaseObjectStore],
3562 add_store: BaseObjectStore | None = None,
3563 ) -> None:
3564 """Initialize an OverlayObjectStore.
3566 Args:
3567 bases: List of base object stores to overlay
3568 add_store: Optional store to write new objects to
3570 Raises:
3571 ValueError: If stores have different hash algorithms
3572 """
3573 from .object_format import verify_same_object_format
3575 # Verify all stores use the same hash algorithm
3576 store_algorithms = [store.object_format for store in bases]
3577 if add_store:
3578 store_algorithms.append(add_store.object_format)
3580 object_format = verify_same_object_format(*store_algorithms)
3582 super().__init__(object_format=object_format)
3583 self.bases = bases
3584 self.add_store = add_store
3586 def add_object(self, obj: ShaFile) -> None:
3587 """Add a single object to the store.
3589 Args:
3590 obj: Object to add
3592 Raises:
3593 NotImplementedError: If no add_store was provided
3594 """
3595 if self.add_store is None:
3596 raise NotImplementedError(self.add_object)
3597 return self.add_store.add_object(obj)
3599 def add_objects(
3600 self,
3601 objects: Sequence[tuple[ShaFile, str | None]],
3602 progress: Callable[[str], None] | None = None,
3603 ) -> Pack | None:
3604 """Add multiple objects to the store.
3606 Args:
3607 objects: Iterator of objects to add
3608 progress: Optional progress reporting callback
3610 Raises:
3611 NotImplementedError: If no add_store was provided
3612 """
3613 if self.add_store is None:
3614 raise NotImplementedError(self.add_object)
3615 return self.add_store.add_objects(objects, progress)
3617 @property
3618 def packs(self) -> list[Pack]:
3619 """Get the list of packs from all overlaid stores.
3621 Returns:
3622 Combined list of packs from all base stores
3623 """
3624 ret = []
3625 for b in self.bases:
3626 ret.extend(b.packs)
3627 return ret
3629 def __iter__(self) -> Iterator[ObjectID]:
3630 """Iterate over all object SHAs in the overlaid stores.
3632 Returns:
3633 Iterator of object SHAs (deduped across stores)
3634 """
3635 done = set()
3636 for b in self.bases:
3637 for o_id in b:
3638 if o_id not in done:
3639 yield o_id
3640 done.add(o_id)
3642 def iterobjects_subset(
3643 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
3644 ) -> Iterator[ShaFile]:
3645 """Iterate over a subset of objects from the overlaid stores.
3647 Args:
3648 shas: Iterable of object SHAs to retrieve
3649 allow_missing: If True, skip missing objects; if False, raise KeyError
3651 Returns:
3652 Iterator of ShaFile objects
3654 Raises:
3655 KeyError: If an object is missing and allow_missing is False
3656 """
3657 todo = set(shas)
3658 found: set[ObjectID] = set()
3660 for b in self.bases:
3661 # Create a copy of todo for each base to avoid modifying
3662 # the set while iterating through it
3663 current_todo = todo - found
3664 for o in b.iterobjects_subset(current_todo, allow_missing=True):
3665 yield o
3666 found.add(o.id)
3668 # Check for any remaining objects not found
3669 missing = todo - found
3670 if missing and not allow_missing:
3671 raise KeyError(next(iter(missing)))
3673 def iter_unpacked_subset(
3674 self,
3675 shas: Iterable[ObjectID | RawObjectID],
3676 *,
3677 include_comp: bool = False,
3678 allow_missing: bool = False,
3679 convert_ofs_delta: bool = True,
3680 ) -> Iterator[UnpackedObject]:
3681 """Iterate over unpacked objects from the overlaid stores.
3683 Args:
3684 shas: Iterable of object SHAs to retrieve
3685 include_comp: Whether to include compressed data
3686 allow_missing: If True, skip missing objects; if False, raise KeyError
3687 convert_ofs_delta: Whether to convert OFS_DELTA objects
3689 Returns:
3690 Iterator of unpacked objects
3692 Raises:
3693 KeyError: If an object is missing and allow_missing is False
3694 """
3695 todo: set[ObjectID | RawObjectID] = set(shas)
3696 for b in self.bases:
3697 for o in b.iter_unpacked_subset(
3698 todo,
3699 include_comp=include_comp,
3700 allow_missing=True,
3701 convert_ofs_delta=convert_ofs_delta,
3702 ):
3703 yield o
3704 todo.remove(o.sha())
3705 if todo and not allow_missing:
3706 raise KeyError(next(iter(todo)))
3708 def get_raw(self, name: ObjectID | RawObjectID) -> tuple[int, bytes]:
3709 """Get the raw object data from the overlaid stores.
3711 Args:
3712 name: SHA of the object
3714 Returns:
3715 Tuple of (type_num, raw_data)
3717 Raises:
3718 KeyError: If object not found in any base store
3719 """
3720 for b in self.bases:
3721 try:
3722 return b.get_raw(name)
3723 except KeyError:
3724 pass
3725 raise KeyError(name)
3727 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
3728 """Check if an object is packed in any base store.
3730 Args:
3731 sha: SHA of the object
3733 Returns:
3734 True if object is packed in any base store
3735 """
3736 for b in self.bases:
3737 if b.contains_packed(sha):
3738 return True
3739 return False
3741 def contains_loose(self, sha: ObjectID) -> bool:
3742 """Check if an object is loose in any base store.
3744 Args:
3745 sha: SHA of the object
3747 Returns:
3748 True if object is loose in any base store
3749 """
3750 for b in self.bases:
3751 if b.contains_loose(sha):
3752 return True
3753 return False
3756def read_packs_file(f: BinaryIO) -> Iterator[str]:
3757 """Yield the packs listed in a packs file."""
3758 for line in f.read().splitlines():
3759 if not line:
3760 continue
3761 (kind, name) = line.split(b" ", 1)
3762 if kind != b"P":
3763 continue
3764 yield os.fsdecode(name)
3767class BucketBasedObjectStore(PackBasedObjectStore):
3768 """Object store implementation that uses a bucket store like S3 as backend."""
3770 def _iter_loose_objects(self) -> Iterator[ObjectID]:
3771 """Iterate over the SHAs of all loose objects."""
3772 return iter([])
3774 def _get_loose_object(self, sha: ObjectID) -> None:
3775 return None
3777 def delete_loose_object(self, sha: ObjectID) -> None:
3778 """Delete a loose object (no-op for bucket stores).
3780 Bucket-based stores don't have loose objects, so this is a no-op.
3782 Args:
3783 sha: SHA of the object to delete
3784 """
3785 # Doesn't exist..
3787 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
3788 """Pack loose objects. Returns number of objects packed.
3790 BucketBasedObjectStore doesn't support loose objects, so this is a no-op.
3792 Args:
3793 progress: Optional progress reporting callback (ignored)
3794 """
3795 return 0
3797 def _remove_pack_by_name(self, name: str) -> None:
3798 """Remove a pack by name. Subclasses should implement this."""
3799 raise NotImplementedError(self._remove_pack_by_name)
3801 def _iter_pack_names(self) -> Iterator[str]:
3802 raise NotImplementedError(self._iter_pack_names)
3804 def _get_pack(self, name: str) -> Pack:
3805 raise NotImplementedError(self._get_pack)
3807 def _update_pack_cache(self) -> list[Pack]:
3808 pack_files = set(self._iter_pack_names())
3810 # Open newly appeared pack files
3811 new_packs = []
3812 for f in pack_files:
3813 if f not in self._pack_cache:
3814 pack = self._get_pack(f)
3815 new_packs.append(pack)
3816 self._pack_cache[f] = pack
3817 # Remove disappeared pack files
3818 for f in set(self._pack_cache) - pack_files:
3819 self._pack_cache.pop(f).close()
3820 return new_packs
3822 def _upload_pack(
3823 self, basename: str, pack_file: BinaryIO, index_file: BinaryIO
3824 ) -> None:
3825 raise NotImplementedError
3827 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
3828 """Add a new pack to this object store.
3830 Returns: Fileobject to write to, a commit function to
3831 call when the pack is finished and an abort
3832 function.
3833 """
3834 import tempfile
3836 pf = tempfile.SpooledTemporaryFile(
3837 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3838 )
3840 def commit() -> Pack | None:
3841 if pf.tell() == 0:
3842 pf.close()
3843 return None
3845 pf.seek(0)
3847 p = PackData(pf.name, file=pf, object_format=self.object_format)
3848 entries = p.sorted_entries()
3849 basename = iter_sha1(entry[0] for entry in entries).decode("ascii")
3850 idxf = tempfile.SpooledTemporaryFile(
3851 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3852 )
3853 checksum = p.get_stored_checksum()
3854 write_pack_index(idxf, entries, checksum, version=self.pack_index_version)
3855 idxf.seek(0)
3856 idx = load_pack_index_file(basename + ".idx", idxf, self.object_format)
3857 for pack in self.packs:
3858 if pack.get_stored_checksum() == p.get_stored_checksum():
3859 p.close()
3860 idx.close()
3861 pf.close()
3862 idxf.close()
3863 return pack
3864 pf.seek(0)
3865 idxf.seek(0)
3866 self._upload_pack(basename, pf, idxf) # type: ignore[arg-type]
3867 final_pack = Pack.from_objects(p, idx)
3868 self._add_cached_pack(basename, final_pack)
3869 pf.close()
3870 idxf.close()
3871 return final_pack
3873 return pf, commit, pf.close # type: ignore[return-value]
3876def _collect_ancestors(
3877 store: ObjectContainer,
3878 heads: Iterable[ObjectID],
3879 common: frozenset[ObjectID] = frozenset(),
3880 shallow: frozenset[ObjectID] = frozenset(),
3881 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3882) -> tuple[set[ObjectID], set[ObjectID]]:
3883 """Collect all ancestors of heads up to (excluding) those in common.
3885 Args:
3886 store: Object store to get commits from
3887 heads: commits to start from
3888 common: commits to end at, or empty set to walk repository
3889 completely
3890 shallow: Set of shallow commits
3891 get_parents: Optional function for getting the parents of a
3892 commit.
3893 Returns: a tuple (A, B) where A - all commits reachable
3894 from heads but not present in common, B - common (shared) elements
3895 that are directly reachable from heads
3896 """
3897 bases = set()
3898 commits = set()
3899 queue: deque[ObjectID] = deque()
3900 queue.extend(heads)
3902 # Try to use commit graph if available
3903 commit_graph = store.get_commit_graph()
3905 while queue:
3906 e = queue.popleft()
3907 if e in common:
3908 bases.add(e)
3909 elif e not in commits:
3910 commits.add(e)
3911 if e in shallow:
3912 continue
3914 # Try to use commit graph for parent lookup
3915 parents = None
3916 if commit_graph:
3917 parents = commit_graph.get_parents(e)
3919 if parents is None:
3920 # Fall back to loading the object
3921 cmt = store[e]
3922 assert isinstance(cmt, Commit)
3923 parents = get_parents(cmt)
3925 queue.extend(parents)
3926 return (commits, bases)
3929def iter_tree_contents(
3930 store: ObjectContainer, tree_id: ObjectID | None, *, include_trees: bool = False
3931) -> Iterator[TreeEntry]:
3932 """Iterate the contents of a tree and all subtrees.
3934 Iteration is depth-first pre-order, as in e.g. os.walk.
3936 Args:
3937 store: Object store to get trees from
3938 tree_id: SHA1 of the tree.
3939 include_trees: If True, include tree objects in the iteration.
3941 Yields: TreeEntry namedtuples for all the objects in a tree.
3942 """
3943 if tree_id is None:
3944 return
3945 # This could be fairly easily generalized to >2 trees if we find a use
3946 # case.
3947 todo = [TreeEntry(b"", stat.S_IFDIR, tree_id)]
3948 while todo:
3949 entry = todo.pop()
3950 assert entry.mode is not None
3951 if stat.S_ISDIR(entry.mode):
3952 extra = []
3953 assert entry.sha is not None
3954 tree = store[entry.sha]
3955 assert isinstance(tree, Tree)
3956 for subentry in tree.iteritems(name_order=True):
3957 assert entry.path is not None
3958 extra.append(subentry.in_path(entry.path))
3959 todo.extend(reversed(extra))
3960 if not stat.S_ISDIR(entry.mode) or include_trees:
3961 yield entry
3964def iter_commit_contents(
3965 store: ObjectContainer,
3966 commit: Commit | ObjectID | RawObjectID,
3967 *,
3968 include: Sequence[str | bytes | Path] | None = None,
3969) -> Iterator[TreeEntry]:
3970 """Iterate the contents of the repository at the specified commit.
3972 This is a wrapper around iter_tree_contents() and
3973 tree_lookup_path() to simplify the common task of getting the
3974 contest of a repo at a particular commit. See also
3975 dulwich.index.build_file_from_blob() for writing individual files
3976 to disk.
3978 Args:
3979 store: Object store to get trees from
3980 commit: Commit object, or SHA1 of a commit
3981 include: if provided, only the entries whose paths are in the
3982 list, or whose parent tree is in the list, will be
3983 included. Note that duplicate or overlapping paths
3984 (e.g. ["foo", "foo/bar"]) may result in duplicate entries
3986 Yields: TreeEntry namedtuples for all matching files in a commit.
3987 """
3988 sha = commit.id if isinstance(commit, Commit) else commit
3989 if not isinstance(obj := store[sha], Commit):
3990 raise TypeError(
3991 f"{sha.decode('ascii')} should be ID of a Commit, but is {type(obj)}"
3992 )
3993 commit = obj
3994 encoding = commit.encoding or "utf-8"
3995 include_bytes: list[bytes] = (
3996 [
3997 path if isinstance(path, bytes) else str(path).encode(encoding)
3998 for path in include
3999 ]
4000 if include is not None
4001 else [b""]
4002 )
4004 for path in include_bytes:
4005 mode, obj_id = tree_lookup_path(store.__getitem__, commit.tree, path)
4006 # Iterate all contained files if path points to a dir, otherwise just get that
4007 # single file
4008 if isinstance(store[obj_id], Tree):
4009 for entry in iter_tree_contents(store, obj_id):
4010 yield entry.in_path(path)
4011 else:
4012 yield TreeEntry(path, mode, obj_id)
4015def peel_sha(
4016 store: ObjectContainer,
4017 sha: ObjectID | RawObjectID,
4018 cache: PeelCache | None = None,
4019) -> tuple[ShaFile, ShaFile]:
4020 """Peel all tags from a SHA.
4022 Args:
4023 store: Object store to get objects from
4024 sha: The object SHA to peel.
4025 cache: Optional mapping from tag id to fully-peeled id, updated as tags
4026 are peeled. Most callers want ObjectContainer.peel(), which supplies a
4027 cache kept on the store.
4028 Returns: The fully-peeled SHA1 of a tag object, after peeling all
4029 intermediate tags; if the original ref does not point to a tag,
4030 this will equal the original SHA1.
4031 """
4032 if cache is None:
4033 cache = {}
4034 unpeeled = obj = store[sha]
4035 seen: list[ObjectID] = []
4036 obj_class = object_class(obj.type_name)
4037 while obj_class is Tag:
4038 assert isinstance(obj, Tag)
4039 cached = cache.get(obj.id)
4040 if cached is not None:
4041 obj = store[cached]
4042 break
4043 seen.append(obj.id)
4044 obj_class, sha = obj.object
4045 obj = store[sha]
4046 for tag_id in seen:
4047 cache[tag_id] = obj.id
4048 return unpeeled, obj
4051class GraphTraversalReachability:
4052 """Naive graph traversal implementation of ObjectReachabilityProvider.
4054 This implementation wraps existing graph traversal functions
4055 (_collect_ancestors, _collect_filetree_revs) to provide the standard
4056 reachability interface without any performance optimizations.
4057 """
4059 def __init__(self, object_store: BaseObjectStore) -> None:
4060 """Initialize the graph traversal provider.
4062 Args:
4063 object_store: Object store to query
4064 """
4065 self.store = object_store
4067 def get_reachable_commits(
4068 self,
4069 heads: Iterable[ObjectID],
4070 exclude: Iterable[ObjectID] | None = None,
4071 shallow: Set[ObjectID] | None = None,
4072 ) -> set[ObjectID]:
4073 """Get all commits reachable from heads, excluding those in exclude.
4075 Uses _collect_ancestors for commit traversal.
4077 Args:
4078 heads: Starting commit SHAs
4079 exclude: Commit SHAs to exclude (and their ancestors)
4080 shallow: Set of shallow commit boundaries
4082 Returns:
4083 Set of commit SHAs reachable from heads but not from exclude
4084 """
4085 exclude_set = frozenset(exclude) if exclude else frozenset()
4086 shallow_set = frozenset(shallow) if shallow else frozenset()
4087 commits, _bases = _collect_ancestors(
4088 self.store, heads, exclude_set, shallow_set
4089 )
4090 return commits
4092 def get_tree_objects(
4093 self,
4094 tree_shas: Iterable[ObjectID],
4095 ) -> set[ObjectID]:
4096 """Get all trees and blobs reachable from the given trees.
4098 Uses _collect_filetree_revs for tree traversal.
4100 Args:
4101 tree_shas: Starting tree SHAs
4103 Returns:
4104 Set of tree and blob SHAs
4105 """
4106 result: set[ObjectID] = set()
4107 for tree_sha in tree_shas:
4108 _collect_filetree_revs(self.store, tree_sha, result)
4109 return result
4111 def get_reachable_objects(
4112 self,
4113 commits: Iterable[ObjectID],
4114 exclude_commits: Iterable[ObjectID] | None = None,
4115 ) -> set[ObjectID]:
4116 """Get all objects (commits + trees + blobs) reachable from commits.
4118 Args:
4119 commits: Starting commit SHAs
4120 exclude_commits: Commits whose objects should be excluded
4122 Returns:
4123 Set of all object SHAs (commits, trees, blobs)
4124 """
4125 commits_set = set(commits)
4126 result = set(commits_set)
4128 # Get trees for all commits
4129 tree_shas = []
4130 for commit_sha in commits_set:
4131 try:
4132 commit = self.store[commit_sha]
4133 if isinstance(commit, Commit):
4134 tree_shas.append(commit.tree)
4135 except KeyError:
4136 # Commit not in store, skip
4137 continue
4139 # Collect all tree/blob objects
4140 result.update(self.get_tree_objects(tree_shas))
4142 # Exclude objects from exclude_commits if needed
4143 if exclude_commits:
4144 exclude_objects = self.get_reachable_objects(exclude_commits, None)
4145 result -= exclude_objects
4147 return result
4150class BitmapReachability:
4151 """Bitmap-accelerated implementation of ObjectReachabilityProvider.
4153 This implementation uses packfile bitmap indexes where available to
4154 accelerate reachability queries. Falls back to graph traversal when
4155 bitmaps don't cover the requested commits.
4156 """
4158 def __init__(self, object_store: "PackBasedObjectStore") -> None:
4159 """Initialize the bitmap provider.
4161 Args:
4162 object_store: Pack-based object store with bitmap support
4163 """
4164 self.store = object_store
4165 # Fallback to graph traversal for operations not yet optimized
4166 self._fallback = GraphTraversalReachability(object_store)
4168 def _combine_commit_bitmaps(
4169 self,
4170 commit_shas: set[ObjectID],
4171 exclude_shas: set[ObjectID] | None = None,
4172 ) -> tuple["EWAHBitmap", "Pack"] | None:
4173 """Combine bitmaps for multiple commits using OR, with optional exclusion.
4175 Args:
4176 commit_shas: Set of commit SHAs to combine
4177 exclude_shas: Optional set of commit SHAs to exclude
4179 Returns:
4180 Tuple of (combined_bitmap, pack) or None if bitmaps unavailable
4181 """
4182 from .bitmap import find_commit_bitmaps
4184 # Find bitmaps for the commits
4185 commit_bitmaps = find_commit_bitmaps(commit_shas, self.store.packs)
4187 # If we can't find bitmaps for all commits, return None
4188 if len(commit_bitmaps) < len(commit_shas):
4189 return None
4191 # Combine bitmaps using OR
4192 combined_bitmap = None
4193 result_pack = None
4195 for commit_sha in commit_shas:
4196 pack, pack_bitmap, _sha_to_pos = commit_bitmaps[commit_sha]
4197 commit_bitmap = pack_bitmap.get_bitmap(commit_sha)
4199 if commit_bitmap is None:
4200 return None
4202 if combined_bitmap is None:
4203 combined_bitmap = commit_bitmap
4204 result_pack = pack
4205 elif pack == result_pack:
4206 # Same pack, can OR directly
4207 combined_bitmap = combined_bitmap | commit_bitmap
4208 else:
4209 # Different packs, can't combine
4210 return None
4212 # Handle exclusions if provided
4213 if exclude_shas and result_pack and combined_bitmap:
4214 exclude_bitmaps = find_commit_bitmaps(exclude_shas, [result_pack])
4216 if len(exclude_bitmaps) == len(exclude_shas):
4217 # All excludes have bitmaps, compute exclusion
4218 exclude_combined = None
4220 for commit_sha in exclude_shas:
4221 _pack, pack_bitmap, _sha_to_pos = exclude_bitmaps[commit_sha]
4222 exclude_bitmap = pack_bitmap.get_bitmap(commit_sha)
4224 if exclude_bitmap is None:
4225 break
4227 if exclude_combined is None:
4228 exclude_combined = exclude_bitmap
4229 else:
4230 exclude_combined = exclude_combined | exclude_bitmap
4232 # Subtract excludes using set difference
4233 if exclude_combined:
4234 combined_bitmap = combined_bitmap - exclude_combined
4236 if combined_bitmap and result_pack:
4237 return (combined_bitmap, result_pack)
4238 return None
4240 def get_reachable_commits(
4241 self,
4242 heads: Iterable[ObjectID],
4243 exclude: Iterable[ObjectID] | None = None,
4244 shallow: Set[ObjectID] | None = None,
4245 ) -> set[ObjectID]:
4246 """Get all commits reachable from heads using bitmaps where possible.
4248 Args:
4249 heads: Starting commit SHAs
4250 exclude: Commit SHAs to exclude (and their ancestors)
4251 shallow: Set of shallow commit boundaries
4253 Returns:
4254 Set of commit SHAs reachable from heads but not from exclude
4255 """
4256 from .bitmap import bitmap_to_object_shas
4258 # If shallow is specified, fall back to graph traversal
4259 # (bitmaps don't support shallow boundaries well)
4260 if shallow:
4261 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4263 heads_set = set(heads)
4264 exclude_set = set(exclude) if exclude else None
4266 # Try to combine bitmaps
4267 result = self._combine_commit_bitmaps(heads_set, exclude_set)
4268 if result is None:
4269 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4271 combined_bitmap, result_pack = result
4273 # Convert bitmap to commit SHAs, filtering for commits only
4274 pack_bitmap = result_pack.bitmap
4275 if pack_bitmap is None:
4276 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4277 commit_type_filter = pack_bitmap.commit_bitmap
4278 return bitmap_to_object_shas(
4279 combined_bitmap, result_pack.index, commit_type_filter
4280 )
4282 def get_tree_objects(
4283 self,
4284 tree_shas: Iterable[ObjectID],
4285 ) -> set[ObjectID]:
4286 """Get all trees and blobs reachable from the given trees.
4288 Args:
4289 tree_shas: Starting tree SHAs
4291 Returns:
4292 Set of tree and blob SHAs
4293 """
4294 # Tree traversal doesn't benefit much from bitmaps, use fallback
4295 return self._fallback.get_tree_objects(tree_shas)
4297 def get_reachable_objects(
4298 self,
4299 commits: Iterable[ObjectID],
4300 exclude_commits: Iterable[ObjectID] | None = None,
4301 ) -> set[ObjectID]:
4302 """Get all objects reachable from commits using bitmaps.
4304 Args:
4305 commits: Starting commit SHAs
4306 exclude_commits: Commits whose objects should be excluded
4308 Returns:
4309 Set of all object SHAs (commits, trees, blobs)
4310 """
4311 from .bitmap import bitmap_to_object_shas
4313 commits_set = set(commits)
4314 exclude_set = set(exclude_commits) if exclude_commits else None
4316 # Try to combine bitmaps
4317 result = self._combine_commit_bitmaps(commits_set, exclude_set)
4318 if result is None:
4319 return self._fallback.get_reachable_objects(commits, exclude_commits)
4321 combined_bitmap, result_pack = result
4323 # Convert bitmap to all object SHAs (no type filter)
4324 return bitmap_to_object_shas(combined_bitmap, result_pack.index, None)