Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/object_store.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# object_store.py -- Object store for git objects
2# Copyright (C) 2008-2013 Jelmer Vernooij <jelmer@jelmer.uk>
3# and others
4#
5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU
7# General Public License as published by the Free Software Foundation; version 2.0
8# or (at your option) any later version. You can redistribute it and/or
9# modify it under the terms of either of these two licenses.
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17# You should have received a copy of the licenses; if not, see
18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License
19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache
20# License, Version 2.0.
21#
24"""Git object store interfaces and implementation."""
26__all__ = [
27 "DEFAULT_TEMPFILE_GRACE_PERIOD",
28 "INFODIR",
29 "PACKDIR",
30 "PACK_MODE",
31 "BaseObjectStore",
32 "BitmapReachability",
33 "BucketBasedObjectStore",
34 "DiskObjectStore",
35 "GraphTraversalReachability",
36 "GraphWalker",
37 "MemoryObjectStore",
38 "MissingObjectFinder",
39 "ObjectIterator",
40 "ObjectReachabilityProvider",
41 "ObjectStoreGraphWalker",
42 "OverlayObjectStore",
43 "PackBasedObjectStore",
44 "PackCapableObjectStore",
45 "PackContainer",
46 "PackInputTooLarge",
47 "commit_tree_changes",
48 "find_shallow",
49 "get_depth",
50 "iter_commit_contents",
51 "iter_tree_contents",
52 "peel_sha",
53 "read_packs_file",
54 "tree_lookup_path",
55]
57import binascii
58import logging
59import os
60import stat
61import sys
62import time
63import warnings
64from collections import deque
65from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence, Set
66from contextlib import closing, suppress
67from io import BytesIO
68from pathlib import Path
69from typing import (
70 TYPE_CHECKING,
71 BinaryIO,
72 Protocol,
73 TypeVar,
74 cast,
75)
77if TYPE_CHECKING:
78 from .object_format import ObjectFormat
80from .errors import NotTreeError
81from .file import GitFile, SharedPerm, _GitFile, adjust_shared_perm
82from .midx import MultiPackIndex, load_midx
83from .objects import (
84 DEFAULT_LOOSE_OBJECT_SIZE_LIMIT,
85 S_ISGITLINK,
86 Blob,
87 Commit,
88 ObjectID,
89 RawObjectID,
90 ShaFile,
91 Tag,
92 Tree,
93 TreeEntry,
94 hex_to_filename,
95 hex_to_sha,
96 object_class,
97 sha_to_hex,
98 valid_hexsha,
99)
100from .pack import (
101 PACK_SPOOL_FILE_MAX_SIZE,
102 ObjectContainer,
103 Pack,
104 PackData,
105 PackedObjectContainer,
106 PackFileDisappeared,
107 PackHint,
108 PackIndexEntry,
109 PackIndexer,
110 PackInflater,
111 PackStreamCopier,
112 UnpackedObject,
113 extend_pack,
114 full_unpacked_object,
115 generate_unpacked_objects,
116 iter_sha1,
117 load_pack_index_file,
118 pack_objects_to_data,
119 write_pack_data,
120 write_pack_index,
121)
122from .protocol import DEPTH_INFINITE, PEELED_TAG_SUFFIX
123from .refs import Ref
125if TYPE_CHECKING:
126 from .bitmap import EWAHBitmap
127 from .commit_graph import CommitGraph
128 from .config import Config
129 from .diff_tree import RenameDetector
130 from .pack import FilePackIndex, Pack
133logger = logging.getLogger(__name__)
135# Maximum number of times to rescan the pack directory after a pack file
136# disappears between snapshot and lazy open (e.g. concurrent repack).
137# Mirrors git's bounded reprepare_packed_git() retry.
138_MAX_PACK_RESCAN_ATTEMPTS = 3
140_T = TypeVar("_T")
143class GraphWalker(Protocol):
144 """Protocol for graph walker objects.
146 Implementations may also expose a ``shallow`` set, an ``unshallow`` set,
147 and an ``update_shallow`` callable for shallow-clone negotiation. These
148 are not part of the minimal protocol and callers must use ``hasattr`` or
149 ``getattr`` to access them.
150 """
152 def __next__(self) -> ObjectID | None:
153 """Return the next object SHA to visit."""
154 ...
156 def ack(self, sha: ObjectID, /) -> None:
157 """Acknowledge that an object has been received."""
158 ...
160 def nak(self) -> None:
161 """Nothing in common was found."""
162 ...
165class ObjectReachabilityProvider(Protocol):
166 """Protocol for computing object reachability queries.
168 This abstraction allows reachability computations to be backed by either
169 naive graph traversal or optimized bitmap indexes, with a consistent interface.
170 """
172 def get_reachable_commits(
173 self,
174 heads: Iterable[ObjectID],
175 exclude: Iterable[ObjectID] | None = None,
176 shallow: Set[ObjectID] | None = None,
177 ) -> set[ObjectID]:
178 """Get all commits reachable from heads, excluding those in exclude.
180 Args:
181 heads: Starting commit SHAs
182 exclude: Commit SHAs to exclude (and their ancestors)
183 shallow: Set of shallow commit boundaries (traversal stops here)
185 Returns:
186 Set of commit SHAs reachable from heads but not from exclude
187 """
188 ...
190 def get_reachable_objects(
191 self,
192 commits: Iterable[ObjectID],
193 exclude_commits: Iterable[ObjectID] | None = None,
194 ) -> set[ObjectID]:
195 """Get all objects (commits + trees + blobs) reachable from commits.
197 Args:
198 commits: Starting commit SHAs
199 exclude_commits: Commits whose objects should be excluded
201 Returns:
202 Set of all object SHAs (commits, trees, blobs, tags)
203 """
204 ...
206 def get_tree_objects(
207 self,
208 tree_shas: Iterable[ObjectID],
209 ) -> set[ObjectID]:
210 """Get all trees and blobs reachable from the given trees.
212 Args:
213 tree_shas: Starting tree SHAs
215 Returns:
216 Set of tree and blob SHAs
217 """
218 ...
221INFODIR = "info"
222PACKDIR = "pack"
224# use permissions consistent with Git; just readable by everyone
225# TODO: should packs also be non-writable on Windows? if so, that
226# would requite some rather significant adjustments to the test suite
227PACK_MODE = 0o444 if sys.platform != "win32" else 0o644
229# Grace period for cleaning up temporary pack files (in seconds)
230# Matches git's default of 2 weeks
231DEFAULT_TEMPFILE_GRACE_PERIOD = 14 * 24 * 60 * 60 # 2 weeks
234def _remove_readonly(path: str) -> None:
235 """Remove a file, clearing the read-only attribute first on Windows.
237 git stores pack files and loose objects read-only. Unix lets you unlink a
238 read-only file in a writable directory, but Windows refuses with
239 PermissionError, so clear the attribute and retry there.
240 """
241 try:
242 os.remove(path)
243 except PermissionError:
244 if sys.platform != "win32":
245 raise
246 os.chmod(path, stat.S_IWRITE | stat.S_IREAD)
247 os.remove(path)
250class PackInputTooLarge(OSError):
251 """Raised when a received pack exceeds the configured input size cap.
253 Mirrors the failure mode of git's ``receive.maxInputSize`` /
254 ``git index-pack --max-input-size``.
255 """
258def _bound_read_callables(
259 read_all: Callable[[int], bytes],
260 read_some: Callable[[int], bytes] | None,
261 max_input_size: int,
262) -> tuple[Callable[[int], bytes], Callable[[int], bytes] | None]:
263 """Wrap pack-stream read callbacks so total bytes are capped.
265 When the cumulative number of bytes returned across ``read_all`` and
266 ``read_some`` exceeds ``max_input_size``, the next read raises
267 ``PackInputTooLarge``. This is the in-process analogue of
268 ``git index-pack --max-input-size``.
269 """
270 bytes_read = [0]
272 def _check(n: int) -> None:
273 bytes_read[0] += n
274 if bytes_read[0] > max_input_size:
275 raise PackInputTooLarge(
276 f"pack exceeds maximum input size of {max_input_size} bytes"
277 )
279 def wrapped_read_all(n: int) -> bytes:
280 data = read_all(n)
281 _check(len(data))
282 return data
284 if read_some is None:
285 return wrapped_read_all, None
287 def wrapped_read_some(n: int) -> bytes:
288 data = read_some(n)
289 _check(len(data))
290 return data
292 return wrapped_read_all, wrapped_read_some
295def find_shallow(
296 store: ObjectContainer, heads: Iterable[ObjectID], depth: int
297) -> tuple[set[ObjectID], set[ObjectID]]:
298 """Find shallow commits according to a given depth.
300 Args:
301 store: An ObjectStore for looking up objects.
302 heads: Iterable of head SHAs to start walking from.
303 depth: The depth of ancestors to include. A depth of one includes
304 only the heads themselves.
305 Returns: A tuple of (shallow, not_shallow), sets of SHAs that should be
306 considered shallow and unshallow according to the arguments. Note that
307 these sets may overlap if a commit is reachable along multiple paths.
308 """
309 parents: dict[ObjectID, list[ObjectID]] = {}
310 commit_graph = store.get_commit_graph()
312 def get_parents(sha: ObjectID) -> list[ObjectID]:
313 result = parents.get(sha, None)
314 if not result:
315 # Try to use commit graph first if available
316 if commit_graph:
317 graph_parents = commit_graph.get_parents(sha)
318 if graph_parents is not None:
319 result = graph_parents
320 parents[sha] = result
321 return result
322 # Fall back to loading the object
323 commit = store[sha]
324 assert isinstance(commit, Commit)
325 result = commit.parents
326 parents[sha] = result
327 return result
329 todo = [] # stack of (sha, depth)
330 for head_sha in heads:
331 obj = store[head_sha]
332 # Peel tags if necessary
333 while isinstance(obj, Tag):
334 _, sha = obj.object
335 obj = store[sha]
336 if isinstance(obj, Commit):
337 todo.append((obj.id, 1))
339 not_shallow = set()
340 shallow = set()
341 # A commit reachable along N distinct paths was popped and re-expanded N
342 # times, so a merge-heavy history walked in exponential time. Deduplicate
343 # on the (sha, depth) state: re-processing a state adds to the same set and
344 # pushes the same parents, so skipping repeats leaves the result unchanged.
345 seen: set[tuple[ObjectID, int]] = set()
346 while todo:
347 state = todo.pop()
348 if state in seen:
349 continue
350 seen.add(state)
351 sha, cur_depth = state
352 if cur_depth < depth:
353 not_shallow.add(sha)
354 new_depth = cur_depth + 1
355 todo.extend((p, new_depth) for p in get_parents(sha))
356 else:
357 shallow.add(sha)
359 return shallow, not_shallow
362def get_depth(
363 store: ObjectContainer,
364 head: ObjectID,
365 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
366 max_depth: int | None = None,
367) -> int:
368 """Return the current available depth for the given head.
370 For commits with multiple parents, the largest possible depth will be
371 returned.
373 Args:
374 store: Object store to search in
375 head: commit to start from
376 get_parents: optional function for getting the parents of a commit
377 max_depth: maximum depth to search
378 """
379 if head not in store:
380 return 0
381 current_depth = 1
382 queue = deque([(head, current_depth)])
383 commit_graph = store.get_commit_graph()
385 # Without deduplication a commit reachable along several paths is expanded
386 # once per path, so a merge-heavy history is walked in exponential time.
387 # Track the (sha, depth) states already queued; re-visiting one only
388 # recomputes the same max and re-queues the same parents. deque.popleft()
389 # keeps the O(1) breadth-first order that list.pop(0) made O(n).
390 seen: set[tuple[ObjectID, int]] = set()
391 while queue and (max_depth is None or current_depth < max_depth):
392 e, depth = queue.popleft()
393 if (e, depth) in seen:
394 continue
395 seen.add((e, depth))
396 current_depth = max(current_depth, depth)
398 # Try to use commit graph for parent lookup if available
399 parents = None
400 if commit_graph:
401 parents = commit_graph.get_parents(e)
403 if parents is None:
404 # Fall back to loading the object
405 cmt = store[e]
406 if isinstance(cmt, Tag):
407 _cls, sha = cmt.object
408 cmt = store[sha]
409 parents = get_parents(cmt)
411 queue.extend((parent, depth + 1) for parent in parents if parent in store)
412 return current_depth
415class PackContainer(Protocol):
416 """Protocol for containers that can accept pack files."""
418 def add_pack(self) -> tuple[BytesIO, Callable[[], None], Callable[[], None]]:
419 """Add a new pack."""
420 ...
423class BaseObjectStore:
424 """Object store interface."""
426 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
427 """Initialize object store.
429 Args:
430 object_format: Object format to use (defaults to DEFAULT_OBJECT_FORMAT)
431 """
432 from .object_format import DEFAULT_OBJECT_FORMAT
434 self.object_format = object_format if object_format else DEFAULT_OBJECT_FORMAT
436 def determine_wants_all(
437 self, refs: Mapping[Ref, ObjectID], depth: int | None = None
438 ) -> list[ObjectID]:
439 """Determine which objects are wanted based on refs."""
441 def _want_deepen(sha: ObjectID) -> bool:
442 if not depth:
443 return False
444 if depth == DEPTH_INFINITE:
445 return True
446 return depth > self._get_depth(sha)
448 return [
449 sha
450 for (ref, sha) in refs.items()
451 if (sha not in self or _want_deepen(sha))
452 and not ref.endswith(PEELED_TAG_SUFFIX)
453 ]
455 def contains_loose(self, sha: ObjectID) -> bool:
456 """Check if a particular object is present by SHA1 and is loose."""
457 raise NotImplementedError(self.contains_loose)
459 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
460 """Check if a particular object is present by SHA1 and is packed."""
461 return False # Default implementation for stores that don't support packing
463 def __contains__(self, sha1: ObjectID) -> bool:
464 """Check if a particular object is present by SHA1.
466 This method makes no distinction between loose and packed objects.
467 """
468 return self.contains_loose(sha1)
470 @property
471 def packs(self) -> list[Pack]:
472 """Iterable of pack objects."""
473 raise NotImplementedError
475 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
476 """Obtain the raw text for an object.
478 Args:
479 name: sha for the object.
480 Returns: tuple with numeric type and object contents.
481 """
482 raise NotImplementedError(self.get_raw)
484 def __getitem__(self, sha1: ObjectID | RawObjectID) -> ShaFile:
485 """Obtain an object by SHA1.
487 Raises:
488 ChecksumMismatch: if the stored contents do not hash to the
489 requested object id.
490 """
491 if len(sha1) == self.object_format.oid_length:
492 hexsha = sha_to_hex(RawObjectID(sha1))
493 else:
494 hexsha = ObjectID(sha1)
495 type_num, uncomp = self.get_raw(sha1)
496 return ShaFile.from_raw_string(
497 type_num, uncomp, verify_sha=hexsha, object_format=self.object_format
498 )
500 def __iter__(self) -> Iterator[ObjectID]:
501 """Iterate over the SHAs that are present in this store."""
502 raise NotImplementedError(self.__iter__)
504 def add_object(self, obj: ShaFile) -> None:
505 """Add a single object to this object store."""
506 raise NotImplementedError(self.add_object)
508 def add_objects(
509 self,
510 objects: Sequence[tuple[ShaFile, str | None]],
511 progress: Callable[..., None] | None = None,
512 ) -> "Pack | None":
513 """Add a set of objects to this object store.
515 Args:
516 objects: Iterable over a list of (object, path) tuples
517 progress: Optional progress callback
518 """
519 raise NotImplementedError(self.add_objects)
521 def get_reachability_provider(
522 self, prefer_bitmaps: bool = True
523 ) -> ObjectReachabilityProvider:
524 """Get a reachability provider for this object store.
526 Returns an ObjectReachabilityProvider that can efficiently compute
527 object reachability queries. Subclasses can override this to provide
528 optimized implementations (e.g., using bitmap indexes).
530 Args:
531 prefer_bitmaps: Whether to prefer bitmap-based reachability if
532 available.
534 Returns:
535 ObjectReachabilityProvider instance
536 """
537 return GraphTraversalReachability(self)
539 def tree_changes(
540 self,
541 source: ObjectID | None,
542 target: ObjectID | None,
543 want_unchanged: bool = False,
544 include_trees: bool = False,
545 change_type_same: bool = False,
546 rename_detector: "RenameDetector | None" = None,
547 paths: Sequence[bytes] | None = None,
548 ) -> Iterator[
549 tuple[
550 tuple[bytes | None, bytes | None],
551 tuple[int | None, int | None],
552 tuple[ObjectID | None, ObjectID | None],
553 ]
554 ]:
555 """Find the differences between the contents of two trees.
557 Args:
558 source: SHA1 of the source tree
559 target: SHA1 of the target tree
560 want_unchanged: Whether unchanged files should be reported
561 include_trees: Whether to include trees
562 change_type_same: Whether to report files changing
563 type in the same entry.
564 rename_detector: RenameDetector object for detecting renames.
565 paths: Optional list of paths to filter to (as bytes).
566 Returns: Iterator over tuples with
567 (oldpath, newpath), (oldmode, newmode), (oldsha, newsha)
568 """
569 from .diff_tree import tree_changes
571 for change in tree_changes(
572 self,
573 source,
574 target,
575 want_unchanged=want_unchanged,
576 include_trees=include_trees,
577 change_type_same=change_type_same,
578 rename_detector=rename_detector,
579 paths=paths,
580 ):
581 old_path = change.old.path if change.old is not None else None
582 new_path = change.new.path if change.new is not None else None
583 old_mode = change.old.mode if change.old is not None else None
584 new_mode = change.new.mode if change.new is not None else None
585 old_sha = change.old.sha if change.old is not None else None
586 new_sha = change.new.sha if change.new is not None else None
587 yield (
588 (old_path, new_path),
589 (old_mode, new_mode),
590 (old_sha, new_sha),
591 )
593 def iter_tree_contents(
594 self, tree_id: ObjectID, include_trees: bool = False
595 ) -> Iterator[TreeEntry]:
596 """Iterate the contents of a tree and all subtrees.
598 Iteration is depth-first pre-order, as in e.g. os.walk.
600 Args:
601 tree_id: SHA1 of the tree.
602 include_trees: If True, include tree objects in the iteration.
603 Returns: Iterator over TreeEntry namedtuples for all the objects in a
604 tree.
605 """
606 warnings.warn(
607 "Please use dulwich.object_store.iter_tree_contents",
608 DeprecationWarning,
609 stacklevel=2,
610 )
611 return iter_tree_contents(self, tree_id, include_trees=include_trees)
613 def iterobjects_subset(
614 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
615 ) -> Iterator[ShaFile]:
616 """Iterate over a subset of objects in the store.
618 Args:
619 shas: Iterable of object SHAs to retrieve
620 allow_missing: If True, skip missing objects; if False, raise KeyError
622 Returns:
623 Iterator of ShaFile objects
625 Raises:
626 KeyError: If an object is missing and allow_missing is False
627 """
628 for sha in shas:
629 try:
630 yield self[sha]
631 except KeyError:
632 if not allow_missing:
633 raise
635 def iter_unpacked_subset(
636 self,
637 shas: Iterable[ObjectID | RawObjectID],
638 *,
639 include_comp: bool = False,
640 allow_missing: bool = False,
641 convert_ofs_delta: bool = True,
642 ) -> "Iterator[UnpackedObject]":
643 """Iterate over unpacked objects for a subset of SHAs.
645 Default implementation that converts ShaFile objects to UnpackedObject.
646 Subclasses may override for more efficient unpacked access.
648 Args:
649 shas: Iterable of object SHAs to retrieve
650 include_comp: Whether to include compressed data (ignored in base
651 implementation)
652 allow_missing: If True, skip missing objects; if False, raise
653 KeyError
654 convert_ofs_delta: Whether to convert OFS_DELTA objects (ignored in
655 base implementation)
657 Returns:
658 Iterator of UnpackedObject instances
660 Raises:
661 KeyError: If an object is missing and allow_missing is False
662 """
663 from .pack import UnpackedObject
665 for sha in shas:
666 try:
667 obj = self[sha]
668 # Convert ShaFile to UnpackedObject
669 unpacked = UnpackedObject(
670 obj.type_num, decomp_chunks=obj.as_raw_chunks(), sha=obj.id
671 )
672 yield unpacked
673 except KeyError:
674 if not allow_missing:
675 raise
677 def find_missing_objects(
678 self,
679 haves: Iterable[ObjectID],
680 wants: Iterable[ObjectID],
681 shallow: Set[ObjectID] | None = None,
682 progress: Callable[..., None] | None = None,
683 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
684 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
685 ) -> Iterator[tuple[ObjectID, PackHint | None]]:
686 """Find the missing objects required for a set of revisions.
688 Args:
689 haves: Iterable over SHAs already in common.
690 wants: Iterable over SHAs of objects to fetch.
691 shallow: Set of shallow commit SHA1s to skip
692 progress: Simple progress function that will be called with
693 updated progress strings.
694 get_tagged: Function that returns a dict of pointed-to sha ->
695 tag sha for including tags.
696 get_parents: Optional function for getting the parents of a
697 commit.
698 Returns: Iterator over (sha, path) pairs.
699 """
700 warnings.warn("Please use MissingObjectFinder(store)", DeprecationWarning)
701 finder = MissingObjectFinder(
702 self,
703 haves=haves,
704 wants=wants,
705 shallow=shallow,
706 progress=progress,
707 get_tagged=get_tagged,
708 get_parents=get_parents,
709 )
710 return iter(finder)
712 def find_common_revisions(self, graphwalker: GraphWalker) -> list[ObjectID]:
713 """Find which revisions this store has in common using graphwalker.
715 Args:
716 graphwalker: A graphwalker object.
717 Returns: List of SHAs that are in common
718 """
719 haves = []
720 sha = next(graphwalker)
721 while sha:
722 if sha in self:
723 haves.append(sha)
724 graphwalker.ack(sha)
725 sha = next(graphwalker)
726 return haves
728 def generate_pack_data(
729 self,
730 have: Iterable[ObjectID],
731 want: Iterable[ObjectID],
732 *,
733 shallow: Set[ObjectID] | None = None,
734 progress: Callable[..., None] | None = None,
735 ofs_delta: bool = True,
736 ) -> tuple[int, Iterator[UnpackedObject]]:
737 """Generate pack data objects for a set of wants/haves.
739 Args:
740 have: List of SHA1s of objects that should not be sent
741 want: List of SHA1s of objects that should be sent
742 shallow: Set of shallow commit SHA1s to skip
743 ofs_delta: Whether OFS deltas can be included
744 progress: Optional progress reporting method
745 """
746 # Note that the pack-specific implementation below is more efficient,
747 # as it reuses deltas
748 missing_objects = MissingObjectFinder(
749 self, haves=have, wants=want, shallow=shallow, progress=progress
750 )
751 object_ids = list(missing_objects)
752 return pack_objects_to_data(
753 [(self[oid], path) for oid, path in object_ids],
754 ofs_delta=ofs_delta,
755 progress=progress,
756 )
758 def peel_sha(self, sha: ObjectID | RawObjectID) -> ObjectID:
759 """Peel all tags from a SHA.
761 Args:
762 sha: The object SHA to peel.
763 Returns: The fully-peeled SHA1 of a tag object, after peeling all
764 intermediate tags; if the original ref does not point to a tag,
765 this will equal the original SHA1.
766 """
767 warnings.warn(
768 "Please use dulwich.object_store.peel_sha()",
769 DeprecationWarning,
770 stacklevel=2,
771 )
772 return peel_sha(self, sha)[1].id
774 def _get_depth(
775 self,
776 head: ObjectID,
777 get_parents: Callable[..., list[ObjectID]] = lambda commit: commit.parents,
778 max_depth: int | None = None,
779 ) -> int:
780 """Return the current available depth for the given head.
782 For commits with multiple parents, the largest possible depth will be
783 returned.
785 Args:
786 head: commit to start from
787 get_parents: optional function for getting the parents of a commit
788 max_depth: maximum depth to search
789 """
790 return get_depth(self, head, get_parents=get_parents, max_depth=max_depth)
792 def close(self) -> None:
793 """Close any files opened by this object store."""
794 # Default implementation is a NO-OP
796 def prune(self, grace_period: int | None = None) -> None:
797 """Prune/clean up this object store.
799 This includes removing orphaned temporary files and other
800 housekeeping tasks. Default implementation is a NO-OP.
802 Args:
803 grace_period: Grace period in seconds for removing temporary files.
804 If None, uses the default grace period.
805 """
806 # Default implementation is a NO-OP
808 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
809 """Iterate over all SHA1s that start with a given prefix.
811 The default implementation is a naive iteration over all objects.
812 However, subclasses may override this method with more efficient
813 implementations.
814 """
815 for sha in self:
816 if sha.startswith(prefix):
817 yield sha
819 def get_commit_graph(self) -> "CommitGraph | None":
820 """Get the commit graph for this object store.
822 Returns:
823 CommitGraph object if available, None otherwise
824 """
825 return None
827 def write_commit_graph(
828 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
829 ) -> None:
830 """Write a commit graph file for this object store.
832 Args:
833 refs: List of refs to include. If None, includes all refs from object store.
834 reachable: If True, includes all commits reachable from refs.
835 If False, only includes the direct ref targets.
837 Note:
838 Default implementation does nothing. Subclasses should override
839 this method to provide commit graph writing functionality.
840 """
841 raise NotImplementedError(self.write_commit_graph)
843 def get_object_mtime(self, sha: ObjectID) -> float:
844 """Get the modification time of an object.
846 Args:
847 sha: SHA1 of the object
849 Returns:
850 Modification time as seconds since epoch
852 Raises:
853 KeyError: if the object is not found
854 """
855 # Default implementation raises KeyError
856 # Subclasses should override to provide actual mtime
857 raise KeyError(sha)
860class PackCapableObjectStore(BaseObjectStore, PackedObjectContainer):
861 """Object store that supports pack operations.
863 This is a base class for object stores that can handle pack files,
864 including both disk-based and memory-based stores.
865 """
867 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
868 """Add a new pack to this object store.
870 Returns: Tuple of (file, commit_func, abort_func)
871 """
872 raise NotImplementedError(self.add_pack)
874 def add_pack_data(
875 self,
876 count: int,
877 unpacked_objects: Iterator["UnpackedObject"],
878 progress: Callable[..., None] | None = None,
879 ) -> "Pack | None":
880 """Add pack data to this object store.
882 Args:
883 count: Number of objects
884 unpacked_objects: Iterator over unpacked objects
885 progress: Optional progress callback
886 """
887 raise NotImplementedError(self.add_pack_data)
889 def get_unpacked_object(
890 self, sha1: ObjectID | RawObjectID, *, include_comp: bool = False
891 ) -> "UnpackedObject":
892 """Get a raw unresolved object.
894 Args:
895 sha1: SHA-1 hash of the object
896 include_comp: Whether to include compressed data
898 Returns:
899 UnpackedObject instance
900 """
901 from .pack import UnpackedObject
903 obj = self[sha1]
904 return UnpackedObject(obj.type_num, sha=sha1, decomp_chunks=obj.as_raw_chunks())
906 def iterobjects_subset(
907 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
908 ) -> Iterator[ShaFile]:
909 """Iterate over a subset of objects.
911 Args:
912 shas: Iterable of object SHAs to retrieve
913 allow_missing: If True, skip missing objects
915 Returns:
916 Iterator of ShaFile objects
917 """
918 for sha in shas:
919 try:
920 yield self[sha]
921 except KeyError:
922 if not allow_missing:
923 raise
926class PackBasedObjectStore(PackCapableObjectStore, PackedObjectContainer):
927 """Object store that uses pack files for storage.
929 This class provides a base implementation for object stores that use
930 Git pack files as their primary storage mechanism. It handles caching
931 of open pack files and provides configuration for pack file operations.
932 """
934 def __init__(
935 self,
936 pack_compression_level: int = -1,
937 pack_index_version: int | None = None,
938 pack_delta_window_size: int | None = None,
939 pack_window_memory: int | None = None,
940 pack_delta_cache_size: int | None = None,
941 pack_depth: int | None = None,
942 pack_threads: int | None = None,
943 pack_big_file_threshold: int | None = None,
944 *,
945 packed_git_limit: int | None = None,
946 delta_base_cache_limit: int | None = None,
947 object_format: "ObjectFormat | None" = None,
948 ) -> None:
949 """Initialize a PackBasedObjectStore.
951 Args:
952 pack_compression_level: Compression level for pack files (-1 to 9)
953 pack_index_version: Pack index version to use
954 pack_delta_window_size: Window size for delta compression
955 pack_window_memory: Maximum memory to use for delta window
956 pack_delta_cache_size: Cache size for delta operations
957 pack_depth: Maximum depth for pack deltas
958 pack_threads: Number of threads to use for packing
959 pack_big_file_threshold: Threshold for treating files as "big"
960 packed_git_limit: Maximum total bytes for mmapped pack files.
961 When exceeded, least-recently-used packs are closed to free memory.
962 delta_base_cache_limit: Maximum bytes for caching delta base objects.
963 Controls memory used to cache resolved base objects during delta
964 unpacking, corresponding to Git's core.deltaBaseCacheLimit.
965 object_format: Hash algorithm to use
966 """
967 super().__init__(object_format=object_format)
968 self._pack_cache: dict[str, Pack] = {}
969 self._pack_access_order: list[str] = []
970 self.packed_git_limit = packed_git_limit
971 self.delta_base_cache_limit = delta_base_cache_limit
972 self.pack_compression_level = pack_compression_level
973 self.pack_index_version = pack_index_version
974 self.pack_delta_window_size = pack_delta_window_size
975 self.pack_window_memory = pack_window_memory
976 self.pack_delta_cache_size = pack_delta_cache_size
977 self.pack_depth = pack_depth
978 self.pack_threads = pack_threads
979 self.pack_big_file_threshold = pack_big_file_threshold
981 def get_reachability_provider(
982 self,
983 prefer_bitmaps: bool = True,
984 ) -> ObjectReachabilityProvider:
985 """Get the best reachability provider for the object store.
987 Args:
988 prefer_bitmaps: Whether to use bitmaps if available
990 Returns:
991 ObjectReachabilityProvider implementation (either bitmap-accelerated
992 or graph traversal)
993 """
994 if prefer_bitmaps:
995 # Check if any packs have bitmaps. ``self.packs`` rescans the pack
996 # directory, so a pack removed by a concurrent repack is dropped
997 # from the cache and its replacement, if any, is probed here too.
998 for pack in self.packs:
999 try:
1000 if pack.bitmap is not None:
1001 return BitmapReachability(self)
1002 except FileNotFoundError:
1003 # Bitmap file doesn't exist for this pack
1004 continue
1005 except PackFileDisappeared as exc:
1006 # The pack vanished between the scan and the bitmap probe.
1007 self._evict_pack(exc.obj)
1008 continue
1010 # Fall back to graph traversal
1011 return GraphTraversalReachability(self)
1013 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
1014 """Add a new pack to this object store."""
1015 raise NotImplementedError(self.add_pack)
1017 def add_pack_data(
1018 self,
1019 count: int,
1020 unpacked_objects: Iterator[UnpackedObject],
1021 progress: Callable[..., None] | None = None,
1022 ) -> "Pack | None":
1023 """Add pack data to this object store.
1025 Args:
1026 count: Number of items to add
1027 unpacked_objects: Iterator of UnpackedObject instances
1028 progress: Optional progress callback
1029 """
1030 if count == 0:
1031 # Don't bother writing an empty pack file
1032 return None
1033 f, commit, abort = self.add_pack()
1034 try:
1035 write_pack_data(
1036 f.write,
1037 unpacked_objects,
1038 num_records=count,
1039 progress=progress,
1040 compression_level=self.pack_compression_level,
1041 object_format=self.object_format,
1042 )
1043 except BaseException:
1044 abort()
1045 raise
1046 else:
1047 return commit()
1049 @property
1050 def alternates(self) -> list["BaseObjectStore"]:
1051 """Return list of alternate object stores."""
1052 return []
1054 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
1055 """Check if a particular object is present by SHA1 and is packed.
1057 This does not check alternates.
1058 """
1060 def lookup(p: "Pack") -> bool:
1061 if sha in p:
1062 return True
1063 raise KeyError
1065 try:
1066 return self._lookup_in_packs(lookup)
1067 except KeyError:
1068 return False
1070 def __contains__(self, sha: ObjectID) -> bool:
1071 """Check if a particular object is present by SHA1.
1073 This method makes no distinction between loose and packed objects.
1074 """
1075 if self.contains_packed(sha) or self.contains_loose(sha):
1076 return True
1077 for alternate in self.alternates:
1078 if sha in alternate:
1079 return True
1080 return False
1082 def _add_cached_pack(self, base_name: str, pack: Pack) -> None:
1083 """Add a newly appeared pack to the cache by path."""
1084 prev_pack = self._pack_cache.get(base_name)
1085 if prev_pack is not pack:
1086 self._pack_cache[base_name] = pack
1087 if prev_pack:
1088 prev_pack.close()
1089 self._mark_pack_used(base_name)
1090 self._enforce_packed_git_limit()
1092 def generate_pack_data(
1093 self,
1094 have: Iterable[ObjectID],
1095 want: Iterable[ObjectID],
1096 *,
1097 shallow: Set[ObjectID] | None = None,
1098 progress: Callable[..., None] | None = None,
1099 ofs_delta: bool = True,
1100 ) -> tuple[int, Iterator[UnpackedObject]]:
1101 """Generate pack data objects for a set of wants/haves.
1103 Args:
1104 have: List of SHA1s of objects that should not be sent
1105 want: List of SHA1s of objects that should be sent
1106 shallow: Set of shallow commit SHA1s to skip
1107 ofs_delta: Whether OFS deltas can be included
1108 progress: Optional progress reporting method
1109 """
1110 missing_objects = MissingObjectFinder(
1111 self, haves=have, wants=want, shallow=shallow, progress=progress
1112 )
1113 remote_has = missing_objects.get_remote_has()
1114 object_ids = list(missing_objects)
1115 return len(object_ids), generate_unpacked_objects(
1116 self,
1117 object_ids,
1118 progress=progress,
1119 ofs_delta=ofs_delta,
1120 other_haves=remote_has,
1121 )
1123 def _clear_cached_packs(self) -> None:
1124 pack_cache = self._pack_cache
1125 self._pack_cache = {}
1126 self._pack_access_order = []
1127 while pack_cache:
1128 (_name, pack) = pack_cache.popitem()
1129 pack.close()
1131 def _total_pack_mmap_size(self) -> int:
1132 """Return the total mmapped memory across all cached packs."""
1133 return sum(pack.mmap_size for pack in self._pack_cache.values())
1135 def _mark_pack_used(self, pack_hash: str) -> None:
1136 """Mark a pack as recently used for LRU tracking."""
1137 try:
1138 self._pack_access_order.remove(pack_hash)
1139 except ValueError:
1140 pass
1141 self._pack_access_order.append(pack_hash)
1143 def _enforce_packed_git_limit(self) -> None:
1144 """Evict least-recently-used packs if the memory limit is exceeded."""
1145 if self.packed_git_limit is None:
1146 return
1147 while (
1148 self._pack_access_order
1149 and self._total_pack_mmap_size() > self.packed_git_limit
1150 ):
1151 oldest = self._pack_access_order.pop(0)
1152 pack = self._pack_cache.get(oldest)
1153 if pack is not None:
1154 pack.close()
1155 del self._pack_cache[oldest]
1157 def _iter_cached_packs(self) -> Iterator[Pack]:
1158 return iter(list(self._pack_cache.values()))
1160 def _evict_pack(self, pack: "Pack | FilePackIndex") -> None:
1161 """Evict a pack from the cache after its backing file disappeared.
1163 ``pack`` may be a :class:`Pack` or a :class:`FilePackIndex`; in the
1164 latter case the index's owning ``Pack`` is matched via the cached
1165 pack's ``_idx`` reference.
1166 """
1167 for key, cached in list(self._pack_cache.items()):
1168 if cached is pack or cached._idx is pack:
1169 del self._pack_cache[key]
1170 try:
1171 self._pack_access_order.remove(key)
1172 except ValueError:
1173 pass
1174 try:
1175 cached.close()
1176 except OSError:
1177 pass
1178 break
1180 def _lookup_in_packs(self, lookup: "Callable[[Pack], _T]") -> "_T":
1181 """Run ``lookup(pack)`` against each cached pack and return the first hit.
1183 ``lookup`` should raise ``KeyError`` if the pack does not contain the
1184 target. ``PackFileDisappeared`` from a concurrent ``git repack`` /
1185 ``gc --auto`` is caught: the stale pack is evicted, the pack
1186 directory is rescanned, and the search retries — bounded, mirroring
1187 git's ``reprepare_packed_git()``. If no cached pack has the object
1188 the pack directory is rescanned once to pick up any newly-arrived
1189 packs (e.g. another writer just landed one). ``KeyError`` is raised
1190 if no pack — old or new — has the object.
1191 """
1192 rescanned = False
1193 for _attempt in range(_MAX_PACK_RESCAN_ATTEMPTS):
1194 disappeared = False
1195 for pack_hash, pack in list(self._pack_cache.items()):
1196 try:
1197 result = lookup(pack)
1198 except KeyError:
1199 continue
1200 except PackFileDisappeared as exc:
1201 self._evict_pack(exc.obj)
1202 disappeared = True
1203 continue
1204 self._mark_pack_used(pack_hash)
1205 self._enforce_packed_git_limit()
1206 return result
1207 if disappeared:
1208 self._update_pack_cache()
1209 rescanned = True
1210 continue
1211 if not rescanned:
1212 # Maybe another process just landed a pack with the object.
1213 if self._update_pack_cache():
1214 rescanned = True
1215 continue
1216 break
1217 raise KeyError
1219 def _update_pack_cache(self) -> list[Pack]:
1220 raise NotImplementedError(self._update_pack_cache)
1222 def close(self) -> None:
1223 """Close the object store and release resources.
1225 This method closes all cached pack files and frees associated resources.
1226 Can be called multiple times safely.
1227 """
1228 self._clear_cached_packs()
1230 def __del__(self) -> None:
1231 """Warn if the object store is being deleted with unclosed packs."""
1232 if self._pack_cache:
1233 import warnings
1235 warnings.warn(
1236 f"ObjectStore {self!r} was destroyed with {len(self._pack_cache)} "
1237 "unclosed pack(s). Please call close() explicitly.",
1238 ResourceWarning,
1239 stacklevel=2,
1240 )
1241 self.close()
1243 @property
1244 def packs(self) -> list[Pack]:
1245 """List with pack objects."""
1246 return list(self._iter_cached_packs()) + list(self._update_pack_cache())
1248 def count_pack_files(self) -> int:
1249 """Count the number of pack files.
1251 Returns:
1252 Number of pack files (excluding those with .keep files)
1253 """
1254 count = 0
1255 for pack in self.packs:
1256 # Check if there's a .keep file for this pack
1257 keep_path = pack._basename + ".keep"
1258 if not os.path.exists(keep_path):
1259 count += 1
1260 return count
1262 def _iter_alternate_objects(self) -> Iterator[ObjectID]:
1263 """Iterate over the SHAs of all the objects in alternate stores."""
1264 for alternate in self.alternates:
1265 yield from alternate
1267 def _iter_loose_objects(self) -> Iterator[ObjectID]:
1268 """Iterate over the SHAs of all loose objects."""
1269 raise NotImplementedError(self._iter_loose_objects)
1271 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
1272 raise NotImplementedError(self._get_loose_object)
1274 def delete_loose_object(self, sha: ObjectID) -> None:
1275 """Delete a loose object.
1277 This method only handles loose objects. For packed objects,
1278 use repack(exclude=...) to exclude them during repacking.
1279 """
1280 raise NotImplementedError(self.delete_loose_object)
1282 def _remove_pack(self, pack: "Pack") -> None:
1283 raise NotImplementedError(self._remove_pack)
1285 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
1286 """Pack loose objects.
1288 Args:
1289 progress: Optional progress reporting callback
1291 Returns: Number of objects packed
1292 """
1293 objects: list[tuple[ShaFile, None]] = []
1294 for sha in self._iter_loose_objects():
1295 obj = self._get_loose_object(sha)
1296 if obj is not None:
1297 objects.append((obj, None))
1298 self.add_objects(objects, progress=progress)
1299 for obj, path in objects:
1300 self.delete_loose_object(obj.id)
1301 return len(objects)
1303 def repack(
1304 self,
1305 exclude: Set[bytes] | None = None,
1306 progress: Callable[[str], None] | None = None,
1307 ) -> int:
1308 """Repack the packs in this repository.
1310 Note that this implementation is fairly naive and currently keeps all
1311 objects in memory while it repacks.
1313 Args:
1314 exclude: Optional set of object SHAs to exclude from repacking
1315 progress: Optional progress reporting callback
1316 """
1317 if exclude is None:
1318 exclude = set()
1320 loose_objects = set()
1321 excluded_loose_objects = set()
1322 for sha in self._iter_loose_objects():
1323 if sha not in exclude:
1324 obj = self._get_loose_object(sha)
1325 if obj is not None:
1326 loose_objects.add(obj)
1327 else:
1328 excluded_loose_objects.add(sha)
1330 objects: set[tuple[ShaFile, None]] = {(obj, None) for obj in loose_objects}
1331 old_packs = {p.name(): p for p in self.packs}
1332 for name, pack in old_packs.items():
1333 objects.update(
1334 (obj, None) for obj in pack.iterobjects() if obj.id not in exclude
1335 )
1337 # Only create a new pack if there are objects to pack
1338 if objects:
1339 # The name of the consolidated pack might match the name of a
1340 # pre-existing pack. Take care not to remove the newly created
1341 # consolidated pack.
1342 consolidated = self.add_objects(list(objects), progress=progress)
1343 if consolidated is not None:
1344 old_packs.pop(consolidated.name(), None)
1346 # Delete loose objects that were packed
1347 for obj in loose_objects:
1348 if obj is not None:
1349 self.delete_loose_object(obj.id)
1350 # Delete excluded loose objects
1351 for sha in excluded_loose_objects:
1352 self.delete_loose_object(sha)
1353 for name, pack in old_packs.items():
1354 self._remove_pack(pack)
1355 self._update_pack_cache()
1356 return len(objects)
1358 def generate_pack_bitmaps(
1359 self,
1360 refs: dict[Ref, ObjectID],
1361 *,
1362 commit_interval: int | None = None,
1363 progress: Callable[[str], None] | None = None,
1364 ) -> int:
1365 """Generate bitmap indexes for all packs that don't have them.
1367 This generates .bitmap files for packfiles, enabling fast reachability
1368 queries. Equivalent to the bitmap generation part of 'git repack -b'.
1370 Args:
1371 refs: Dictionary of ref names to commit SHAs
1372 commit_interval: Include every Nth commit in bitmap index (None for default)
1373 progress: Optional progress reporting callback
1375 Returns:
1376 Number of bitmaps generated
1377 """
1378 count = 0
1379 for pack in self.packs:
1380 pack.ensure_bitmap(
1381 self, refs, commit_interval=commit_interval, progress=progress
1382 )
1383 count += 1
1385 # Update cache to pick up new bitmaps
1386 self._update_pack_cache()
1388 return count
1390 def __iter__(self) -> Iterator[ObjectID]:
1391 """Iterate over the SHAs that are present in this store."""
1392 self._update_pack_cache()
1393 for pack in self._iter_cached_packs():
1394 try:
1395 yield from pack
1396 except PackFileDisappeared as exc:
1397 self._evict_pack(exc.obj)
1398 yield from self._iter_loose_objects()
1399 yield from self._iter_alternate_objects()
1401 def contains_loose(self, sha: ObjectID) -> bool:
1402 """Check if a particular object is present by SHA1 and is loose.
1404 This does not check alternates.
1405 """
1406 return self._get_loose_object(sha) is not None
1408 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
1409 """Obtain the raw fulltext for an object.
1411 Args:
1412 name: sha for the object.
1413 Returns: tuple with numeric type and object contents.
1414 """
1415 sha: RawObjectID
1416 hexsha: ObjectID | None
1417 if len(name) == self.object_format.hex_length:
1418 sha = hex_to_sha(ObjectID(name))
1419 hexsha = cast(ObjectID, name)
1420 elif len(name) == self.object_format.oid_length:
1421 sha = RawObjectID(name)
1422 hexsha = None
1423 else:
1424 raise AssertionError(f"Invalid object name {name!r}")
1425 try:
1426 return self._lookup_in_packs(lambda p: p.get_raw(sha))
1427 except KeyError:
1428 pass
1429 if hexsha is None:
1430 hexsha = sha_to_hex(sha)
1431 ret = self._get_loose_object(hexsha)
1432 if ret is not None:
1433 return ret.type_num, ret.as_raw_string()
1434 for alternate in self.alternates:
1435 try:
1436 return alternate.get_raw(hexsha)
1437 except KeyError:
1438 pass
1439 raise KeyError(hexsha)
1441 def iter_unpacked_subset(
1442 self,
1443 shas: Iterable[ObjectID | RawObjectID],
1444 *,
1445 include_comp: bool = False,
1446 allow_missing: bool = False,
1447 convert_ofs_delta: bool = True,
1448 ) -> Iterator[UnpackedObject]:
1449 """Iterate over a subset of objects, yielding UnpackedObject instances.
1451 Args:
1452 shas: Set of object SHAs to retrieve
1453 include_comp: Whether to include compressed data
1454 allow_missing: If True, skip missing objects; if False, raise KeyError
1455 convert_ofs_delta: Whether to convert OFS_DELTA objects
1457 Returns:
1458 Iterator of UnpackedObject instances
1460 Raises:
1461 KeyError: If an object is missing and allow_missing is False
1462 """
1463 todo: set[ObjectID | RawObjectID] = set(shas)
1464 for p in self._iter_cached_packs():
1465 try:
1466 for unpacked in p.iter_unpacked_subset(
1467 todo,
1468 include_comp=include_comp,
1469 allow_missing=True,
1470 convert_ofs_delta=convert_ofs_delta,
1471 ):
1472 yield unpacked
1473 hexsha = sha_to_hex(unpacked.sha())
1474 todo.remove(hexsha)
1475 except PackFileDisappeared as exc:
1476 self._evict_pack(exc.obj)
1477 # Maybe something else has added a pack with the object
1478 # in the mean time?
1479 for p in self._update_pack_cache():
1480 try:
1481 for unpacked in p.iter_unpacked_subset(
1482 todo,
1483 include_comp=include_comp,
1484 allow_missing=True,
1485 convert_ofs_delta=convert_ofs_delta,
1486 ):
1487 yield unpacked
1488 hexsha = sha_to_hex(unpacked.sha())
1489 todo.remove(hexsha)
1490 except PackFileDisappeared as exc:
1491 self._evict_pack(exc.obj)
1492 for alternate in self.alternates:
1493 assert isinstance(alternate, PackBasedObjectStore)
1494 for unpacked in alternate.iter_unpacked_subset(
1495 todo,
1496 include_comp=include_comp,
1497 allow_missing=True,
1498 convert_ofs_delta=convert_ofs_delta,
1499 ):
1500 yield unpacked
1501 hexsha = sha_to_hex(unpacked.sha())
1502 todo.remove(hexsha)
1504 def iterobjects_subset(
1505 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
1506 ) -> Iterator[ShaFile]:
1507 """Iterate over a subset of objects in the store.
1509 This method searches for objects in pack files, alternates, and loose storage.
1511 Args:
1512 shas: Iterable of object SHAs to retrieve
1513 allow_missing: If True, skip missing objects; if False, raise KeyError
1515 Returns:
1516 Iterator of ShaFile objects
1518 Raises:
1519 KeyError: If an object is missing and allow_missing is False
1520 """
1521 todo: set[ObjectID] = set(shas)
1522 for p in self._iter_cached_packs():
1523 try:
1524 for o in p.iterobjects_subset(todo, allow_missing=True):
1525 yield o
1526 todo.remove(o.id)
1527 except PackFileDisappeared as exc:
1528 self._evict_pack(exc.obj)
1529 # Maybe something else has added a pack with the object
1530 # in the mean time?
1531 for p in self._update_pack_cache():
1532 try:
1533 for o in p.iterobjects_subset(todo, allow_missing=True):
1534 yield o
1535 todo.remove(o.id)
1536 except PackFileDisappeared as exc:
1537 self._evict_pack(exc.obj)
1538 for alternate in self.alternates:
1539 for o in alternate.iterobjects_subset(todo, allow_missing=True):
1540 yield o
1541 todo.remove(o.id)
1542 for oid in todo:
1543 loose_obj: ShaFile | None = self._get_loose_object(oid)
1544 if loose_obj is not None:
1545 yield loose_obj
1546 elif not allow_missing:
1547 raise KeyError(oid)
1549 def get_unpacked_object(
1550 self, sha1: bytes, *, include_comp: bool = False
1551 ) -> UnpackedObject:
1552 """Obtain the unpacked object.
1554 Args:
1555 sha1: sha for the object.
1556 include_comp: Whether to include compression metadata.
1557 """
1558 if len(sha1) == self.object_format.hex_length:
1559 sha = hex_to_sha(cast(ObjectID, sha1))
1560 hexsha = cast(ObjectID, sha1)
1561 elif len(sha1) == self.object_format.oid_length:
1562 sha = cast(RawObjectID, sha1)
1563 hexsha = None
1564 else:
1565 raise AssertionError(f"Invalid object sha1 {sha1!r}")
1566 try:
1567 return self._lookup_in_packs(
1568 lambda p: p.get_unpacked_object(sha, include_comp=include_comp)
1569 )
1570 except KeyError:
1571 pass
1572 if hexsha is None:
1573 hexsha = sha_to_hex(sha)
1574 for alternate in self.alternates:
1575 assert isinstance(alternate, PackBasedObjectStore)
1576 try:
1577 return alternate.get_unpacked_object(hexsha, include_comp=include_comp)
1578 except KeyError:
1579 pass
1580 raise KeyError(hexsha)
1582 def add_objects(
1583 self,
1584 objects: Sequence[tuple[ShaFile, str | None]],
1585 progress: Callable[[str], None] | None = None,
1586 ) -> "Pack | None":
1587 """Add a set of objects to this object store.
1589 Args:
1590 objects: Iterable over (object, path) tuples, should support
1591 __len__.
1592 progress: Optional progress reporting function.
1593 Returns: Pack object of the objects written.
1594 """
1595 count = len(objects)
1596 record_iter = (full_unpacked_object(o) for (o, p) in objects)
1597 return self.add_pack_data(count, record_iter, progress=progress)
1600class DiskObjectStore(PackBasedObjectStore):
1601 """Git-style object store that exists on disk."""
1603 path: str | os.PathLike[str]
1604 pack_dir: str | os.PathLike[str]
1605 _alternates: "list[BaseObjectStore] | None"
1606 _commit_graph: "CommitGraph | None"
1608 def __init__(
1609 self,
1610 path: str | os.PathLike[str],
1611 *,
1612 loose_compression_level: int = -1,
1613 pack_compression_level: int = -1,
1614 pack_index_version: int | None = None,
1615 pack_delta_window_size: int | None = None,
1616 pack_window_memory: int | None = None,
1617 pack_delta_cache_size: int | None = None,
1618 pack_depth: int | None = None,
1619 pack_threads: int | None = None,
1620 pack_big_file_threshold: int | None = None,
1621 packed_git_limit: int | None = None,
1622 delta_base_cache_limit: int | None = None,
1623 fsync_object_files: bool = False,
1624 pack_write_bitmaps: bool = False,
1625 pack_write_bitmap_hash_cache: bool = True,
1626 pack_write_bitmap_lookup_table: bool = True,
1627 shared_perm: "SharedPerm | None" = None,
1628 object_format: "ObjectFormat | None" = None,
1629 loose_object_size_limit: int | None = None,
1630 alternates: "Iterable[str | os.PathLike[str]] | None" = None,
1631 ) -> None:
1632 """Open an object store.
1634 Args:
1635 path: Path of the object store.
1636 loose_compression_level: zlib compression level for loose objects
1637 pack_compression_level: zlib compression level for pack objects
1638 pack_index_version: pack index version to use (1, 2, or 3)
1639 pack_delta_window_size: sliding window size for delta compression
1640 pack_window_memory: memory limit for delta window operations
1641 pack_delta_cache_size: size of cache for delta operations
1642 pack_depth: maximum delta chain depth
1643 pack_threads: number of threads for pack operations
1644 pack_big_file_threshold: threshold for treating files as big
1645 packed_git_limit: maximum total bytes for mmapped pack files
1646 delta_base_cache_limit: maximum bytes for delta base object cache
1647 fsync_object_files: whether to fsync object files for durability
1648 pack_write_bitmaps: whether to write bitmap indexes for packs
1649 pack_write_bitmap_hash_cache: whether to include name-hash cache in bitmaps
1650 pack_write_bitmap_lookup_table: whether to include lookup table in bitmaps
1651 shared_perm: Shared repository permission setting
1652 object_format: Hash algorithm to use (SHA1 or SHA256)
1653 loose_object_size_limit: Maximum inflated size of a single loose
1654 object. Defaults to core.bigFileThreshold's Git default (512 MiB)
1655 via :data:`DEFAULT_LOOSE_OBJECT_SIZE_LIMIT` when None. Guards against
1656 decompression-bomb attacks.
1657 alternates: Extra alternate object directory paths to consult, in
1658 addition to those listed in ``objects/info/alternates``. Used to
1659 plumb ``GIT_ALTERNATE_OBJECT_DIRECTORIES`` through from porcelain.
1660 """
1661 # Import here to avoid circular dependency
1662 from .object_format import DEFAULT_OBJECT_FORMAT
1664 super().__init__(
1665 pack_compression_level=pack_compression_level,
1666 pack_index_version=pack_index_version,
1667 pack_delta_window_size=pack_delta_window_size,
1668 pack_window_memory=pack_window_memory,
1669 pack_delta_cache_size=pack_delta_cache_size,
1670 pack_depth=pack_depth,
1671 pack_threads=pack_threads,
1672 pack_big_file_threshold=pack_big_file_threshold,
1673 packed_git_limit=packed_git_limit,
1674 delta_base_cache_limit=delta_base_cache_limit,
1675 object_format=object_format if object_format else DEFAULT_OBJECT_FORMAT,
1676 )
1677 self.path = path
1678 self.pack_dir = os.path.join(self.path, PACKDIR)
1679 self._alternates = None
1680 self._extra_alternate_paths: list[str] = (
1681 [os.fsdecode(os.fspath(p)) for p in alternates]
1682 if alternates is not None
1683 else []
1684 )
1685 self.loose_compression_level = loose_compression_level
1686 self.pack_compression_level = pack_compression_level
1687 self.pack_index_version = pack_index_version
1688 self.fsync_object_files = fsync_object_files
1689 self.pack_write_bitmaps = pack_write_bitmaps
1690 self.pack_write_bitmap_hash_cache = pack_write_bitmap_hash_cache
1691 self.pack_write_bitmap_lookup_table = pack_write_bitmap_lookup_table
1692 self.shared_perm = shared_perm
1693 self.loose_object_size_limit = (
1694 loose_object_size_limit
1695 if loose_object_size_limit is not None
1696 else DEFAULT_LOOSE_OBJECT_SIZE_LIMIT
1697 )
1699 # Commit graph support - lazy loaded
1700 self._commit_graph = None
1701 self._use_commit_graph = True # Default to true
1703 # Multi-pack-index support - lazy loaded
1704 self._midx: MultiPackIndex | None = None
1705 self._use_midx = True # Default to true
1707 def __repr__(self) -> str:
1708 """Return string representation of DiskObjectStore.
1710 Returns:
1711 String representation including the store path
1712 """
1713 return f"<{self.__class__.__name__}({self.path!r})>"
1715 @classmethod
1716 def from_config(
1717 cls,
1718 path: str | os.PathLike[str],
1719 config: "Config",
1720 *,
1721 shared_perm: "SharedPerm | None" = None,
1722 alternates: "Iterable[str | os.PathLike[str]] | None" = None,
1723 ) -> "DiskObjectStore":
1724 """Create a DiskObjectStore from a configuration object.
1726 Args:
1727 path: Path to the object store directory
1728 config: Configuration object to read settings from
1729 shared_perm: Optional shared repository permission setting
1730 alternates: Extra alternate object directory paths to consult,
1731 appended to those listed in ``objects/info/alternates``.
1733 Returns:
1734 New DiskObjectStore instance configured according to config
1735 """
1736 try:
1737 default_compression_level = int(
1738 config.get((b"core",), b"compression").decode()
1739 )
1740 except KeyError:
1741 default_compression_level = -1
1742 try:
1743 loose_compression_level = int(
1744 config.get((b"core",), b"looseCompression").decode()
1745 )
1746 except KeyError:
1747 loose_compression_level = default_compression_level
1748 try:
1749 pack_compression_level = int(
1750 config.get((b"core",), "packCompression").decode()
1751 )
1752 except KeyError:
1753 pack_compression_level = default_compression_level
1754 try:
1755 pack_index_version = int(config.get((b"pack",), b"indexVersion").decode())
1756 except KeyError:
1757 pack_index_version = None
1759 # Read pack configuration options
1760 try:
1761 pack_delta_window_size = int(
1762 config.get((b"pack",), b"deltaWindowSize").decode()
1763 )
1764 except KeyError:
1765 pack_delta_window_size = None
1766 try:
1767 pack_window_memory = int(config.get((b"pack",), b"windowMemory").decode())
1768 except KeyError:
1769 pack_window_memory = None
1770 try:
1771 pack_delta_cache_size = int(
1772 config.get((b"pack",), b"deltaCacheSize").decode()
1773 )
1774 except KeyError:
1775 pack_delta_cache_size = None
1776 try:
1777 pack_depth = int(config.get((b"pack",), b"depth").decode())
1778 except KeyError:
1779 pack_depth = None
1780 try:
1781 pack_threads = int(config.get((b"pack",), b"threads").decode())
1782 except KeyError:
1783 pack_threads = None
1784 try:
1785 pack_big_file_threshold = int(
1786 config.get((b"pack",), b"bigFileThreshold").decode()
1787 )
1788 except KeyError:
1789 pack_big_file_threshold = None
1791 # Read core.packedGitLimit setting
1792 try:
1793 packed_git_limit = int(config.get((b"core",), b"packedGitLimit").decode())
1794 except KeyError:
1795 packed_git_limit = None
1797 # Read core.deltaBaseCacheLimit setting
1798 try:
1799 delta_base_cache_limit = int(
1800 config.get((b"core",), b"deltaBaseCacheLimit").decode()
1801 )
1802 except KeyError:
1803 delta_base_cache_limit = None
1805 # Read core.bigFileThreshold setting; used as the upper bound for
1806 # inflating a single loose object, guarding against decompression bombs.
1807 try:
1808 loose_object_size_limit: int | None = int(
1809 config.get((b"core",), b"bigFileThreshold").decode()
1810 )
1811 except KeyError:
1812 loose_object_size_limit = None
1814 # Read core.commitGraph setting
1815 use_commit_graph = config.get_boolean((b"core",), b"commitGraph", True)
1817 # Read core.multiPackIndex setting
1818 use_midx = config.get_boolean((b"core",), b"multiPackIndex", True)
1820 # Read core.fsyncObjectFiles setting
1821 fsync_object_files = config.get_boolean((b"core",), b"fsyncObjectFiles", False)
1823 # Read bitmap settings
1824 pack_write_bitmaps = config.get_boolean((b"pack",), b"writeBitmaps", False)
1825 pack_write_bitmap_hash_cache = config.get_boolean(
1826 (b"pack",), b"writeBitmapHashCache", True
1827 )
1828 pack_write_bitmap_lookup_table = config.get_boolean(
1829 (b"pack",), b"writeBitmapLookupTable", True
1830 )
1831 # Also check repack.writeBitmaps for backwards compatibility
1832 if not pack_write_bitmaps:
1833 pack_write_bitmaps = config.get_boolean(
1834 (b"repack",), b"writeBitmaps", False
1835 )
1837 # Get hash algorithm from config
1838 from .object_format import get_object_format
1840 object_format = None
1841 try:
1842 try:
1843 version = int(config.get((b"core",), b"repositoryformatversion"))
1844 except KeyError:
1845 version = 0
1846 if version == 1:
1847 try:
1848 object_format_name = config.get((b"extensions",), b"objectformat")
1849 except KeyError:
1850 object_format_name = b"sha1"
1851 object_format = get_object_format(object_format_name.decode("ascii"))
1852 except (KeyError, ValueError):
1853 pass
1855 instance = cls(
1856 path,
1857 loose_compression_level=loose_compression_level,
1858 pack_compression_level=pack_compression_level,
1859 pack_index_version=pack_index_version,
1860 pack_delta_window_size=pack_delta_window_size,
1861 pack_window_memory=pack_window_memory,
1862 pack_delta_cache_size=pack_delta_cache_size,
1863 pack_depth=pack_depth,
1864 pack_threads=pack_threads,
1865 pack_big_file_threshold=pack_big_file_threshold,
1866 packed_git_limit=packed_git_limit,
1867 delta_base_cache_limit=delta_base_cache_limit,
1868 fsync_object_files=fsync_object_files,
1869 pack_write_bitmaps=pack_write_bitmaps,
1870 pack_write_bitmap_hash_cache=pack_write_bitmap_hash_cache,
1871 pack_write_bitmap_lookup_table=pack_write_bitmap_lookup_table,
1872 shared_perm=shared_perm,
1873 object_format=object_format,
1874 loose_object_size_limit=loose_object_size_limit,
1875 alternates=alternates,
1876 )
1877 instance._use_commit_graph = use_commit_graph
1878 instance._use_midx = use_midx
1879 return instance
1881 @property
1882 def alternates(self) -> list["BaseObjectStore"]:
1883 """Get the list of alternate object stores.
1885 Reads from .git/objects/info/alternates if not already cached.
1887 Returns:
1888 List of DiskObjectStore instances for alternate object directories
1889 """
1890 if self._alternates is not None:
1891 return self._alternates
1892 self._alternates = []
1893 for path in self._read_alternate_paths():
1894 self._alternates.append(DiskObjectStore(path))
1895 for path in self._extra_alternate_paths:
1896 self._alternates.append(DiskObjectStore(path))
1897 return self._alternates
1899 def _read_alternate_paths(self) -> Iterator[str]:
1900 try:
1901 f = GitFile(os.path.join(self.path, INFODIR, "alternates"), "rb")
1902 except FileNotFoundError:
1903 return
1904 with f:
1905 for line in f.readlines():
1906 line = line.rstrip(b"\n")
1907 if line.startswith(b"#"):
1908 continue
1909 if os.path.isabs(line):
1910 yield os.fsdecode(line)
1911 else:
1912 yield os.fsdecode(os.path.join(os.fsencode(self.path), line))
1914 def add_alternate_path(self, path: str | os.PathLike[str]) -> None:
1915 """Add an alternate path to this object store."""
1916 info_dir = os.path.join(self.path, INFODIR)
1917 try:
1918 os.mkdir(info_dir)
1919 adjust_shared_perm(info_dir, self.shared_perm)
1920 except FileExistsError:
1921 pass
1922 alternates_path = os.path.join(self.path, INFODIR, "alternates")
1923 with GitFile(alternates_path, "wb", shared_perm=self.shared_perm) as f:
1924 try:
1925 orig_f = open(alternates_path, "rb")
1926 except FileNotFoundError:
1927 pass
1928 else:
1929 with orig_f:
1930 f.write(orig_f.read())
1931 f.write(os.fsencode(path) + b"\n")
1933 if not os.path.isabs(path):
1934 path = os.path.join(self.path, path)
1935 self.alternates.append(DiskObjectStore(path))
1937 def _update_pack_cache(self) -> list[Pack]:
1938 """Read and iterate over new pack files and cache them."""
1939 try:
1940 pack_dir_contents = set(os.listdir(self.pack_dir))
1941 except FileNotFoundError:
1942 return []
1943 pack_files = set()
1944 for name in pack_dir_contents:
1945 # Index any ".pack" file with a matching ".idx", not just
1946 # "pack-<hash>". ``git maintenance`` writes packs named
1947 # "loose-<hash>.pack"; these are ordinary packs and Git indexes
1948 # any .pack file present. The matching ".idx" also confirms the
1949 # pack is fully written.
1950 if name.endswith(".pack"):
1951 basename = name[: -len(".pack")]
1952 if basename + ".idx" in pack_dir_contents:
1953 pack_files.add(basename)
1955 # Open newly appeared pack files
1956 new_packs = []
1957 for basename in pack_files:
1958 if basename not in self._pack_cache:
1959 pack = Pack(
1960 os.path.join(self.pack_dir, basename),
1961 object_format=self.object_format,
1962 delta_window_size=self.pack_delta_window_size,
1963 window_memory=self.pack_window_memory,
1964 delta_cache_size=self.pack_delta_cache_size,
1965 depth=self.pack_depth,
1966 threads=self.pack_threads,
1967 big_file_threshold=self.pack_big_file_threshold,
1968 delta_base_cache_limit=self.delta_base_cache_limit,
1969 )
1970 new_packs.append(pack)
1971 self._pack_cache[basename] = pack
1972 self._mark_pack_used(basename)
1973 # Remove disappeared pack files
1974 for f in set(self._pack_cache) - pack_files:
1975 self._pack_cache.pop(f).close()
1976 try:
1977 self._pack_access_order.remove(f)
1978 except ValueError:
1979 pass
1980 self._enforce_packed_git_limit()
1981 return new_packs
1983 def _get_shafile_path(self, sha: ObjectID) -> str:
1984 # Reject anything that is neither a valid hex object id nor a raw
1985 # binary oid before building a path. Otherwise a malformed id (e.g.
1986 # one containing path separators) would be joined into a filename
1987 # that escapes the objects directory.
1988 if not valid_hexsha(sha):
1989 raise ValueError(f"Invalid object id {sha!r}")
1990 # Check from object dir
1991 return hex_to_filename(os.fspath(self.path), sha)
1993 def _iter_loose_objects(self) -> Iterator[ObjectID]:
1994 for base in os.listdir(self.path):
1995 if len(base) != 2:
1996 continue
1997 for rest in os.listdir(os.path.join(self.path, base)):
1998 sha = os.fsencode(base + rest)
1999 if not valid_hexsha(sha):
2000 continue
2001 yield ObjectID(sha)
2003 def count_loose_objects(self) -> int:
2004 """Count the number of loose objects in the object store.
2006 Returns:
2007 Number of loose objects
2008 """
2009 # Calculate expected filename length for loose
2010 # objects (excluding directory)
2011 fn_length = self.object_format.hex_length - 2
2012 count = 0
2013 if not os.path.exists(self.path):
2014 return 0
2016 for i in range(256):
2017 subdir = os.path.join(self.path, f"{i:02x}")
2018 try:
2019 count += len(
2020 [name for name in os.listdir(subdir) if len(name) == fn_length]
2021 )
2022 except FileNotFoundError:
2023 # Directory may have been removed or is inaccessible
2024 continue
2026 return count
2028 def _get_loose_object(self, sha: ObjectID) -> ShaFile | None:
2029 try:
2030 # Load the object from path with SHA and hash algorithm from object store
2031 # Convert to hex ObjectID if needed
2032 if len(sha) == self.object_format.oid_length:
2033 hex_sha: ObjectID = sha_to_hex(RawObjectID(sha))
2034 else:
2035 hex_sha = ObjectID(sha)
2036 path = self._get_shafile_path(hex_sha)
2037 return ShaFile.from_path(
2038 path,
2039 hex_sha,
2040 object_format=self.object_format,
2041 max_size=self.loose_object_size_limit,
2042 )
2043 except FileNotFoundError:
2044 return None
2046 def delete_loose_object(self, sha: ObjectID) -> None:
2047 """Delete a loose object from disk.
2049 Args:
2050 sha: SHA1 of the object to delete
2052 Raises:
2053 FileNotFoundError: If the object file doesn't exist
2054 """
2055 _remove_readonly(self._get_shafile_path(sha))
2057 def get_object_mtime(self, sha: ObjectID) -> float:
2058 """Get the modification time of an object.
2060 Args:
2061 sha: SHA1 of the object
2063 Returns:
2064 Modification time as seconds since epoch
2066 Raises:
2067 KeyError: if the object is not found
2068 """
2069 # First check if it's a loose object
2070 if self.contains_loose(sha):
2071 path = self._get_shafile_path(sha)
2072 try:
2073 return os.path.getmtime(path)
2074 except FileNotFoundError:
2075 pass
2077 # Check if it's in a pack file
2078 for pack in self.packs:
2079 try:
2080 if sha in pack:
2081 # Use the pack file's mtime for packed objects
2082 pack_path = pack._data_path
2083 try:
2084 return os.path.getmtime(pack_path)
2085 except (FileNotFoundError, AttributeError):
2086 pass
2087 except PackFileDisappeared:
2088 pass
2090 raise KeyError(sha)
2092 def _remove_pack(self, pack: Pack) -> None:
2093 # _pack_cache is keyed by the full pack basename (e.g. "pack-<hash>"
2094 # or "loose-<hash>"), matching pack._basename.
2095 basename = os.path.basename(pack._basename)
2096 self._pack_cache.pop(basename, None)
2097 try:
2098 self._pack_access_order.remove(basename)
2099 except ValueError:
2100 pass
2101 # Store paths before closing to avoid re-opening files on Windows
2102 data_path = pack._data_path
2103 idx_path = pack._idx_path
2104 pack.close()
2105 _remove_readonly(data_path)
2106 if os.path.exists(idx_path):
2107 _remove_readonly(idx_path)
2109 def _get_pack_basepath(
2110 self, entries: Iterable[tuple[bytes, int, int | None]]
2111 ) -> str:
2112 suffix_bytes = iter_sha1(entry[0] for entry in entries)
2113 # TODO: Handle self.pack_dir being bytes
2114 suffix = suffix_bytes.decode("ascii")
2115 return os.path.join(self.pack_dir, "pack-" + suffix)
2117 def _index_pack(
2118 self,
2119 indexer: PackIndexer,
2120 num_objects: int,
2121 progress: Callable[..., None] | None = None,
2122 ) -> tuple[list[PackIndexEntry], set[RawObjectID]]:
2123 """Drain an indexer into index entries and the external refs it needs.
2125 Args:
2126 indexer: A PackIndexer over the pack being completed.
2127 num_objects: Number of objects in the pack, for progress reporting.
2128 progress: Optional progress reporting function.
2130 Returns: Tuple of (index entries, external refs). ext_refs() is only
2131 populated once the indexer has been drained.
2132 """
2133 entries = []
2134 for i, entry in enumerate(indexer):
2135 if progress is not None:
2136 progress(f"generating index: {i}/{num_objects}\r".encode("ascii"))
2137 entries.append(entry)
2138 return entries, set(indexer.ext_refs())
2140 def _complete_pack(
2141 self,
2142 f: BinaryIO,
2143 path: str,
2144 entries: list[PackIndexEntry],
2145 ext_refs: set[RawObjectID],
2146 progress: Callable[..., None] | None = None,
2147 refs: dict[Ref, ObjectID] | None = None,
2148 ) -> Pack:
2149 """Move a specific file containing a pack into the pack directory.
2151 Note: The file should be on the same file system as the
2152 packs directory.
2154 This takes ownership of ``f``: it appends any missing base objects,
2155 closes the file and renames it into place. Callers must have finished
2156 reading the pack (see :meth:`_index_pack`) before calling this; on
2157 Windows a mapping left over the file blocks both the write and the
2158 rename.
2160 Args:
2161 f: Open file object for the pack.
2162 path: Path to the pack file.
2163 entries: Index entries for the objects already in the pack.
2164 ext_refs: Objects the pack deltas against that it does not contain.
2165 progress: Optional progress reporting function.
2166 refs: Optional dictionary of refs for bitmap generation.
2167 """
2168 pack_sha, extra_entries = extend_pack(
2169 f,
2170 ext_refs,
2171 get_raw=self.get_raw,
2172 compression_level=self.pack_compression_level,
2173 progress=progress,
2174 object_format=self.object_format,
2175 )
2176 f.flush()
2177 if self.fsync_object_files:
2178 try:
2179 fileno = f.fileno()
2180 except AttributeError as e:
2181 raise OSError("fsync requested but file has no fileno()") from e
2182 else:
2183 os.fsync(fileno)
2184 f.close()
2186 entries.extend(extra_entries)
2188 # Move the pack in.
2189 entries.sort()
2190 pack_base_name = self._get_pack_basepath(entries)
2192 # A pack's identity is the SHA over its object SHAs, which is the
2193 # "<hash>" suffix _get_pack_basepath builds the name from. Compare by
2194 # that rather than by basename so an existing pack holding the same
2195 # objects under a different prefix (e.g. a "loose-<hash>" pack written
2196 # by git maintenance) is recognised and not duplicated.
2197 pack_name = os.path.basename(pack_base_name)[len("pack-") :].encode("ascii")
2198 for pack in self.packs:
2199 if pack.name() == pack_name:
2200 # The objects are already packed; drop the temporary pack we
2201 # were about to move in rather than leaking it into pack_dir.
2202 _remove_readonly(path)
2203 return pack
2205 target_pack_path = pack_base_name + ".pack"
2206 target_index_path = pack_base_name + ".idx"
2207 if sys.platform == "win32":
2208 # Windows might have the target pack file lingering. Attempt
2209 # removal, silently passing if the target does not exist.
2210 with suppress(FileNotFoundError):
2211 os.remove(target_pack_path)
2212 os.rename(path, target_pack_path)
2214 # Write the index.
2215 with GitFile(
2216 target_index_path,
2217 "wb",
2218 mask=PACK_MODE,
2219 fsync=self.fsync_object_files,
2220 shared_perm=self.shared_perm,
2221 ) as index_file:
2222 write_pack_index(
2223 index_file, entries, pack_sha, version=self.pack_index_version
2224 )
2226 # Generate bitmap if configured and refs are available
2227 if self.pack_write_bitmaps and refs:
2228 from .bitmap import generate_bitmap, write_bitmap
2229 from .pack import load_pack_index
2231 if progress:
2232 progress("Generating bitmap index\r".encode("ascii"))
2234 # Load the index we just wrote. load_pack_index keeps the file
2235 # open for the lifetime of the index (it mmaps it), so close it
2236 # once the bitmap is generated rather than leaving the .idx
2237 # mapped, which would lock it on Windows.
2238 with closing(
2239 load_pack_index(target_index_path, self.object_format)
2240 ) as pack_index:
2241 bitmap = generate_bitmap(
2242 pack_index=pack_index,
2243 object_store=self,
2244 refs=refs,
2245 pack_checksum=pack_sha,
2246 include_hash_cache=self.pack_write_bitmap_hash_cache,
2247 include_lookup_table=self.pack_write_bitmap_lookup_table,
2248 progress=lambda msg: (
2249 progress(msg.encode("ascii"))
2250 if progress and isinstance(msg, str)
2251 else None
2252 ),
2253 )
2255 # Write the bitmap
2256 target_bitmap_path = pack_base_name + ".bitmap"
2257 write_bitmap(target_bitmap_path, bitmap)
2259 if progress:
2260 progress("Bitmap index written\r".encode("ascii"))
2262 # Add the pack to the store and return it.
2263 final_pack = Pack(
2264 pack_base_name,
2265 object_format=self.object_format,
2266 delta_window_size=self.pack_delta_window_size,
2267 window_memory=self.pack_window_memory,
2268 delta_cache_size=self.pack_delta_cache_size,
2269 depth=self.pack_depth,
2270 threads=self.pack_threads,
2271 big_file_threshold=self.pack_big_file_threshold,
2272 delta_base_cache_limit=self.delta_base_cache_limit,
2273 )
2274 try:
2275 final_pack.check_length_and_checksum()
2276 # Materialise every object so payloads that fail to parse
2277 # (e.g. tree entries with garbage modes) are rejected rather
2278 # than silently landed on disk. MemoryObjectStore already
2279 # validates ingested objects this way via PackInflater; without
2280 # the same check DiskObjectStore was strictly weaker.
2281 for _obj in PackInflater.for_pack_data(
2282 final_pack.data, resolve_ext_ref=self.get_raw
2283 ):
2284 pass
2285 except BaseException:
2286 final_pack.close()
2287 with suppress(FileNotFoundError):
2288 os.remove(target_pack_path)
2289 with suppress(FileNotFoundError):
2290 os.remove(target_index_path)
2291 if self.pack_write_bitmaps and refs:
2292 with suppress(FileNotFoundError):
2293 os.remove(pack_base_name + ".bitmap")
2294 raise
2295 # _pack_cache is keyed by the full basename (/path/to/pack-HASH -> pack-HASH)
2296 self._add_cached_pack(os.path.basename(pack_base_name), final_pack)
2297 return final_pack
2299 def add_thin_pack(
2300 self,
2301 read_all: Callable[[int], bytes],
2302 read_some: Callable[[int], bytes] | None,
2303 progress: Callable[..., None] | None = None,
2304 *,
2305 max_input_size: int | None = None,
2306 ) -> "Pack":
2307 """Add a new thin pack to this object store.
2309 Thin packs are packs that contain deltas with parents that exist
2310 outside the pack. They should never be placed in the object store
2311 directly, and always indexed and completed as they are copied.
2313 Args:
2314 read_all: Read function that blocks until the number of
2315 requested bytes are read.
2316 read_some: Read function that returns at least one byte, but may
2317 not return the number of bytes requested.
2318 progress: Optional progress reporting function.
2319 max_input_size: Maximum number of bytes that may be read from
2320 the wire while ingesting this pack. Matches git's
2321 ``receive.maxInputSize`` / ``index-pack --max-input-size``
2322 semantics: ``None`` (the default) or ``0`` mean unlimited.
2323 Exceeding the cap raises ``PackInputTooLarge``.
2324 Returns: A Pack object pointing at the now-completed thin pack in the
2325 objects/pack directory.
2326 """
2327 import tempfile
2329 if max_input_size:
2330 read_all, read_some = _bound_read_callables(
2331 read_all, read_some, max_input_size
2332 )
2334 fd, path = tempfile.mkstemp(dir=self.path, prefix="tmp_pack_")
2335 with os.fdopen(fd, "w+b") as f:
2336 os.chmod(path, PACK_MODE)
2337 indexer = PackIndexer(
2338 f,
2339 self.object_format.hash_func,
2340 resolve_ext_ref=self.get_raw,
2341 )
2342 copier = PackStreamCopier(
2343 self.object_format.hash_func,
2344 read_all,
2345 read_some,
2346 f,
2347 delta_iter=indexer, # type: ignore[arg-type]
2348 )
2349 copier.verify(progress=progress)
2350 entries, ext_refs = self._index_pack(
2351 indexer, len(copier), progress=progress
2352 )
2353 return self._complete_pack(f, path, entries, ext_refs, progress=progress)
2355 def add_pack(
2356 self,
2357 ) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2358 """Add a new pack to this object store.
2360 Returns: Fileobject to write to, a commit function to
2361 call when the pack is finished and an abort
2362 function.
2363 """
2364 import tempfile
2366 fd, path = tempfile.mkstemp(dir=self.pack_dir, suffix=".pack")
2367 f = os.fdopen(fd, "w+b")
2368 os.chmod(path, PACK_MODE)
2369 adjust_shared_perm(path, self.shared_perm)
2371 def commit() -> "Pack | None":
2372 if f.tell() > 0:
2373 f.seek(0)
2375 # Scope the mapping to indexing: _complete_pack writes to and
2376 # renames this same file, which a live mapping blocks on
2377 # Windows. PackData.close() leaves f open for it to finish.
2378 with PackData(path, file=f, object_format=self.object_format) as pd:
2379 indexer = PackIndexer.for_pack_data(
2380 pd,
2381 resolve_ext_ref=self.get_raw,
2382 )
2383 entries, ext_refs = self._index_pack(indexer, len(pd)) # type: ignore[arg-type]
2384 return self._complete_pack(f, path, entries, ext_refs)
2385 else:
2386 f.close()
2387 os.remove(path)
2388 return None
2390 def abort() -> None:
2391 f.close()
2392 os.remove(path)
2394 return f, commit, abort # type: ignore[return-value]
2396 def add_object(self, obj: ShaFile) -> None:
2397 """Add a single object to this object store.
2399 Args:
2400 obj: Object to add
2401 """
2402 # Use the correct hash algorithm for the object ID
2403 obj_id = ObjectID(obj.get_id(self.object_format))
2404 path = self._get_shafile_path(obj_id)
2405 dir = os.path.dirname(path)
2406 try:
2407 os.mkdir(dir)
2408 adjust_shared_perm(dir, self.shared_perm)
2409 except FileExistsError:
2410 pass
2411 try:
2412 # Refresh the mtime instead of just checking for existence. A
2413 # loose object with a stale mtime is a candidate for age-based
2414 # pruning, so a concurrent "git gc" could remove it before the
2415 # caller has created a reference to it.
2416 os.utime(path, None)
2417 except FileNotFoundError:
2418 pass # Not there after all, write it out below.
2419 except PermissionError:
2420 # Owned by another user in a shared repository. The mtime stays
2421 # stale, so write the object out to give it a fresh one.
2422 pass
2423 else:
2424 return # Already there and freshened, no need to write again
2425 with GitFile(
2426 path,
2427 "wb",
2428 mask=PACK_MODE,
2429 fsync=self.fsync_object_files,
2430 shared_perm=self.shared_perm,
2431 ) as f:
2432 f.write(
2433 obj.as_legacy_object(compression_level=self.loose_compression_level)
2434 )
2436 @classmethod
2437 def init(
2438 cls,
2439 path: str | os.PathLike[str],
2440 *,
2441 shared_perm: "SharedPerm | None" = None,
2442 object_format: "ObjectFormat | None" = None,
2443 ) -> "DiskObjectStore":
2444 """Initialize a new disk object store.
2446 Creates the necessary directory structure for a Git object store.
2448 Args:
2449 path: Path where the object store should be created
2450 shared_perm: Optional shared repository permission setting
2451 object_format: Hash algorithm to use (SHA1 or SHA256)
2453 Returns:
2454 New DiskObjectStore instance
2455 """
2456 try:
2457 os.mkdir(path)
2458 adjust_shared_perm(path, shared_perm)
2459 except FileExistsError:
2460 pass
2461 info_path = os.path.join(path, "info")
2462 pack_path = os.path.join(path, PACKDIR)
2463 os.mkdir(info_path)
2464 os.mkdir(pack_path)
2465 adjust_shared_perm(info_path, shared_perm)
2466 adjust_shared_perm(pack_path, shared_perm)
2467 return cls(path, shared_perm=shared_perm, object_format=object_format)
2469 def iter_prefix(self, prefix: bytes) -> Iterator[ObjectID]:
2470 """Iterate over all object SHAs with the given prefix.
2472 Args:
2473 prefix: Hex prefix to search for (as bytes)
2475 Returns:
2476 Iterator of object SHAs (as ObjectID) matching the prefix
2477 """
2478 if len(prefix) < 2:
2479 yield from super().iter_prefix(prefix)
2480 return
2481 seen = set()
2482 dir = prefix[:2].decode()
2483 rest = prefix[2:].decode()
2484 try:
2485 for name in os.listdir(os.path.join(self.path, dir)):
2486 if name.startswith(rest):
2487 sha = ObjectID(os.fsencode(dir + name))
2488 if sha not in seen:
2489 seen.add(sha)
2490 yield sha
2491 except FileNotFoundError:
2492 pass
2494 for p in self.packs:
2495 bin_prefix = (
2496 binascii.unhexlify(prefix)
2497 if len(prefix) % 2 == 0
2498 else binascii.unhexlify(prefix[:-1])
2499 )
2500 for bin_sha in p.index.iter_prefix(bin_prefix):
2501 sha = sha_to_hex(bin_sha)
2502 if sha.startswith(prefix) and sha not in seen:
2503 seen.add(sha)
2504 yield sha
2505 for alternate in self.alternates:
2506 for sha in alternate.iter_prefix(prefix):
2507 if sha not in seen:
2508 seen.add(sha)
2509 yield sha
2511 def get_commit_graph(self) -> "CommitGraph | None":
2512 """Get the commit graph for this object store.
2514 Returns:
2515 CommitGraph object if available, None otherwise
2516 """
2517 if not self._use_commit_graph:
2518 return None
2520 if self._commit_graph is None:
2521 from .commit_graph import read_commit_graph
2523 # Look for commit graph in our objects directory
2524 graph_file = os.path.join(self.path, "info", "commit-graph")
2525 if os.path.exists(graph_file):
2526 self._commit_graph = read_commit_graph(graph_file)
2527 return self._commit_graph
2529 def get_midx(self) -> MultiPackIndex | None:
2530 """Get the multi-pack-index for this object store.
2532 Returns:
2533 MultiPackIndex object if available, None otherwise
2535 Raises:
2536 ValueError: If MIDX file is corrupt
2537 OSError: If MIDX file cannot be read
2538 """
2539 if not self._use_midx:
2540 return None
2542 if self._midx is None:
2543 # Look for MIDX in pack directory
2544 midx_file = os.path.join(self.pack_dir, "multi-pack-index")
2545 if os.path.exists(midx_file):
2546 self._midx = load_midx(midx_file)
2547 return self._midx
2549 def _get_pack_by_name(self, pack_name: str) -> Pack:
2550 """Get a pack referenced by a multi-pack-index entry.
2552 Args:
2553 pack_name: Pack index file name as stored in the MIDX. Usually
2554 ``pack-<hash>.idx``, but ``git maintenance`` writes packs
2555 named ``loose-<hash>.idx``, so any ``.idx`` basename is
2556 accepted.
2558 Returns:
2559 Pack object
2561 Raises:
2562 KeyError: If pack doesn't exist
2563 """
2564 if not pack_name.endswith(".idx"):
2565 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2566 # The name is joined under pack_dir below, so reject any path
2567 # separators that a corrupt or hostile MIDX could use to traverse
2568 # directories (backslash matters on Windows).
2569 if "/" in pack_name or "\\" in pack_name:
2570 raise KeyError(f"unexpected MIDX pack name {pack_name!r}")
2571 basename = pack_name[: -len(".idx")]
2573 # _pack_cache is keyed by full basename and _update_pack_cache
2574 # discovers every "<name>-<hash>.pack" file (including the
2575 # "loose-<hash>" packs that git maintenance writes and the MIDX
2576 # references), so a referenced pack is normally already cached.
2577 try:
2578 return self._pack_cache[basename]
2579 except KeyError:
2580 pass
2582 pack_path = os.path.join(self.pack_dir, basename)
2583 if not os.path.exists(pack_path + ".pack"):
2584 raise KeyError(f"Pack {pack_name} not found")
2586 pack = Pack(
2587 pack_path,
2588 object_format=self.object_format,
2589 delta_window_size=self.pack_delta_window_size,
2590 window_memory=self.pack_window_memory,
2591 delta_cache_size=self.pack_delta_cache_size,
2592 depth=self.pack_depth,
2593 threads=self.pack_threads,
2594 big_file_threshold=self.pack_big_file_threshold,
2595 delta_base_cache_limit=self.delta_base_cache_limit,
2596 )
2597 self._pack_cache[basename] = pack
2598 self._mark_pack_used(basename)
2599 return pack
2601 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2602 """Check if a particular object is present by SHA1 and is packed.
2604 This checks the MIDX first if available, then falls back to checking
2605 individual pack indexes.
2607 Args:
2608 sha: SHA of the object (20/32 bytes binary or 40/64 bytes hex)
2610 Returns:
2611 True if the object is in a pack file
2612 """
2613 # Normalise to binary once: MIDX requires it, and passing binary to
2614 # the per-pack fallback avoids N redundant hex->binary conversions
2615 # inside PackIndex.object_offset. Mirrors ``get_raw`` below.
2616 if len(sha) == self.object_format.hex_length:
2617 sha = hex_to_sha(cast(ObjectID, sha))
2619 midx = self.get_midx()
2620 if midx is not None and sha in midx:
2621 return True
2623 # Fall back to checking individual packs
2624 return super().contains_packed(sha)
2626 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2627 """Obtain the raw fulltext for an object.
2629 This uses the MIDX if available for faster lookups.
2631 Args:
2632 name: SHA for the object (20 bytes binary or 40 bytes hex)
2634 Returns:
2635 Tuple with numeric type and object contents
2637 Raises:
2638 KeyError: If object not found
2639 """
2640 sha: RawObjectID
2641 if len(name) in (40, 64):
2642 # name is ObjectID (hex), convert to RawObjectID
2643 # Support both SHA1 (40) and SHA256 (64)
2644 sha = hex_to_sha(cast(ObjectID, name))
2645 elif len(name) in (20, 32):
2646 # name is already RawObjectID (binary)
2647 # Support both SHA1 (20) and SHA256 (32)
2648 sha = RawObjectID(name)
2649 else:
2650 raise AssertionError(f"Invalid object name {name!r}")
2652 # Try MIDX first for faster lookup
2653 midx = self.get_midx()
2654 if midx is not None:
2655 result = midx.object_offset(sha)
2656 if result is not None:
2657 pack_name, _offset = result
2658 try:
2659 pack = self._get_pack_by_name(pack_name)
2660 return pack.get_raw(sha)
2661 except (KeyError, PackFileDisappeared):
2662 # Pack disappeared or object not found, fall through to standard lookup
2663 pass
2665 # Fall back to the standard implementation
2666 return super().get_raw(name)
2668 def write_midx(self) -> bytes:
2669 """Write a multi-pack-index file for this object store.
2671 Creates a MIDX file that indexes all pack files in the pack directory.
2673 Returns:
2674 SHA-1 checksum of the written MIDX file
2676 Raises:
2677 OSError: If the pack directory doesn't exist or MIDX can't be written
2678 """
2679 from .midx import write_midx_file
2681 midx_path = os.path.join(self.pack_dir, "multi-pack-index")
2682 # Skip packs that vanish mid-collection (e.g. concurrent
2683 # ``git repack``); the survivors still produce a valid MIDX.
2684 pack_entries: list[tuple[str, list[tuple[RawObjectID, int, int | None]]]] = []
2685 for pack in self.packs:
2686 try:
2687 entries = list(pack.index.iterentries())
2688 except PackFileDisappeared as exc:
2689 self._evict_pack(exc.obj)
2690 continue
2691 pack_entries.append((os.path.basename(pack._basename) + ".idx", entries))
2692 if not pack_entries:
2693 return b"\x00" * 20
2694 return write_midx_file(midx_path, pack_entries)
2696 def write_commit_graph(
2697 self, refs: Iterable[ObjectID] | None = None, reachable: bool = True
2698 ) -> None:
2699 """Write a commit graph file for this object store.
2701 Args:
2702 refs: List of refs to include. If None, includes all refs from object store.
2703 reachable: If True, includes all commits reachable from refs.
2704 If False, only includes the direct ref targets.
2705 """
2706 from .commit_graph import get_reachable_commits
2708 if refs is None:
2709 # Get all commit objects from the object store
2710 all_refs = []
2711 # Iterate through all objects to find commits
2712 for sha in self:
2713 try:
2714 obj = self[sha]
2715 if obj.type_name == b"commit":
2716 all_refs.append(sha)
2717 except KeyError:
2718 continue
2719 else:
2720 # Use provided refs
2721 all_refs = list(refs)
2723 if not all_refs:
2724 return # No commits to include
2726 if reachable:
2727 # Get all reachable commits
2728 commit_ids = get_reachable_commits(self, all_refs)
2729 else:
2730 # Just use the direct ref targets - ensure they're hex ObjectIDs
2731 commit_ids = []
2732 for ref in all_refs:
2733 if isinstance(ref, bytes) and len(ref) == self.object_format.hex_length:
2734 # Already hex ObjectID
2735 commit_ids.append(ref)
2736 elif (
2737 isinstance(ref, bytes) and len(ref) == self.object_format.oid_length
2738 ):
2739 # Binary SHA, convert to hex ObjectID
2740 commit_ids.append(sha_to_hex(RawObjectID(ref)))
2741 else:
2742 # Assume it's already correct format
2743 commit_ids.append(ref)
2745 if commit_ids:
2746 # Write commit graph directly to our object store path
2747 # Generate the commit graph
2748 from .commit_graph import generate_commit_graph
2750 graph = generate_commit_graph(self, commit_ids)
2752 if graph.entries:
2753 # Ensure the info directory exists
2754 info_dir = os.path.join(self.path, "info")
2755 os.makedirs(info_dir, exist_ok=True)
2756 adjust_shared_perm(info_dir, self.shared_perm)
2758 # Write using GitFile for atomic operation
2759 graph_path = os.path.join(info_dir, "commit-graph")
2760 with GitFile(graph_path, "wb", shared_perm=self.shared_perm) as f:
2761 assert isinstance(
2762 f, _GitFile
2763 ) # GitFile in write mode always returns _GitFile
2764 graph.write_to_file(f)
2766 # Clear cached commit graph so it gets reloaded
2767 self._commit_graph = None
2769 def prune(self, grace_period: int | None = None) -> None:
2770 """Prune/clean up this object store.
2772 This removes temporary files that were left behind by interrupted
2773 pack operations. These are files that start with ``tmp_pack_`` in the
2774 repository directory or files with .pack extension but no corresponding
2775 .idx file in the pack directory.
2777 Args:
2778 grace_period: Grace period in seconds for removing temporary files.
2779 If None, uses DEFAULT_TEMPFILE_GRACE_PERIOD.
2780 """
2781 import glob
2783 if grace_period is None:
2784 grace_period = DEFAULT_TEMPFILE_GRACE_PERIOD
2786 # Clean up tmp_pack_* files in the repository directory
2787 for tmp_file in glob.glob(os.path.join(self.path, "tmp_pack_*")):
2788 # Check if file is old enough (more than grace period)
2789 mtime = os.path.getmtime(tmp_file)
2790 if time.time() - mtime > grace_period:
2791 os.remove(tmp_file)
2793 # Clean up orphaned .pack files without corresponding .idx files
2794 try:
2795 pack_dir_contents = os.listdir(self.pack_dir)
2796 except FileNotFoundError:
2797 return
2799 pack_files = {}
2800 idx_files = set()
2802 for name in pack_dir_contents:
2803 if name.endswith(".pack"):
2804 base_name = name[:-5] # Remove .pack extension
2805 pack_files[base_name] = name
2806 elif name.endswith(".idx"):
2807 base_name = name[:-4] # Remove .idx extension
2808 idx_files.add(base_name)
2810 # Remove .pack files without corresponding .idx files
2811 for base_name, pack_name in pack_files.items():
2812 if base_name not in idx_files:
2813 pack_path = os.path.join(self.pack_dir, pack_name)
2814 # Check if file is old enough (more than grace period)
2815 mtime = os.path.getmtime(pack_path)
2816 if time.time() - mtime > grace_period:
2817 os.remove(pack_path)
2819 def close(self) -> None:
2820 """Close the object store and release resources.
2822 This method closes all cached pack files, MIDX, and frees associated resources.
2823 Can be called multiple times safely.
2824 """
2825 # Close MIDX if it's loaded
2826 if self._midx is not None:
2827 self._midx.close()
2828 self._midx = None
2830 # Close alternates
2831 if self._alternates is not None:
2832 for alt in self._alternates:
2833 alt.close()
2834 self._alternates = None
2836 # Call parent class close to handle pack files
2837 super().close()
2840class MemoryObjectStore(PackCapableObjectStore):
2841 """Object store that keeps all objects in memory."""
2843 def __init__(self, *, object_format: "ObjectFormat | None" = None) -> None:
2844 """Initialize a MemoryObjectStore.
2846 Creates an empty in-memory object store.
2848 Args:
2849 object_format: Hash algorithm to use (defaults to SHA1)
2850 """
2851 super().__init__(object_format=object_format)
2852 self._data: dict[ObjectID, ShaFile] = {}
2853 self.pack_compression_level = -1
2855 def _to_hexsha(self, sha: ObjectID | RawObjectID) -> ObjectID:
2856 if len(sha) == self.object_format.hex_length:
2857 return cast(ObjectID, sha)
2858 elif len(sha) == self.object_format.oid_length:
2859 return sha_to_hex(cast(RawObjectID, sha))
2860 else:
2861 raise ValueError(f"Invalid sha {sha!r}")
2863 def contains_loose(self, sha: ObjectID) -> bool:
2864 """Check if a particular object is present by SHA1 and is loose."""
2865 return self._to_hexsha(sha) in self._data
2867 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
2868 """Check if a particular object is present by SHA1 and is packed."""
2869 return False
2871 def __iter__(self) -> Iterator[ObjectID]:
2872 """Iterate over the SHAs that are present in this store."""
2873 return iter(self._data.keys())
2875 @property
2876 def packs(self) -> list[Pack]:
2877 """List with pack objects."""
2878 return []
2880 def get_raw(self, name: RawObjectID | ObjectID) -> tuple[int, bytes]:
2881 """Obtain the raw text for an object.
2883 Args:
2884 name: sha for the object.
2885 Returns: tuple with numeric type and object contents.
2886 """
2887 obj = self[self._to_hexsha(name)]
2888 return obj.type_num, obj.as_raw_string()
2890 def __getitem__(self, name: ObjectID | RawObjectID) -> ShaFile:
2891 """Retrieve an object by SHA.
2893 Args:
2894 name: SHA of the object (as hex string or bytes)
2896 Returns:
2897 Copy of the ShaFile object
2899 Raises:
2900 KeyError: If the object is not found
2901 """
2902 return self._data[self._to_hexsha(name)].copy()
2904 def __delitem__(self, name: ObjectID) -> None:
2905 """Delete an object from this store, for testing only."""
2906 del self._data[self._to_hexsha(name)]
2908 def add_object(self, obj: ShaFile) -> None:
2909 """Add a single object to this object store."""
2910 self._data[obj.id] = obj.copy()
2912 def add_objects(
2913 self,
2914 objects: Iterable[tuple[ShaFile, str | None]],
2915 progress: Callable[[str], None] | None = None,
2916 ) -> None:
2917 """Add a set of objects to this object store.
2919 Args:
2920 objects: Iterable over a list of (object, path) tuples
2921 progress: Optional progress reporting function.
2922 """
2923 for obj, path in objects:
2924 self.add_object(obj)
2926 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
2927 """Add a new pack to this object store.
2929 Because this object store doesn't support packs, we extract and add the
2930 individual objects.
2932 Returns: Fileobject to write to and a commit function to
2933 call when the pack is finished.
2934 """
2935 from tempfile import SpooledTemporaryFile
2937 f = SpooledTemporaryFile(max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-")
2939 def commit() -> None:
2940 size = f.tell()
2941 if size > 0:
2942 f.seek(0)
2944 p = PackData.from_file(f, self.object_format, size)
2945 try:
2946 # Verify the trailing pack checksum before extracting
2947 # objects. Without this, a fetch that delivered a
2948 # truncated pack would still be accepted: ``add_pack``
2949 # iterates objects by offset and never reaches the
2950 # trailing bytes, so a stream that lost the last few
2951 # bytes of its trailer slipped through silently.
2952 # ``add_thin_pack`` already validates via
2953 # ``PackStreamCopier.verify``; do the equivalent here.
2954 p.check()
2955 for obj in PackInflater.for_pack_data(p, self.get_raw):
2956 self.add_object(obj)
2957 finally:
2958 p.close()
2959 f.close()
2960 else:
2961 f.close()
2963 def abort() -> None:
2964 f.close()
2966 return f, commit, abort # type: ignore[return-value]
2968 def add_pack_data(
2969 self,
2970 count: int,
2971 unpacked_objects: Iterator[UnpackedObject],
2972 progress: Callable[[str], None] | None = None,
2973 ) -> None:
2974 """Add pack data to this object store.
2976 Args:
2977 count: Number of items to add
2978 unpacked_objects: Iterator of UnpackedObject instances
2979 progress: Optional progress reporting function.
2980 """
2981 if count == 0:
2982 return
2984 # Since MemoryObjectStore doesn't support pack files, we need to
2985 # extract individual objects. To handle deltas properly, we write
2986 # to a temporary pack and then use PackInflater to resolve them.
2987 f, commit, abort = self.add_pack()
2988 try:
2989 write_pack_data(
2990 f.write,
2991 unpacked_objects,
2992 num_records=count,
2993 progress=progress,
2994 object_format=self.object_format,
2995 )
2996 except BaseException:
2997 abort()
2998 raise
2999 else:
3000 commit()
3002 def add_thin_pack(
3003 self,
3004 read_all: Callable[[int], bytes],
3005 read_some: Callable[[int], bytes] | None,
3006 progress: Callable[[str], None] | None = None,
3007 ) -> None:
3008 """Add a new thin pack to this object store.
3010 Thin packs are packs that contain deltas with parents that exist
3011 outside the pack. Because this object store doesn't support packs, we
3012 extract and add the individual objects.
3014 Args:
3015 read_all: Read function that blocks until the number of
3016 requested bytes are read.
3017 read_some: Read function that returns at least one byte, but may
3018 not return the number of bytes requested.
3019 progress: Optional progress reporting function.
3020 """
3021 f, commit, abort = self.add_pack()
3022 try:
3023 copier = PackStreamCopier(
3024 self.object_format.hash_func,
3025 read_all,
3026 read_some,
3027 f,
3028 )
3029 copier.verify()
3030 except BaseException:
3031 abort()
3032 raise
3033 else:
3034 commit()
3037class ObjectIterator(Protocol):
3038 """Interface for iterating over objects."""
3040 def iterobjects(self) -> Iterator[ShaFile]:
3041 """Iterate over all objects.
3043 Returns:
3044 Iterator of ShaFile objects
3045 """
3046 raise NotImplementedError(self.iterobjects)
3049def tree_lookup_path(
3050 lookup_obj: Callable[[ObjectID | RawObjectID], ShaFile],
3051 root_sha: ObjectID | RawObjectID,
3052 path: bytes,
3053) -> tuple[int, ObjectID]:
3054 """Look up an object in a Git tree.
3056 Args:
3057 lookup_obj: Callback for retrieving object by SHA1
3058 root_sha: SHA1 of the root tree
3059 path: Path to lookup
3060 Returns: A tuple of (mode, SHA) of the resulting path.
3061 """
3062 tree = lookup_obj(root_sha)
3063 if not isinstance(tree, Tree):
3064 raise NotTreeError(root_sha)
3065 return tree.lookup_path(lookup_obj, path)
3068def _collect_filetree_revs(
3069 obj_store: ObjectContainer, tree_sha: ObjectID, kset: set[ObjectID]
3070) -> None:
3071 """Collect SHA1s of files and directories for specified tree.
3073 Args:
3074 obj_store: Object store to get objects by SHA from
3075 tree_sha: tree reference to walk
3076 kset: set to fill with references to files and directories
3077 """
3078 filetree = obj_store[tree_sha]
3079 assert isinstance(filetree, Tree)
3080 for name, mode, sha in filetree.iteritems():
3081 assert mode is not None
3082 assert sha is not None
3083 if not S_ISGITLINK(mode) and sha not in kset:
3084 kset.add(sha)
3085 if stat.S_ISDIR(mode):
3086 _collect_filetree_revs(obj_store, sha, kset)
3089def _split_commits_and_tags(
3090 obj_store: ObjectContainer,
3091 lst: Iterable[ObjectID],
3092 *,
3093 unknown: str = "error",
3094) -> tuple[set[ObjectID], set[ObjectID], set[ObjectID]]:
3095 """Split object id list into three lists with commit, tag, and other SHAs.
3097 Commits referenced by tags are included into commits
3098 list as well. Only SHA1s known in this repository will get
3099 through, controlled by the unknown parameter.
3101 Args:
3102 obj_store: Object store to get objects by SHA1 from
3103 lst: Collection of commit and tag SHAs
3104 unknown: How to handle unknown objects: "error", "warn", or "ignore"
3105 Returns: A tuple of (commits, tags, others) SHA1s
3106 """
3107 if unknown not in ("error", "warn", "ignore"):
3108 raise ValueError(
3109 f"unknown must be 'error', 'warn', or 'ignore', got {unknown!r}"
3110 )
3112 commits: set[ObjectID] = set()
3113 tags: set[ObjectID] = set()
3114 others: set[ObjectID] = set()
3115 for e in lst:
3116 try:
3117 o = obj_store[e]
3118 except KeyError:
3119 if unknown == "error":
3120 raise
3121 elif unknown == "warn":
3122 logger.warning("Object %s not found in object store", e.decode("ascii"))
3123 # else: ignore
3124 else:
3125 if isinstance(o, Commit):
3126 commits.add(e)
3127 elif isinstance(o, Tag):
3128 tags.add(e)
3129 tagged = o.object[1]
3130 c, t, os = _split_commits_and_tags(obj_store, [tagged], unknown=unknown)
3131 commits |= c
3132 tags |= t
3133 others |= os
3134 else:
3135 others.add(e)
3136 return (commits, tags, others)
3139class MissingObjectFinder:
3140 """Find the objects missing from another object store.
3142 Args:
3143 object_store: Object store containing at least all objects to be
3144 sent
3145 haves: SHA1s of commits not to send (already present in target)
3146 wants: SHA1s of commits to send
3147 progress: Optional function to report progress to.
3148 get_tagged: Function that returns a dict of pointed-to sha -> tag
3149 sha for including tags.
3150 get_parents: Optional function for getting the parents of a commit.
3151 """
3153 def __init__(
3154 self,
3155 object_store: BaseObjectStore,
3156 haves: Iterable[ObjectID],
3157 wants: Iterable[ObjectID],
3158 *,
3159 shallow: Set[ObjectID] | None = None,
3160 progress: Callable[[bytes], None] | None = None,
3161 get_tagged: Callable[[], dict[ObjectID, ObjectID]] | None = None,
3162 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3163 ) -> None:
3164 """Initialize a MissingObjectFinder.
3166 Args:
3167 object_store: Object store containing objects
3168 haves: SHA1s of objects already present in target
3169 wants: SHA1s of objects to send
3170 shallow: Set of shallow commit SHA1s
3171 progress: Optional progress reporting callback
3172 get_tagged: Function returning dict of pointed-to sha -> tag sha
3173 get_parents: Function for getting commit parents
3174 """
3175 self.object_store = object_store
3176 if shallow is None:
3177 shallow = set()
3178 self._get_parents = get_parents
3179 reachability = object_store.get_reachability_provider()
3180 # process Commits and Tags differently
3181 # haves may list commits/tags not available locally (silently ignore them).
3182 # wants should only contain valid SHAs (fail fast if not).
3183 have_commits, have_tags, have_others = _split_commits_and_tags(
3184 object_store, haves, unknown="ignore"
3185 )
3186 want_commits, want_tags, want_others = _split_commits_and_tags(
3187 object_store, wants, unknown="error"
3188 )
3189 # all_ancestors is a set of commits that shall not be sent
3190 # (complete repository up to 'haves')
3191 all_ancestors = reachability.get_reachable_commits(
3192 have_commits, exclude=None, shallow=shallow
3193 )
3194 # all_missing - complete set of commits between haves and wants
3195 # common_commits - boundary commits directly encountered when traversing wants
3196 # We use _collect_ancestors here because we need the exact boundary behavior:
3197 # commits that are in all_ancestors and directly reachable from wants,
3198 # but we don't traverse past them. This is hard to express with the
3199 # reachability abstraction alone.
3200 missing_commits, common_commits = _collect_ancestors(
3201 object_store,
3202 want_commits,
3203 frozenset(all_ancestors),
3204 shallow=frozenset(shallow),
3205 get_parents=self._get_parents,
3206 )
3208 self.remote_has: set[ObjectID] = set()
3209 # Now, fill sha_done with commits and revisions of
3210 # files and directories known to be both locally
3211 # and on target. Thus these commits and files
3212 # won't get selected for fetch
3213 for h in common_commits:
3214 self.remote_has.add(h)
3215 cmt = object_store[h]
3216 assert isinstance(cmt, Commit)
3217 # Get tree objects for this commit
3218 tree_objects = reachability.get_tree_objects([cmt.tree])
3219 self.remote_has.update(tree_objects)
3221 # record tags we have as visited, too
3222 for t in have_tags:
3223 self.remote_has.add(t)
3224 self.sha_done = set(self.remote_has)
3226 # in fact, what we 'want' is commits, tags, and others
3227 # we've found missing
3228 self.objects_to_send: set[tuple[ObjectID, bytes | None, int | None, bool]] = {
3229 (w, None, Commit.type_num, False) for w in missing_commits
3230 }
3231 missing_tags = want_tags.difference(have_tags)
3232 self.objects_to_send.update(
3233 {(w, None, Tag.type_num, False) for w in missing_tags}
3234 )
3235 missing_others = want_others.difference(have_others)
3236 self.objects_to_send.update({(w, None, None, False) for w in missing_others})
3238 if progress is None:
3239 self.progress: Callable[[bytes], None] = lambda x: None
3240 else:
3241 self.progress = progress
3242 self._tagged = (get_tagged and get_tagged()) or {}
3244 def get_remote_has(self) -> set[ObjectID]:
3245 """Get the set of SHAs the remote has.
3247 Returns:
3248 Set of SHA1s that the remote side already has
3249 """
3250 return self.remote_has
3252 def add_todo(
3253 self, entries: Iterable[tuple[ObjectID, bytes | None, int | None, bool]]
3254 ) -> None:
3255 """Add objects to the todo list.
3257 Args:
3258 entries: Iterable of tuples (sha, name, type_num, is_leaf)
3259 """
3260 self.objects_to_send.update([e for e in entries if e[0] not in self.sha_done])
3262 def __next__(self) -> tuple[ObjectID, PackHint | None]:
3263 """Get the next object to send.
3265 Returns:
3266 Tuple of (sha, pack_hint)
3268 Raises:
3269 StopIteration: When no more objects to send
3270 """
3271 while True:
3272 if not self.objects_to_send:
3273 self.progress(
3274 f"counting objects: {len(self.sha_done)}, done.\n".encode("ascii")
3275 )
3276 raise StopIteration
3277 (sha, name, type_num, leaf) = self.objects_to_send.pop()
3278 if sha not in self.sha_done:
3279 break
3280 if not leaf:
3281 o = self.object_store[sha]
3282 if isinstance(o, Commit):
3283 self.add_todo([(o.tree, b"", Tree.type_num, False)])
3284 elif isinstance(o, Tree):
3285 todos = []
3286 for n, m, s in o.iteritems():
3287 assert m is not None
3288 assert n is not None
3289 assert s is not None
3290 if not S_ISGITLINK(m):
3291 todos.append(
3292 (
3293 s,
3294 n,
3295 (Blob.type_num if stat.S_ISREG(m) else Tree.type_num),
3296 not stat.S_ISDIR(m),
3297 )
3298 )
3299 self.add_todo(todos)
3300 elif isinstance(o, Tag):
3301 self.add_todo([(o.object[1], None, o.object[0].type_num, False)])
3302 if sha in self._tagged:
3303 self.add_todo([(self._tagged[sha], None, None, True)])
3304 self.sha_done.add(sha)
3305 if len(self.sha_done) % 1000 == 0:
3306 self.progress(f"counting objects: {len(self.sha_done)}\r".encode("ascii"))
3307 if type_num is None:
3308 pack_hint = None
3309 else:
3310 pack_hint = (type_num, name)
3311 return (sha, pack_hint)
3313 def __iter__(self) -> Iterator[tuple[ObjectID, PackHint | None]]:
3314 """Return iterator over objects to send.
3316 Returns:
3317 Self (this class implements the iterator protocol)
3318 """
3319 return self
3322class ObjectStoreGraphWalker:
3323 """Graph walker that finds what commits are missing from an object store."""
3325 heads: set[ObjectID]
3326 """Revisions without descendants in the local repo."""
3328 get_parents: Callable[[ObjectID], list[ObjectID]]
3329 """Function to retrieve parents in the local repo."""
3331 shallow: set[ObjectID]
3333 def __init__(
3334 self,
3335 local_heads: Iterable[ObjectID],
3336 get_parents: Callable[[ObjectID], list[ObjectID]],
3337 shallow: set[ObjectID] | None = None,
3338 update_shallow: Callable[[set[ObjectID] | None, set[ObjectID] | None], None]
3339 | None = None,
3340 ) -> None:
3341 """Create a new instance.
3343 Args:
3344 local_heads: Heads to start search with
3345 get_parents: Function for finding the parents of a SHA1.
3346 shallow: Set of shallow commits.
3347 update_shallow: Function to update shallow commits.
3348 """
3349 self.heads = set(local_heads)
3350 self.get_parents = get_parents
3351 self.parents: dict[ObjectID, list[ObjectID] | None] = {}
3352 if shallow is None:
3353 shallow = set()
3354 self.shallow = shallow
3355 self.update_shallow = update_shallow
3357 def nak(self) -> None:
3358 """Nothing in common was found."""
3360 def ack(self, sha: ObjectID) -> None:
3361 """Ack that a revision and its ancestors are present in the source."""
3362 if len(sha) != 40:
3363 # TODO: support SHA256
3364 raise ValueError(f"unexpected sha {sha!r} received")
3365 ancestors = {sha}
3367 # stop if we run out of heads to remove
3368 while self.heads:
3369 for a in ancestors:
3370 if a in self.heads:
3371 self.heads.remove(a)
3373 # collect all ancestors
3374 new_ancestors = set()
3375 for a in ancestors:
3376 ps = self.parents.get(a)
3377 if ps is not None:
3378 new_ancestors.update(ps)
3379 self.parents[a] = None
3381 # no more ancestors; stop
3382 if not new_ancestors:
3383 break
3385 ancestors = new_ancestors
3387 def next(self) -> ObjectID | None:
3388 """Iterate over ancestors of heads in the target."""
3389 if self.heads:
3390 ret = self.heads.pop()
3391 try:
3392 ps = self.get_parents(ret)
3393 except KeyError:
3394 return None
3395 self.parents[ret] = ps
3396 self.heads.update([p for p in ps if p not in self.parents])
3397 return ret
3398 return None
3400 __next__ = next
3403def commit_tree_changes(
3404 object_store: BaseObjectStore,
3405 tree: ObjectID | Tree,
3406 changes: Sequence[tuple[bytes, int | None, ObjectID | None]],
3407) -> ObjectID:
3408 """Commit a specified set of changes to a tree structure.
3410 This will apply a set of changes on top of an existing tree, storing new
3411 objects in object_store.
3413 changes are a list of tuples with (path, mode, object_sha).
3414 Paths can be both blobs and trees. See the mode and
3415 object sha to None deletes the path.
3417 This method works especially well if there are only a small
3418 number of changes to a big tree. For a large number of changes
3419 to a large tree, use e.g. commit_tree.
3421 Args:
3422 object_store: Object store to store new objects in
3423 and retrieve old ones from.
3424 tree: Original tree root (SHA or Tree object)
3425 changes: changes to apply
3426 Returns: New tree root object
3427 """
3428 # TODO(jelmer): Save up the objects and add them using .add_objects
3429 # rather than with individual calls to .add_object.
3430 # Handle both Tree object and SHA
3431 if isinstance(tree, Tree):
3432 tree_obj: Tree = tree
3433 else:
3434 sha_obj = object_store[tree]
3435 assert isinstance(sha_obj, Tree)
3436 tree_obj = sha_obj
3437 nested_changes: dict[bytes, list[tuple[bytes, int | None, ObjectID | None]]] = {}
3438 for path, new_mode, new_sha in changes:
3439 try:
3440 (dirname, subpath) = path.split(b"/", 1)
3441 except ValueError:
3442 if new_sha is None:
3443 del tree_obj[path]
3444 else:
3445 assert new_mode is not None
3446 tree_obj[path] = (new_mode, new_sha)
3447 else:
3448 nested_changes.setdefault(dirname, []).append((subpath, new_mode, new_sha))
3449 for name, subchanges in nested_changes.items():
3450 try:
3451 orig_subtree_id: ObjectID | Tree = tree_obj[name][1]
3452 except KeyError:
3453 # For new directories, pass an empty Tree object
3454 orig_subtree_id = Tree()
3455 subtree_id = commit_tree_changes(object_store, orig_subtree_id, subchanges)
3456 subtree = object_store[subtree_id]
3457 assert isinstance(subtree, Tree)
3458 if len(subtree) == 0:
3459 del tree_obj[name]
3460 else:
3461 tree_obj[name] = (stat.S_IFDIR, subtree.id)
3462 object_store.add_object(tree_obj)
3463 return tree_obj.id
3466class OverlayObjectStore(BaseObjectStore):
3467 """Object store that can overlay multiple object stores."""
3469 def __init__(
3470 self,
3471 bases: list[BaseObjectStore],
3472 add_store: BaseObjectStore | None = None,
3473 ) -> None:
3474 """Initialize an OverlayObjectStore.
3476 Args:
3477 bases: List of base object stores to overlay
3478 add_store: Optional store to write new objects to
3480 Raises:
3481 ValueError: If stores have different hash algorithms
3482 """
3483 from .object_format import verify_same_object_format
3485 # Verify all stores use the same hash algorithm
3486 store_algorithms = [store.object_format for store in bases]
3487 if add_store:
3488 store_algorithms.append(add_store.object_format)
3490 object_format = verify_same_object_format(*store_algorithms)
3492 super().__init__(object_format=object_format)
3493 self.bases = bases
3494 self.add_store = add_store
3496 def add_object(self, obj: ShaFile) -> None:
3497 """Add a single object to the store.
3499 Args:
3500 obj: Object to add
3502 Raises:
3503 NotImplementedError: If no add_store was provided
3504 """
3505 if self.add_store is None:
3506 raise NotImplementedError(self.add_object)
3507 return self.add_store.add_object(obj)
3509 def add_objects(
3510 self,
3511 objects: Sequence[tuple[ShaFile, str | None]],
3512 progress: Callable[[str], None] | None = None,
3513 ) -> Pack | None:
3514 """Add multiple objects to the store.
3516 Args:
3517 objects: Iterator of objects to add
3518 progress: Optional progress reporting callback
3520 Raises:
3521 NotImplementedError: If no add_store was provided
3522 """
3523 if self.add_store is None:
3524 raise NotImplementedError(self.add_object)
3525 return self.add_store.add_objects(objects, progress)
3527 @property
3528 def packs(self) -> list[Pack]:
3529 """Get the list of packs from all overlaid stores.
3531 Returns:
3532 Combined list of packs from all base stores
3533 """
3534 ret = []
3535 for b in self.bases:
3536 ret.extend(b.packs)
3537 return ret
3539 def __iter__(self) -> Iterator[ObjectID]:
3540 """Iterate over all object SHAs in the overlaid stores.
3542 Returns:
3543 Iterator of object SHAs (deduped across stores)
3544 """
3545 done = set()
3546 for b in self.bases:
3547 for o_id in b:
3548 if o_id not in done:
3549 yield o_id
3550 done.add(o_id)
3552 def iterobjects_subset(
3553 self, shas: Iterable[ObjectID], *, allow_missing: bool = False
3554 ) -> Iterator[ShaFile]:
3555 """Iterate over a subset of objects from the overlaid stores.
3557 Args:
3558 shas: Iterable of object SHAs to retrieve
3559 allow_missing: If True, skip missing objects; if False, raise KeyError
3561 Returns:
3562 Iterator of ShaFile objects
3564 Raises:
3565 KeyError: If an object is missing and allow_missing is False
3566 """
3567 todo = set(shas)
3568 found: set[ObjectID] = set()
3570 for b in self.bases:
3571 # Create a copy of todo for each base to avoid modifying
3572 # the set while iterating through it
3573 current_todo = todo - found
3574 for o in b.iterobjects_subset(current_todo, allow_missing=True):
3575 yield o
3576 found.add(o.id)
3578 # Check for any remaining objects not found
3579 missing = todo - found
3580 if missing and not allow_missing:
3581 raise KeyError(next(iter(missing)))
3583 def iter_unpacked_subset(
3584 self,
3585 shas: Iterable[ObjectID | RawObjectID],
3586 *,
3587 include_comp: bool = False,
3588 allow_missing: bool = False,
3589 convert_ofs_delta: bool = True,
3590 ) -> Iterator[UnpackedObject]:
3591 """Iterate over unpacked objects from the overlaid stores.
3593 Args:
3594 shas: Iterable of object SHAs to retrieve
3595 include_comp: Whether to include compressed data
3596 allow_missing: If True, skip missing objects; if False, raise KeyError
3597 convert_ofs_delta: Whether to convert OFS_DELTA objects
3599 Returns:
3600 Iterator of unpacked objects
3602 Raises:
3603 KeyError: If an object is missing and allow_missing is False
3604 """
3605 todo: set[ObjectID | RawObjectID] = set(shas)
3606 for b in self.bases:
3607 for o in b.iter_unpacked_subset(
3608 todo,
3609 include_comp=include_comp,
3610 allow_missing=True,
3611 convert_ofs_delta=convert_ofs_delta,
3612 ):
3613 yield o
3614 todo.remove(o.sha())
3615 if todo and not allow_missing:
3616 raise KeyError(next(iter(todo)))
3618 def get_raw(self, name: ObjectID | RawObjectID) -> tuple[int, bytes]:
3619 """Get the raw object data from the overlaid stores.
3621 Args:
3622 name: SHA of the object
3624 Returns:
3625 Tuple of (type_num, raw_data)
3627 Raises:
3628 KeyError: If object not found in any base store
3629 """
3630 for b in self.bases:
3631 try:
3632 return b.get_raw(name)
3633 except KeyError:
3634 pass
3635 raise KeyError(name)
3637 def contains_packed(self, sha: ObjectID | RawObjectID) -> bool:
3638 """Check if an object is packed in any base store.
3640 Args:
3641 sha: SHA of the object
3643 Returns:
3644 True if object is packed in any base store
3645 """
3646 for b in self.bases:
3647 if b.contains_packed(sha):
3648 return True
3649 return False
3651 def contains_loose(self, sha: ObjectID) -> bool:
3652 """Check if an object is loose in any base store.
3654 Args:
3655 sha: SHA of the object
3657 Returns:
3658 True if object is loose in any base store
3659 """
3660 for b in self.bases:
3661 if b.contains_loose(sha):
3662 return True
3663 return False
3666def read_packs_file(f: BinaryIO) -> Iterator[str]:
3667 """Yield the packs listed in a packs file."""
3668 for line in f.read().splitlines():
3669 if not line:
3670 continue
3671 (kind, name) = line.split(b" ", 1)
3672 if kind != b"P":
3673 continue
3674 yield os.fsdecode(name)
3677class BucketBasedObjectStore(PackBasedObjectStore):
3678 """Object store implementation that uses a bucket store like S3 as backend."""
3680 def _iter_loose_objects(self) -> Iterator[ObjectID]:
3681 """Iterate over the SHAs of all loose objects."""
3682 return iter([])
3684 def _get_loose_object(self, sha: ObjectID) -> None:
3685 return None
3687 def delete_loose_object(self, sha: ObjectID) -> None:
3688 """Delete a loose object (no-op for bucket stores).
3690 Bucket-based stores don't have loose objects, so this is a no-op.
3692 Args:
3693 sha: SHA of the object to delete
3694 """
3695 # Doesn't exist..
3697 def pack_loose_objects(self, progress: Callable[[str], None] | None = None) -> int:
3698 """Pack loose objects. Returns number of objects packed.
3700 BucketBasedObjectStore doesn't support loose objects, so this is a no-op.
3702 Args:
3703 progress: Optional progress reporting callback (ignored)
3704 """
3705 return 0
3707 def _remove_pack_by_name(self, name: str) -> None:
3708 """Remove a pack by name. Subclasses should implement this."""
3709 raise NotImplementedError(self._remove_pack_by_name)
3711 def _iter_pack_names(self) -> Iterator[str]:
3712 raise NotImplementedError(self._iter_pack_names)
3714 def _get_pack(self, name: str) -> Pack:
3715 raise NotImplementedError(self._get_pack)
3717 def _update_pack_cache(self) -> list[Pack]:
3718 pack_files = set(self._iter_pack_names())
3720 # Open newly appeared pack files
3721 new_packs = []
3722 for f in pack_files:
3723 if f not in self._pack_cache:
3724 pack = self._get_pack(f)
3725 new_packs.append(pack)
3726 self._pack_cache[f] = pack
3727 # Remove disappeared pack files
3728 for f in set(self._pack_cache) - pack_files:
3729 self._pack_cache.pop(f).close()
3730 return new_packs
3732 def _upload_pack(
3733 self, basename: str, pack_file: BinaryIO, index_file: BinaryIO
3734 ) -> None:
3735 raise NotImplementedError
3737 def add_pack(self) -> tuple[BinaryIO, Callable[[], None], Callable[[], None]]:
3738 """Add a new pack to this object store.
3740 Returns: Fileobject to write to, a commit function to
3741 call when the pack is finished and an abort
3742 function.
3743 """
3744 import tempfile
3746 pf = tempfile.SpooledTemporaryFile(
3747 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3748 )
3750 def commit() -> Pack | None:
3751 if pf.tell() == 0:
3752 pf.close()
3753 return None
3755 pf.seek(0)
3757 p = PackData(pf.name, file=pf, object_format=self.object_format)
3758 entries = p.sorted_entries()
3759 basename = iter_sha1(entry[0] for entry in entries).decode("ascii")
3760 idxf = tempfile.SpooledTemporaryFile(
3761 max_size=PACK_SPOOL_FILE_MAX_SIZE, prefix="incoming-"
3762 )
3763 checksum = p.get_stored_checksum()
3764 write_pack_index(idxf, entries, checksum, version=self.pack_index_version)
3765 idxf.seek(0)
3766 idx = load_pack_index_file(basename + ".idx", idxf, self.object_format)
3767 for pack in self.packs:
3768 if pack.get_stored_checksum() == p.get_stored_checksum():
3769 p.close()
3770 idx.close()
3771 pf.close()
3772 idxf.close()
3773 return pack
3774 pf.seek(0)
3775 idxf.seek(0)
3776 self._upload_pack(basename, pf, idxf) # type: ignore[arg-type]
3777 final_pack = Pack.from_objects(p, idx)
3778 self._add_cached_pack(basename, final_pack)
3779 pf.close()
3780 idxf.close()
3781 return final_pack
3783 return pf, commit, pf.close # type: ignore[return-value]
3786def _collect_ancestors(
3787 store: ObjectContainer,
3788 heads: Iterable[ObjectID],
3789 common: frozenset[ObjectID] = frozenset(),
3790 shallow: frozenset[ObjectID] = frozenset(),
3791 get_parents: Callable[[Commit], list[ObjectID]] = lambda commit: commit.parents,
3792) -> tuple[set[ObjectID], set[ObjectID]]:
3793 """Collect all ancestors of heads up to (excluding) those in common.
3795 Args:
3796 store: Object store to get commits from
3797 heads: commits to start from
3798 common: commits to end at, or empty set to walk repository
3799 completely
3800 shallow: Set of shallow commits
3801 get_parents: Optional function for getting the parents of a
3802 commit.
3803 Returns: a tuple (A, B) where A - all commits reachable
3804 from heads but not present in common, B - common (shared) elements
3805 that are directly reachable from heads
3806 """
3807 bases = set()
3808 commits = set()
3809 queue: list[ObjectID] = []
3810 queue.extend(heads)
3812 # Try to use commit graph if available
3813 commit_graph = store.get_commit_graph()
3815 while queue:
3816 e = queue.pop(0)
3817 if e in common:
3818 bases.add(e)
3819 elif e not in commits:
3820 commits.add(e)
3821 if e in shallow:
3822 continue
3824 # Try to use commit graph for parent lookup
3825 parents = None
3826 if commit_graph:
3827 parents = commit_graph.get_parents(e)
3829 if parents is None:
3830 # Fall back to loading the object
3831 cmt = store[e]
3832 assert isinstance(cmt, Commit)
3833 parents = get_parents(cmt)
3835 queue.extend(parents)
3836 return (commits, bases)
3839def iter_tree_contents(
3840 store: ObjectContainer, tree_id: ObjectID | None, *, include_trees: bool = False
3841) -> Iterator[TreeEntry]:
3842 """Iterate the contents of a tree and all subtrees.
3844 Iteration is depth-first pre-order, as in e.g. os.walk.
3846 Args:
3847 store: Object store to get trees from
3848 tree_id: SHA1 of the tree.
3849 include_trees: If True, include tree objects in the iteration.
3851 Yields: TreeEntry namedtuples for all the objects in a tree.
3852 """
3853 if tree_id is None:
3854 return
3855 # This could be fairly easily generalized to >2 trees if we find a use
3856 # case.
3857 todo = [TreeEntry(b"", stat.S_IFDIR, tree_id)]
3858 while todo:
3859 entry = todo.pop()
3860 assert entry.mode is not None
3861 if stat.S_ISDIR(entry.mode):
3862 extra = []
3863 assert entry.sha is not None
3864 tree = store[entry.sha]
3865 assert isinstance(tree, Tree)
3866 for subentry in tree.iteritems(name_order=True):
3867 assert entry.path is not None
3868 extra.append(subentry.in_path(entry.path))
3869 todo.extend(reversed(extra))
3870 if not stat.S_ISDIR(entry.mode) or include_trees:
3871 yield entry
3874def iter_commit_contents(
3875 store: ObjectContainer,
3876 commit: Commit | ObjectID | RawObjectID,
3877 *,
3878 include: Sequence[str | bytes | Path] | None = None,
3879) -> Iterator[TreeEntry]:
3880 """Iterate the contents of the repository at the specified commit.
3882 This is a wrapper around iter_tree_contents() and
3883 tree_lookup_path() to simplify the common task of getting the
3884 contest of a repo at a particular commit. See also
3885 dulwich.index.build_file_from_blob() for writing individual files
3886 to disk.
3888 Args:
3889 store: Object store to get trees from
3890 commit: Commit object, or SHA1 of a commit
3891 include: if provided, only the entries whose paths are in the
3892 list, or whose parent tree is in the list, will be
3893 included. Note that duplicate or overlapping paths
3894 (e.g. ["foo", "foo/bar"]) may result in duplicate entries
3896 Yields: TreeEntry namedtuples for all matching files in a commit.
3897 """
3898 sha = commit.id if isinstance(commit, Commit) else commit
3899 if not isinstance(obj := store[sha], Commit):
3900 raise TypeError(
3901 f"{sha.decode('ascii')} should be ID of a Commit, but is {type(obj)}"
3902 )
3903 commit = obj
3904 encoding = commit.encoding or "utf-8"
3905 include_bytes: list[bytes] = (
3906 [
3907 path if isinstance(path, bytes) else str(path).encode(encoding)
3908 for path in include
3909 ]
3910 if include is not None
3911 else [b""]
3912 )
3914 for path in include_bytes:
3915 mode, obj_id = tree_lookup_path(store.__getitem__, commit.tree, path)
3916 # Iterate all contained files if path points to a dir, otherwise just get that
3917 # single file
3918 if isinstance(store[obj_id], Tree):
3919 for entry in iter_tree_contents(store, obj_id):
3920 yield entry.in_path(path)
3921 else:
3922 yield TreeEntry(path, mode, obj_id)
3925def peel_sha(
3926 store: ObjectContainer, sha: ObjectID | RawObjectID
3927) -> tuple[ShaFile, ShaFile]:
3928 """Peel all tags from a SHA.
3930 Args:
3931 store: Object store to get objects from
3932 sha: The object SHA to peel.
3933 Returns: The fully-peeled SHA1 of a tag object, after peeling all
3934 intermediate tags; if the original ref does not point to a tag,
3935 this will equal the original SHA1.
3936 """
3937 unpeeled = obj = store[sha]
3938 obj_class = object_class(obj.type_name)
3939 while obj_class is Tag:
3940 assert isinstance(obj, Tag)
3941 obj_class, sha = obj.object
3942 obj = store[sha]
3943 return unpeeled, obj
3946class GraphTraversalReachability:
3947 """Naive graph traversal implementation of ObjectReachabilityProvider.
3949 This implementation wraps existing graph traversal functions
3950 (_collect_ancestors, _collect_filetree_revs) to provide the standard
3951 reachability interface without any performance optimizations.
3952 """
3954 def __init__(self, object_store: BaseObjectStore) -> None:
3955 """Initialize the graph traversal provider.
3957 Args:
3958 object_store: Object store to query
3959 """
3960 self.store = object_store
3962 def get_reachable_commits(
3963 self,
3964 heads: Iterable[ObjectID],
3965 exclude: Iterable[ObjectID] | None = None,
3966 shallow: Set[ObjectID] | None = None,
3967 ) -> set[ObjectID]:
3968 """Get all commits reachable from heads, excluding those in exclude.
3970 Uses _collect_ancestors for commit traversal.
3972 Args:
3973 heads: Starting commit SHAs
3974 exclude: Commit SHAs to exclude (and their ancestors)
3975 shallow: Set of shallow commit boundaries
3977 Returns:
3978 Set of commit SHAs reachable from heads but not from exclude
3979 """
3980 exclude_set = frozenset(exclude) if exclude else frozenset()
3981 shallow_set = frozenset(shallow) if shallow else frozenset()
3982 commits, _bases = _collect_ancestors(
3983 self.store, heads, exclude_set, shallow_set
3984 )
3985 return commits
3987 def get_tree_objects(
3988 self,
3989 tree_shas: Iterable[ObjectID],
3990 ) -> set[ObjectID]:
3991 """Get all trees and blobs reachable from the given trees.
3993 Uses _collect_filetree_revs for tree traversal.
3995 Args:
3996 tree_shas: Starting tree SHAs
3998 Returns:
3999 Set of tree and blob SHAs
4000 """
4001 result: set[ObjectID] = set()
4002 for tree_sha in tree_shas:
4003 _collect_filetree_revs(self.store, tree_sha, result)
4004 return result
4006 def get_reachable_objects(
4007 self,
4008 commits: Iterable[ObjectID],
4009 exclude_commits: Iterable[ObjectID] | None = None,
4010 ) -> set[ObjectID]:
4011 """Get all objects (commits + trees + blobs) reachable from commits.
4013 Args:
4014 commits: Starting commit SHAs
4015 exclude_commits: Commits whose objects should be excluded
4017 Returns:
4018 Set of all object SHAs (commits, trees, blobs)
4019 """
4020 commits_set = set(commits)
4021 result = set(commits_set)
4023 # Get trees for all commits
4024 tree_shas = []
4025 for commit_sha in commits_set:
4026 try:
4027 commit = self.store[commit_sha]
4028 if isinstance(commit, Commit):
4029 tree_shas.append(commit.tree)
4030 except KeyError:
4031 # Commit not in store, skip
4032 continue
4034 # Collect all tree/blob objects
4035 result.update(self.get_tree_objects(tree_shas))
4037 # Exclude objects from exclude_commits if needed
4038 if exclude_commits:
4039 exclude_objects = self.get_reachable_objects(exclude_commits, None)
4040 result -= exclude_objects
4042 return result
4045class BitmapReachability:
4046 """Bitmap-accelerated implementation of ObjectReachabilityProvider.
4048 This implementation uses packfile bitmap indexes where available to
4049 accelerate reachability queries. Falls back to graph traversal when
4050 bitmaps don't cover the requested commits.
4051 """
4053 def __init__(self, object_store: "PackBasedObjectStore") -> None:
4054 """Initialize the bitmap provider.
4056 Args:
4057 object_store: Pack-based object store with bitmap support
4058 """
4059 self.store = object_store
4060 # Fallback to graph traversal for operations not yet optimized
4061 self._fallback = GraphTraversalReachability(object_store)
4063 def _combine_commit_bitmaps(
4064 self,
4065 commit_shas: set[ObjectID],
4066 exclude_shas: set[ObjectID] | None = None,
4067 ) -> tuple["EWAHBitmap", "Pack"] | None:
4068 """Combine bitmaps for multiple commits using OR, with optional exclusion.
4070 Args:
4071 commit_shas: Set of commit SHAs to combine
4072 exclude_shas: Optional set of commit SHAs to exclude
4074 Returns:
4075 Tuple of (combined_bitmap, pack) or None if bitmaps unavailable
4076 """
4077 from .bitmap import find_commit_bitmaps
4079 # Find bitmaps for the commits
4080 commit_bitmaps = find_commit_bitmaps(commit_shas, self.store.packs)
4082 # If we can't find bitmaps for all commits, return None
4083 if len(commit_bitmaps) < len(commit_shas):
4084 return None
4086 # Combine bitmaps using OR
4087 combined_bitmap = None
4088 result_pack = None
4090 for commit_sha in commit_shas:
4091 pack, pack_bitmap, _sha_to_pos = commit_bitmaps[commit_sha]
4092 commit_bitmap = pack_bitmap.get_bitmap(commit_sha)
4094 if commit_bitmap is None:
4095 return None
4097 if combined_bitmap is None:
4098 combined_bitmap = commit_bitmap
4099 result_pack = pack
4100 elif pack == result_pack:
4101 # Same pack, can OR directly
4102 combined_bitmap = combined_bitmap | commit_bitmap
4103 else:
4104 # Different packs, can't combine
4105 return None
4107 # Handle exclusions if provided
4108 if exclude_shas and result_pack and combined_bitmap:
4109 exclude_bitmaps = find_commit_bitmaps(exclude_shas, [result_pack])
4111 if len(exclude_bitmaps) == len(exclude_shas):
4112 # All excludes have bitmaps, compute exclusion
4113 exclude_combined = None
4115 for commit_sha in exclude_shas:
4116 _pack, pack_bitmap, _sha_to_pos = exclude_bitmaps[commit_sha]
4117 exclude_bitmap = pack_bitmap.get_bitmap(commit_sha)
4119 if exclude_bitmap is None:
4120 break
4122 if exclude_combined is None:
4123 exclude_combined = exclude_bitmap
4124 else:
4125 exclude_combined = exclude_combined | exclude_bitmap
4127 # Subtract excludes using set difference
4128 if exclude_combined:
4129 combined_bitmap = combined_bitmap - exclude_combined
4131 if combined_bitmap and result_pack:
4132 return (combined_bitmap, result_pack)
4133 return None
4135 def get_reachable_commits(
4136 self,
4137 heads: Iterable[ObjectID],
4138 exclude: Iterable[ObjectID] | None = None,
4139 shallow: Set[ObjectID] | None = None,
4140 ) -> set[ObjectID]:
4141 """Get all commits reachable from heads using bitmaps where possible.
4143 Args:
4144 heads: Starting commit SHAs
4145 exclude: Commit SHAs to exclude (and their ancestors)
4146 shallow: Set of shallow commit boundaries
4148 Returns:
4149 Set of commit SHAs reachable from heads but not from exclude
4150 """
4151 from .bitmap import bitmap_to_object_shas
4153 # If shallow is specified, fall back to graph traversal
4154 # (bitmaps don't support shallow boundaries well)
4155 if shallow:
4156 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4158 heads_set = set(heads)
4159 exclude_set = set(exclude) if exclude else None
4161 # Try to combine bitmaps
4162 result = self._combine_commit_bitmaps(heads_set, exclude_set)
4163 if result is None:
4164 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4166 combined_bitmap, result_pack = result
4168 # Convert bitmap to commit SHAs, filtering for commits only
4169 pack_bitmap = result_pack.bitmap
4170 if pack_bitmap is None:
4171 return self._fallback.get_reachable_commits(heads, exclude, shallow)
4172 commit_type_filter = pack_bitmap.commit_bitmap
4173 return bitmap_to_object_shas(
4174 combined_bitmap, result_pack.index, commit_type_filter
4175 )
4177 def get_tree_objects(
4178 self,
4179 tree_shas: Iterable[ObjectID],
4180 ) -> set[ObjectID]:
4181 """Get all trees and blobs reachable from the given trees.
4183 Args:
4184 tree_shas: Starting tree SHAs
4186 Returns:
4187 Set of tree and blob SHAs
4188 """
4189 # Tree traversal doesn't benefit much from bitmaps, use fallback
4190 return self._fallback.get_tree_objects(tree_shas)
4192 def get_reachable_objects(
4193 self,
4194 commits: Iterable[ObjectID],
4195 exclude_commits: Iterable[ObjectID] | None = None,
4196 ) -> set[ObjectID]:
4197 """Get all objects reachable from commits using bitmaps.
4199 Args:
4200 commits: Starting commit SHAs
4201 exclude_commits: Commits whose objects should be excluded
4203 Returns:
4204 Set of all object SHAs (commits, trees, blobs)
4205 """
4206 from .bitmap import bitmap_to_object_shas
4208 commits_set = set(commits)
4209 exclude_set = set(exclude_commits) if exclude_commits else None
4211 # Try to combine bitmaps
4212 result = self._combine_commit_bitmaps(commits_set, exclude_set)
4213 if result is None:
4214 return self._fallback.get_reachable_objects(commits, exclude_commits)
4216 combined_bitmap, result_pack = result
4218 # Convert bitmap to all object SHAs (no type filter)
4219 return bitmap_to_object_shas(combined_bitmap, result_pack.index, None)