Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/gc.py: 23%

1# gc.py -- Git garbage collection implementation 

2# Copyright (C) 2025 Jelmer Vernooij <jelmer@jelmer.uk> 

3# 

4# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 

5# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU 

6# General Public License as published by the Free Software Foundation; version 2.0 

7# or (at your option) any later version. You can redistribute it and/or 

8# modify it under the terms of either of these two licenses. 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, 

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

13# See the License for the specific language governing permissions and 

14# limitations under the License. 

15# 

16# You should have received a copy of the licenses; if not, see 

17# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License 

18# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache 

19# License, Version 2.0. 

20# 

21 

22"""Git garbage collection implementation.""" 

23 

24__all__ = [ 

25 "DEFAULT_GC_AUTO", 

26 "DEFAULT_GC_AUTO_PACK_LIMIT", 

27 "DEFAULT_GC_PRUNE_EXPIRE", 

28 "GCStats", 

29 "find_reachable_objects", 

30 "find_unreachable_objects", 

31 "garbage_collect", 

32 "get_prune_grace_period", 

33 "maybe_auto_gc", 

34 "prune_unreachable_objects", 

35 "should_run_gc", 

36] 

37 

38import logging 

39import os 

40import time 

41from collections import deque 

42from collections.abc import Callable 

43from dataclasses import dataclass, field 

44from typing import TYPE_CHECKING 

45 

46from dulwich.object_store import ( 

47 BaseObjectStore, 

48 DiskObjectStore, 

49) 

50from dulwich.objects import Commit, ObjectID, Tag, Tree 

51from dulwich.refs import RefsContainer 

52 

53if TYPE_CHECKING: 

54 from .config import Config 

55 from .repo import BaseRepo, Repo 

56 

57 

58logger = logging.getLogger(__name__) 

59 

60DEFAULT_GC_AUTO = 6700 

61DEFAULT_GC_AUTO_PACK_LIMIT = 50 

62DEFAULT_GC_PRUNE_EXPIRE = 1209600 # 2 weeks in seconds 

63 

64 

65def get_prune_grace_period(config: "Config") -> int: 

66 """Read gc.pruneExpire from config and return grace period in seconds. 

67 

68 If gc.pruneExpire is not set, returns the default of 2 weeks. 

69 

70 Args: 

71 config: Repository configuration 

72 

73 Returns: 

74 Grace period in seconds 

75 

76 Raises: 

77 ValueError: If the configured value cannot be parsed 

78 """ 

79 from .approxidate import parse_approxidate 

80 

81 try: 

82 raw_value = config.get(b"gc", b"pruneExpire") 

83 if isinstance(raw_value, bytes): 

84 value = raw_value.decode("utf-8") 

85 else: 

86 value = raw_value 

87 except KeyError: 

88 return DEFAULT_GC_PRUNE_EXPIRE 

89 

90 value = value.strip() 

91 if value == "now": 

92 return 0 

93 

94 timestamp = parse_approxidate(value) 

95 return max(0, int(time.time() - timestamp)) 

96 

97 

98@dataclass 

99class GCStats: 

100 """Statistics from garbage collection.""" 

101 

102 pruned_objects: set[ObjectID] = field(default_factory=set) 

103 bytes_freed: int = 0 

104 packs_before: int = 0 

105 packs_after: int = 0 

106 loose_objects_before: int = 0 

107 loose_objects_after: int = 0 

108 

109 

110def find_reachable_objects( 

111 object_store: BaseObjectStore, 

112 refs_container: RefsContainer, 

113 include_reflogs: bool = True, 

114 progress: Callable[[str], None] | None = None, 

115) -> set[ObjectID]: 

116 """Find all reachable objects in the repository. 

117 

118 Args: 

119 object_store: Object store to search 

120 refs_container: Reference container 

121 include_reflogs: Whether to include reflog entries 

122 progress: Optional progress callback 

123 

124 Returns: 

125 Set of reachable object SHAs 

126 """ 

127 reachable: set[ObjectID] = set() 

128 pending: deque[ObjectID] = deque() 

129 

130 # Start with all refs 

131 for ref in refs_container.allkeys(): 

132 try: 

133 sha = refs_container[ref] # This follows symbolic refs 

134 if sha and sha not in reachable: 

135 pending.append(sha) 

136 reachable.add(sha) 

137 except KeyError: 

138 # Broken ref 

139 if progress: 

140 progress(f"Warning: Broken ref {ref.decode('utf-8', 'replace')}") 

141 continue 

142 

143 # TODO: Add reflog support when reflog functionality is available 

144 

145 # Walk all reachable objects 

146 while pending: 

147 sha = pending.popleft() 

148 

149 if progress: 

150 progress(f"Checking object {sha.decode('ascii', 'replace')}") 

151 

152 try: 

153 obj = object_store[sha] 

154 except KeyError: 

155 continue 

156 

157 # Add referenced objects 

158 if isinstance(obj, Commit): 

159 # Tree 

160 if obj.tree not in reachable: 

161 pending.append(obj.tree) 

162 reachable.add(obj.tree) 

163 # Parents 

164 for parent in obj.parents: 

165 if parent not in reachable: 

166 pending.append(parent) 

167 reachable.add(parent) 

168 elif isinstance(obj, Tree): 

169 # Tree entries 

170 for entry in obj.items(): 

171 assert entry.sha is not None 

172 if entry.sha not in reachable: 

173 pending.append(entry.sha) 

174 reachable.add(entry.sha) 

175 elif isinstance(obj, Tag): 

176 # Tagged object 

177 if obj.object[1] not in reachable: 

178 pending.append(obj.object[1]) 

179 reachable.add(obj.object[1]) 

180 

181 return reachable 

182 

183 

184def find_unreachable_objects( 

185 object_store: BaseObjectStore, 

186 refs_container: RefsContainer, 

187 include_reflogs: bool = True, 

188 progress: Callable[[str], None] | None = None, 

189) -> set[ObjectID]: 

190 """Find all unreachable objects in the repository. 

191 

192 Args: 

193 object_store: Object store to search 

194 refs_container: Reference container 

195 include_reflogs: Whether to include reflog entries 

196 progress: Optional progress callback 

197 

198 Returns: 

199 Set of unreachable object SHAs 

200 """ 

201 reachable = find_reachable_objects( 

202 object_store, refs_container, include_reflogs, progress 

203 ) 

204 

205 unreachable: set[ObjectID] = set() 

206 for sha in object_store: 

207 if sha not in reachable: 

208 unreachable.add(sha) 

209 

210 return unreachable 

211 

212 

213def prune_unreachable_objects( 

214 object_store: DiskObjectStore, 

215 refs_container: RefsContainer, 

216 grace_period: int | None = None, 

217 dry_run: bool = False, 

218 progress: Callable[[str], None] | None = None, 

219) -> tuple[set[ObjectID], int]: 

220 """Remove unreachable objects from the repository. 

221 

222 Args: 

223 object_store: Object store to prune 

224 refs_container: Reference container 

225 grace_period: Grace period in seconds (objects newer than this are kept) 

226 dry_run: If True, only report what would be deleted 

227 progress: Optional progress callback 

228 

229 Returns: 

230 Tuple of (set of pruned object SHAs, total bytes freed) 

231 """ 

232 unreachable = find_unreachable_objects( 

233 object_store, refs_container, progress=progress 

234 ) 

235 

236 pruned: set[ObjectID] = set() 

237 bytes_freed = 0 

238 

239 for sha in unreachable: 

240 try: 

241 obj = object_store[sha] 

242 

243 # Check grace period 

244 if grace_period is not None: 

245 try: 

246 mtime = object_store.get_object_mtime(sha) 

247 age = time.time() - mtime 

248 if age < grace_period: 

249 if progress: 

250 progress( 

251 f"Keeping {sha.decode('ascii', 'replace')} (age: {age:.0f}s < grace period: {grace_period}s)" 

252 ) 

253 continue 

254 except KeyError: 

255 # Object not found, skip it 

256 continue 

257 

258 if progress: 

259 progress(f"Pruning {sha.decode('ascii', 'replace')}") 

260 

261 # Calculate size before attempting deletion 

262 obj_size = len(obj.as_raw_string()) 

263 

264 if not dry_run: 

265 object_store.delete_loose_object(sha) 

266 

267 # Only count as pruned if we get here (deletion succeeded or dry run) 

268 pruned.add(sha) 

269 bytes_freed += obj_size 

270 

271 except KeyError: 

272 # Object already gone 

273 pass 

274 except OSError as e: 

275 # File system errors during deletion 

276 if progress: 

277 progress(f"Error pruning {sha.decode('ascii', 'replace')}: {e}") 

278 return pruned, bytes_freed 

279 

280 

281def garbage_collect( 

282 repo: "Repo", 

283 auto: bool = False, 

284 aggressive: bool = False, 

285 prune: bool = True, 

286 grace_period: int | None = 1209600, # 2 weeks default 

287 dry_run: bool = False, 

288 progress: Callable[[str], None] | None = None, 

289) -> GCStats: 

290 """Run garbage collection on a repository. 

291 

292 Args: 

293 repo: Repository to garbage collect 

294 auto: Whether this is an automatic gc 

295 aggressive: Whether to use aggressive settings 

296 prune: Whether to prune unreachable objects 

297 grace_period: Grace period for pruning in seconds 

298 dry_run: If True, only report what would be done 

299 progress: Optional progress callback 

300 

301 Returns: 

302 GCStats object with garbage collection statistics 

303 """ 

304 stats = GCStats() 

305 

306 object_store = repo.object_store 

307 refs_container = repo.refs 

308 

309 # Count initial state 

310 stats.packs_before = len(list(object_store.packs)) 

311 stats.loose_objects_before = object_store.count_loose_objects() 

312 

313 # Find unreachable objects to exclude from repacking 

314 unreachable_to_prune = set() 

315 if prune: 

316 if progress: 

317 progress("Finding unreachable objects") 

318 unreachable = find_unreachable_objects( 

319 object_store, refs_container, progress=progress 

320 ) 

321 

322 # Apply grace period check 

323 for sha in unreachable: 

324 try: 

325 if grace_period is not None: 

326 try: 

327 mtime = object_store.get_object_mtime(sha) 

328 age = time.time() - mtime 

329 if age < grace_period: 

330 if progress: 

331 progress( 

332 f"Keeping {sha.decode('ascii', 'replace')} (age: {age:.0f}s < grace period: {grace_period}s)" 

333 ) 

334 continue 

335 except KeyError: 

336 # Object not found, skip it 

337 continue 

338 

339 unreachable_to_prune.add(sha) 

340 obj = object_store[sha] 

341 stats.bytes_freed += len(obj.as_raw_string()) 

342 except KeyError: 

343 pass 

344 

345 stats.pruned_objects = unreachable_to_prune 

346 

347 # Pack refs 

348 if progress: 

349 progress("Packing references") 

350 if not dry_run: 

351 repo.refs.pack_refs() 

352 

353 # Delete loose unreachable objects 

354 if prune and not dry_run: 

355 for sha in unreachable_to_prune: 

356 if object_store.contains_loose(sha): 

357 try: 

358 object_store.delete_loose_object(sha) 

359 except OSError: 

360 pass 

361 

362 # Repack everything, excluding unreachable objects 

363 # This handles both loose object packing and pack consolidation 

364 if progress: 

365 progress("Repacking repository") 

366 if not dry_run: 

367 if prune and unreachable_to_prune: 

368 # Repack excluding unreachable objects 

369 object_store.repack(exclude=unreachable_to_prune, progress=progress) 

370 else: 

371 # Normal repack 

372 object_store.repack(progress=progress) 

373 

374 # Prune orphaned temporary files 

375 if progress: 

376 progress("Pruning temporary files") 

377 if not dry_run: 

378 object_store.prune(grace_period=grace_period) 

379 

380 # Count final state 

381 stats.packs_after = len(list(object_store.packs)) 

382 stats.loose_objects_after = object_store.count_loose_objects() 

383 

384 return stats 

385 

386 

387def should_run_gc(repo: "BaseRepo", config: "Config | None" = None) -> bool: 

388 """Check if automatic garbage collection should run. 

389 

390 Args: 

391 repo: Repository to check 

392 config: Configuration to use (defaults to repo config) 

393 

394 Returns: 

395 True if GC should run, False otherwise 

396 """ 

397 # Check environment variable first 

398 if os.environ.get("GIT_AUTO_GC") == "0": 

399 return False 

400 

401 # Check programmatic disable flag 

402 if getattr(repo, "_autogc_disabled", False): 

403 return False 

404 

405 if config is None: 

406 config = repo.get_config() 

407 

408 # Check if auto GC is disabled 

409 try: 

410 gc_auto = config.get(b"gc", b"auto") 

411 gc_auto_value = int(gc_auto) 

412 except KeyError: 

413 gc_auto_value = DEFAULT_GC_AUTO 

414 

415 if gc_auto_value == 0: 

416 # Auto GC is disabled 

417 return False 

418 

419 # Check loose object count 

420 object_store = repo.object_store 

421 if not isinstance(object_store, DiskObjectStore): 

422 # Can't count loose objects on non-disk stores 

423 return False 

424 

425 loose_count = object_store.count_loose_objects() 

426 if loose_count >= gc_auto_value: 

427 return True 

428 

429 # Check pack file count 

430 try: 

431 gc_auto_pack_limit = config.get(b"gc", b"autoPackLimit") 

432 pack_limit = int(gc_auto_pack_limit) 

433 except KeyError: 

434 pack_limit = DEFAULT_GC_AUTO_PACK_LIMIT 

435 

436 if pack_limit > 0: 

437 pack_count = object_store.count_pack_files() 

438 if pack_count >= pack_limit: 

439 return True 

440 

441 return False 

442 

443 

444def maybe_auto_gc( 

445 repo: "Repo", 

446 config: "Config | None" = None, 

447 progress: Callable[[str], None] | None = None, 

448) -> bool: 

449 """Run automatic garbage collection if needed. 

450 

451 Args: 

452 repo: Repository to potentially GC 

453 config: Configuration to use (defaults to repo config) 

454 progress: Optional progress reporting callback 

455 

456 Returns: 

457 True if GC was run, False otherwise 

458 """ 

459 if not should_run_gc(repo, config): 

460 return False 

461 

462 # Check for gc.log file - only for disk-based repos 

463 if not hasattr(repo, "controldir"): 

464 # For non-disk repos, just run GC without gc.log handling 

465 garbage_collect(repo, auto=True, progress=progress) 

466 return True 

467 

468 gc_log_path = os.path.join(repo.controldir(), "gc.log") 

469 if os.path.exists(gc_log_path): 

470 # Check gc.logExpiry 

471 if config is None: 

472 config = repo.get_config() 

473 try: 

474 log_expiry = config.get(b"gc", b"logExpiry") 

475 except KeyError: 

476 # Default to 1 day 

477 expiry_seconds = 86400 

478 else: 

479 # Parse time value (simplified - just support days for now) 

480 if log_expiry.endswith((b".days", b".day")): 

481 days = int(log_expiry.split(b".")[0]) 

482 expiry_seconds = days * 86400 

483 else: 

484 # Default to 1 day 

485 expiry_seconds = 86400 

486 

487 stat_info = os.stat(gc_log_path) 

488 if time.time() - stat_info.st_mtime < expiry_seconds: 

489 # gc.log exists and is not expired - skip GC 

490 with open(gc_log_path, "rb") as f: 

491 logger.info( 

492 "gc.log content: %s", f.read().decode("utf-8", errors="replace") 

493 ) 

494 return False 

495 

496 # TODO: Support gc.autoDetach to run in background 

497 # For now, run in foreground 

498 

499 try: 

500 # Run GC with auto=True flag 

501 garbage_collect(repo, auto=True, progress=progress) 

502 

503 # Remove gc.log on successful completion 

504 if os.path.exists(gc_log_path): 

505 try: 

506 os.unlink(gc_log_path) 

507 except FileNotFoundError: 

508 pass 

509 

510 return True 

511 except OSError as e: 

512 # Write error to gc.log 

513 with open(gc_log_path, "wb") as f: 

514 f.write(f"Auto GC failed: {e}\n".encode()) 

515 # Don't propagate the error - auto GC failures shouldn't break operations 

516 return False