Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/gc.py: 23%

1# gc.py -- Git garbage collection implementation 

2# Copyright (C) 2025 Jelmer Vernooij <jelmer@jelmer.uk> 

3# 

4# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 

5# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU 

6# General Public License as published by the Free Software Foundation; version 2.0 

7# or (at your option) any later version. You can redistribute it and/or 

8# modify it under the terms of either of these two licenses. 

9# 

10# Unless required by applicable law or agreed to in writing, software 

11# distributed under the License is distributed on an "AS IS" BASIS, 

12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

13# See the License for the specific language governing permissions and 

14# limitations under the License. 

15# 

16# You should have received a copy of the licenses; if not, see 

17# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License 

18# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache 

19# License, Version 2.0. 

20# 

21 

22"""Git garbage collection implementation.""" 

23 

24__all__ = [ 

25 "DEFAULT_GC_AUTO", 

26 "DEFAULT_GC_AUTO_PACK_LIMIT", 

27 "DEFAULT_GC_PRUNE_EXPIRE", 

28 "GCStats", 

29 "find_reachable_objects", 

30 "find_unreachable_objects", 

31 "garbage_collect", 

32 "get_prune_grace_period", 

33 "maybe_auto_gc", 

34 "prune_unreachable_objects", 

35 "should_run_gc", 

36] 

37 

38import logging 

39import os 

40import time 

41from collections import deque 

42from collections.abc import Callable 

43from dataclasses import dataclass, field 

44from typing import TYPE_CHECKING 

45 

46from dulwich.object_store import ( 

47 BaseObjectStore, 

48 DiskObjectStore, 

49) 

50from dulwich.objects import Commit, ObjectID, Tag, Tree 

51from dulwich.refs import RefsContainer 

52 

53if TYPE_CHECKING: 

54 from .config import Config 

55 from .repo import BaseRepo, Repo 

56 

57 

58logger = logging.getLogger(__name__) 

59 

60DEFAULT_GC_AUTO = 6700 

61DEFAULT_GC_AUTO_PACK_LIMIT = 50 

62DEFAULT_GC_PRUNE_EXPIRE = 1209600 # 2 weeks in seconds 

63 

64 

65def get_prune_grace_period(config: "Config") -> int: 

66 """Read gc.pruneExpire from config and return grace period in seconds. 

67 

68 If gc.pruneExpire is not set, returns the default of 2 weeks. 

69 

70 Args: 

71 config: Repository configuration 

72 

73 Returns: 

74 Grace period in seconds 

75 

76 Raises: 

77 ValueError: If the configured value cannot be parsed 

78 """ 

79 from .approxidate import parse_approxidate 

80 

81 try: 

82 raw_value = config.get(b"gc", b"pruneExpire") 

83 if isinstance(raw_value, bytes): 

84 value = raw_value.decode("utf-8") 

85 else: 

86 value = raw_value 

87 except KeyError: 

88 return DEFAULT_GC_PRUNE_EXPIRE 

89 

90 value = value.strip() 

91 if value == "now": 

92 return 0 

93 

94 timestamp = parse_approxidate(value) 

95 return max(0, int(time.time() - timestamp)) 

96 

97 

98@dataclass 

99class GCStats: 

100 """Statistics from garbage collection.""" 

101 

102 pruned_objects: set[ObjectID] = field(default_factory=set) 

103 bytes_freed: int = 0 

104 packs_before: int = 0 

105 packs_after: int = 0 

106 loose_objects_before: int = 0 

107 loose_objects_after: int = 0 

108 

109 

110def find_reachable_objects( 

111 object_store: BaseObjectStore, 

112 refs_container: RefsContainer, 

113 include_reflogs: bool = True, 

114 progress: Callable[[str], None] | None = None, 

115) -> set[ObjectID]: 

116 """Find all reachable objects in the repository. 

117 

118 Args: 

119 object_store: Object store to search 

120 refs_container: Reference container 

121 include_reflogs: Whether to include reflog entries 

122 progress: Optional progress callback 

123 

124 Returns: 

125 Set of reachable object SHAs 

126 """ 

127 reachable: set[ObjectID] = set() 

128 pending: deque[ObjectID] = deque() 

129 

130 # Start with all refs 

131 for ref in refs_container.allkeys(): 

132 try: 

133 sha = refs_container[ref] # This follows symbolic refs 

134 if sha and sha not in reachable: 

135 pending.append(sha) 

136 reachable.add(sha) 

137 except KeyError: 

138 # Broken ref 

139 if progress: 

140 progress(f"Warning: Broken ref {ref.decode('utf-8', 'replace')}") 

141 continue 

142 

143 # TODO: Add reflog support when reflog functionality is available 

144 

145 # Walk all reachable objects 

146 while pending: 

147 sha = pending.popleft() 

148 

149 if progress: 

150 progress(f"Checking object {sha.decode('ascii', 'replace')}") 

151 

152 try: 

153 obj = object_store[sha] 

154 except KeyError: 

155 continue 

156 

157 # Add referenced objects 

158 match obj: 

159 case Commit(): 

160 # Tree 

161 if obj.tree not in reachable: 

162 pending.append(obj.tree) 

163 reachable.add(obj.tree) 

164 # Parents 

165 for parent in obj.parents: 

166 if parent not in reachable: 

167 pending.append(parent) 

168 reachable.add(parent) 

169 case Tree(): 

170 # Tree entries 

171 for entry in obj.items(): 

172 assert entry.sha is not None 

173 if entry.sha not in reachable: 

174 pending.append(entry.sha) 

175 reachable.add(entry.sha) 

176 case Tag(): 

177 # Tagged object 

178 if obj.object[1] not in reachable: 

179 pending.append(obj.object[1]) 

180 reachable.add(obj.object[1]) 

181 

182 return reachable 

183 

184 

185def find_unreachable_objects( 

186 object_store: BaseObjectStore, 

187 refs_container: RefsContainer, 

188 include_reflogs: bool = True, 

189 progress: Callable[[str], None] | None = None, 

190) -> set[ObjectID]: 

191 """Find all unreachable objects in the repository. 

192 

193 Args: 

194 object_store: Object store to search 

195 refs_container: Reference container 

196 include_reflogs: Whether to include reflog entries 

197 progress: Optional progress callback 

198 

199 Returns: 

200 Set of unreachable object SHAs 

201 """ 

202 reachable = find_reachable_objects( 

203 object_store, refs_container, include_reflogs, progress 

204 ) 

205 

206 unreachable: set[ObjectID] = set() 

207 for sha in object_store: 

208 if sha not in reachable: 

209 unreachable.add(sha) 

210 

211 return unreachable 

212 

213 

214def prune_unreachable_objects( 

215 object_store: DiskObjectStore, 

216 refs_container: RefsContainer, 

217 grace_period: int | None = None, 

218 dry_run: bool = False, 

219 progress: Callable[[str], None] | None = None, 

220) -> tuple[set[ObjectID], int]: 

221 """Remove unreachable objects from the repository. 

222 

223 Args: 

224 object_store: Object store to prune 

225 refs_container: Reference container 

226 grace_period: Grace period in seconds (objects newer than this are kept) 

227 dry_run: If True, only report what would be deleted 

228 progress: Optional progress callback 

229 

230 Returns: 

231 Tuple of (set of pruned object SHAs, total bytes freed) 

232 """ 

233 unreachable = find_unreachable_objects( 

234 object_store, refs_container, progress=progress 

235 ) 

236 

237 pruned: set[ObjectID] = set() 

238 bytes_freed = 0 

239 

240 for sha in unreachable: 

241 try: 

242 obj = object_store[sha] 

243 

244 # Check grace period 

245 if grace_period is not None: 

246 try: 

247 mtime = object_store.get_object_mtime(sha) 

248 age = time.time() - mtime 

249 if age < grace_period: 

250 if progress: 

251 progress( 

252 f"Keeping {sha.decode('ascii', 'replace')} (age: {age:.0f}s < grace period: {grace_period}s)" 

253 ) 

254 continue 

255 except KeyError: 

256 # Object not found, skip it 

257 continue 

258 

259 if progress: 

260 progress(f"Pruning {sha.decode('ascii', 'replace')}") 

261 

262 # Calculate size before attempting deletion 

263 obj_size = len(obj.as_raw_string()) 

264 

265 if not dry_run: 

266 object_store.delete_loose_object(sha) 

267 

268 # Only count as pruned if we get here (deletion succeeded or dry run) 

269 pruned.add(sha) 

270 bytes_freed += obj_size 

271 

272 except KeyError: 

273 # Object already gone 

274 pass 

275 except OSError as e: 

276 # File system errors during deletion 

277 if progress: 

278 progress(f"Error pruning {sha.decode('ascii', 'replace')}: {e}") 

279 return pruned, bytes_freed 

280 

281 

282def garbage_collect( 

283 repo: "Repo", 

284 auto: bool = False, 

285 aggressive: bool = False, 

286 prune: bool = True, 

287 grace_period: int | None = 1209600, # 2 weeks default 

288 dry_run: bool = False, 

289 progress: Callable[[str], None] | None = None, 

290) -> GCStats: 

291 """Run garbage collection on a repository. 

292 

293 Args: 

294 repo: Repository to garbage collect 

295 auto: Whether this is an automatic gc 

296 aggressive: Whether to use aggressive settings 

297 prune: Whether to prune unreachable objects 

298 grace_period: Grace period for pruning in seconds 

299 dry_run: If True, only report what would be done 

300 progress: Optional progress callback 

301 

302 Returns: 

303 GCStats object with garbage collection statistics 

304 """ 

305 stats = GCStats() 

306 

307 object_store = repo.object_store 

308 refs_container = repo.refs 

309 

310 # Count initial state 

311 stats.packs_before = len(list(object_store.packs)) 

312 stats.loose_objects_before = object_store.count_loose_objects() 

313 

314 # Find unreachable objects to exclude from repacking 

315 unreachable_to_prune = set() 

316 if prune: 

317 if progress: 

318 progress("Finding unreachable objects") 

319 unreachable = find_unreachable_objects( 

320 object_store, refs_container, progress=progress 

321 ) 

322 

323 # Apply grace period check 

324 for sha in unreachable: 

325 try: 

326 if grace_period is not None: 

327 try: 

328 mtime = object_store.get_object_mtime(sha) 

329 age = time.time() - mtime 

330 if age < grace_period: 

331 if progress: 

332 progress( 

333 f"Keeping {sha.decode('ascii', 'replace')} (age: {age:.0f}s < grace period: {grace_period}s)" 

334 ) 

335 continue 

336 except KeyError: 

337 # Object not found, skip it 

338 continue 

339 

340 unreachable_to_prune.add(sha) 

341 obj = object_store[sha] 

342 stats.bytes_freed += len(obj.as_raw_string()) 

343 except KeyError: 

344 pass 

345 

346 stats.pruned_objects = unreachable_to_prune 

347 

348 # Pack refs 

349 if progress: 

350 progress("Packing references") 

351 if not dry_run: 

352 repo.refs.pack_refs() 

353 

354 # Delete loose unreachable objects 

355 if prune and not dry_run: 

356 for sha in unreachable_to_prune: 

357 if object_store.contains_loose(sha): 

358 try: 

359 object_store.delete_loose_object(sha) 

360 except OSError: 

361 pass 

362 

363 # Repack everything, excluding unreachable objects 

364 # This handles both loose object packing and pack consolidation 

365 if progress: 

366 progress("Repacking repository") 

367 if not dry_run: 

368 if prune and unreachable_to_prune: 

369 # Repack excluding unreachable objects 

370 object_store.repack(exclude=unreachable_to_prune, progress=progress) 

371 else: 

372 # Normal repack 

373 object_store.repack(progress=progress) 

374 

375 # Prune orphaned temporary files 

376 if progress: 

377 progress("Pruning temporary files") 

378 if not dry_run: 

379 object_store.prune(grace_period=grace_period) 

380 

381 # Count final state 

382 stats.packs_after = len(list(object_store.packs)) 

383 stats.loose_objects_after = object_store.count_loose_objects() 

384 

385 return stats 

386 

387 

388def should_run_gc(repo: "BaseRepo", config: "Config | None" = None) -> bool: 

389 """Check if automatic garbage collection should run. 

390 

391 Args: 

392 repo: Repository to check 

393 config: Configuration to use (defaults to repo config) 

394 

395 Returns: 

396 True if GC should run, False otherwise 

397 """ 

398 # Check environment variable first 

399 if os.environ.get("GIT_AUTO_GC") == "0": 

400 return False 

401 

402 # Check programmatic disable flag 

403 if getattr(repo, "_autogc_disabled", False): 

404 return False 

405 

406 if config is None: 

407 config = repo.get_config() 

408 

409 # Check if auto GC is disabled 

410 try: 

411 gc_auto = config.get(b"gc", b"auto") 

412 gc_auto_value = int(gc_auto) 

413 except KeyError: 

414 gc_auto_value = DEFAULT_GC_AUTO 

415 

416 if gc_auto_value == 0: 

417 # Auto GC is disabled 

418 return False 

419 

420 # Check loose object count 

421 object_store = repo.object_store 

422 if not isinstance(object_store, DiskObjectStore): 

423 # Can't count loose objects on non-disk stores 

424 return False 

425 

426 loose_count = object_store.count_loose_objects() 

427 if loose_count >= gc_auto_value: 

428 return True 

429 

430 # Check pack file count 

431 try: 

432 gc_auto_pack_limit = config.get(b"gc", b"autoPackLimit") 

433 pack_limit = int(gc_auto_pack_limit) 

434 except KeyError: 

435 pack_limit = DEFAULT_GC_AUTO_PACK_LIMIT 

436 

437 if pack_limit > 0: 

438 pack_count = object_store.count_pack_files() 

439 if pack_count >= pack_limit: 

440 return True 

441 

442 return False 

443 

444 

445def maybe_auto_gc( 

446 repo: "Repo", 

447 config: "Config | None" = None, 

448 progress: Callable[[str], None] | None = None, 

449) -> bool: 

450 """Run automatic garbage collection if needed. 

451 

452 Args: 

453 repo: Repository to potentially GC 

454 config: Configuration to use (defaults to repo config) 

455 progress: Optional progress reporting callback 

456 

457 Returns: 

458 True if GC was run, False otherwise 

459 """ 

460 if not should_run_gc(repo, config): 

461 return False 

462 

463 # Check for gc.log file - only for disk-based repos 

464 if not hasattr(repo, "controldir"): 

465 # For non-disk repos, just run GC without gc.log handling 

466 garbage_collect(repo, auto=True, progress=progress) 

467 return True 

468 

469 gc_log_path = os.path.join(repo.controldir(), "gc.log") 

470 if os.path.exists(gc_log_path): 

471 # Check gc.logExpiry 

472 if config is None: 

473 config = repo.get_config() 

474 try: 

475 log_expiry = config.get(b"gc", b"logExpiry") 

476 except KeyError: 

477 # Default to 1 day 

478 expiry_seconds = 86400 

479 else: 

480 # Parse time value (simplified - just support days for now) 

481 if log_expiry.endswith((b".days", b".day")): 

482 days = int(log_expiry.split(b".")[0]) 

483 expiry_seconds = days * 86400 

484 else: 

485 # Default to 1 day 

486 expiry_seconds = 86400 

487 

488 stat_info = os.stat(gc_log_path) 

489 if time.time() - stat_info.st_mtime < expiry_seconds: 

490 # gc.log exists and is not expired - skip GC 

491 with open(gc_log_path, "rb") as f: 

492 logger.info( 

493 "gc.log content: %s", f.read().decode("utf-8", errors="replace") 

494 ) 

495 return False 

496 

497 # TODO: Support gc.autoDetach to run in background 

498 # For now, run in foreground 

499 

500 try: 

501 # Run GC with auto=True flag 

502 garbage_collect(repo, auto=True, progress=progress) 

503 

504 # Remove gc.log on successful completion 

505 if os.path.exists(gc_log_path): 

506 try: 

507 os.unlink(gc_log_path) 

508 except FileNotFoundError: 

509 pass 

510 

511 return True 

512 except OSError as e: 

513 # Write error to gc.log 

514 with open(gc_log_path, "wb") as f: 

515 f.write(f"Auto GC failed: {e}\n".encode()) 

516 # Don't propagate the error - auto GC failures shouldn't break operations 

517 return False