Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/dulwich/protocol.py: 32%

1# protocol.py -- Shared parts of the git protocols 

2# Copyright (C) 2008 John Carr <john.carr@unrouted.co.uk> 

3# Copyright (C) 2008-2012 Jelmer Vernooij <jelmer@jelmer.uk> 

4# 

5# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later 

6# Dulwich is dual-licensed under the Apache License, Version 2.0 and the GNU 

7# General Public License as published by the Free Software Foundation; version 2.0 

8# or (at your option) any later version. You can redistribute it and/or 

9# modify it under the terms of either of these two licenses. 

10# 

11# Unless required by applicable law or agreed to in writing, software 

12# distributed under the License is distributed on an "AS IS" BASIS, 

13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

14# See the License for the specific language governing permissions and 

15# limitations under the License. 

16# 

17# You should have received a copy of the licenses; if not, see 

18# <http://www.gnu.org/licenses/> for a copy of the GNU General Public License 

19# and <http://www.apache.org/licenses/LICENSE-2.0> for a copy of the Apache 

20# License, Version 2.0. 

21# 

22 

23"""Generic functions for talking the git smart server protocol.""" 

24 

25__all__ = [ 

26 "CAPABILITIES_REF", 

27 "CAPABILITY_AGENT", 

28 "CAPABILITY_ALLOW_REACHABLE_SHA1_IN_WANT", 

29 "CAPABILITY_ALLOW_TIP_SHA1_IN_WANT", 

30 "CAPABILITY_ATOMIC", 

31 "CAPABILITY_DEEPEN_NOT", 

32 "CAPABILITY_DEEPEN_RELATIVE", 

33 "CAPABILITY_DEEPEN_SINCE", 

34 "CAPABILITY_DELETE_REFS", 

35 "CAPABILITY_FETCH", 

36 "CAPABILITY_FILTER", 

37 "CAPABILITY_INCLUDE_TAG", 

38 "CAPABILITY_MULTI_ACK", 

39 "CAPABILITY_MULTI_ACK_DETAILED", 

40 "CAPABILITY_NO_DONE", 

41 "CAPABILITY_NO_PROGRESS", 

42 "CAPABILITY_OBJECT_FORMAT", 

43 "CAPABILITY_OFS_DELTA", 

44 "CAPABILITY_PACKFILE_URIS", 

45 "CAPABILITY_PUSH_OPTIONS", 

46 "CAPABILITY_QUIET", 

47 "CAPABILITY_REPORT_STATUS", 

48 "CAPABILITY_SHALLOW", 

49 "CAPABILITY_SIDE_BAND", 

50 "CAPABILITY_SIDE_BAND_64K", 

51 "CAPABILITY_SYMREF", 

52 "CAPABILITY_THIN_PACK", 

53 "COMMAND_DEEPEN", 

54 "COMMAND_DEEPEN_NOT", 

55 "COMMAND_DEEPEN_SINCE", 

56 "COMMAND_DONE", 

57 "COMMAND_FILTER", 

58 "COMMAND_HAVE", 

59 "COMMAND_SHALLOW", 

60 "COMMAND_UNSHALLOW", 

61 "COMMAND_WANT", 

62 "COMMON_CAPABILITIES", 

63 "DEFAULT_GIT_PROTOCOL_VERSION_FETCH", 

64 "DEFAULT_GIT_PROTOCOL_VERSION_SEND", 

65 "DEPTH_INFINITE", 

66 "GIT_PROTOCOL_VERSIONS", 

67 "KNOWN_RECEIVE_CAPABILITIES", 

68 "KNOWN_UPLOAD_CAPABILITIES", 

69 "MULTI_ACK", 

70 "MULTI_ACK_DETAILED", 

71 "NAK_LINE", 

72 "PEELED_TAG_SUFFIX", 

73 "SIDE_BAND_CHANNEL_DATA", 

74 "SIDE_BAND_CHANNEL_FATAL", 

75 "SIDE_BAND_CHANNEL_PROGRESS", 

76 "SINGLE_ACK", 

77 "TCP_GIT_PORT", 

78 "ZERO_SHA", 

79 "BufferedPktLineWriter", 

80 "PktLineParser", 

81 "Protocol", 

82 "ReceivableProtocol", 

83 "ack_type", 

84 "agent_string", 

85 "capability_agent", 

86 "capability_object_format", 

87 "capability_symref", 

88 "extract_capabilities", 

89 "extract_capability_names", 

90 "extract_want_line_capabilities", 

91 "find_capability", 

92 "format_ack_line", 

93 "format_capability_line", 

94 "format_cmd_pkt", 

95 "format_ref_line", 

96 "format_shallow_line", 

97 "format_unshallow_line", 

98 "parse_capability", 

99 "parse_cmd_pkt", 

100 "pkt_line", 

101 "pkt_seq", 

102 "serialize_refs", 

103 "split_peeled_refs", 

104 "strip_peeled_refs", 

105 "symref_capabilities", 

106 "write_info_refs", 

107] 

108 

109import logging 

110import sys 

111import types 

112from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence 

113from io import BytesIO 

114from os import SEEK_END 

115from typing import TYPE_CHECKING 

116 

117import dulwich 

118 

119from .errors import GitProtocolError, HangupException 

120from .objects import ObjectID, Tag 

121 

122if sys.version_info >= (3, 11): 

123 from typing import Self 

124else: 

125 from typing_extensions import Self 

126 

127logger = logging.getLogger(__name__) 

128 

129if TYPE_CHECKING: 

130 from .pack import ObjectContainer 

131 from .refs import Ref 

132 

133TCP_GIT_PORT = 9418 

134 

135# Git protocol version 0 is the original Git protocol, which lacked a 

136# version number until Git protocol version 1 was introduced by Brandon 

137# Williams in 2017. 

138# 

139# Protocol version 1 is simply the original v0 protocol with the addition of 

140# a single packet line, which precedes the ref advertisement, indicating the 

141# protocol version being used. This was done in preparation for protocol v2. 

142# 

143# Git protocol version 2 was first introduced by Brandon Williams in 2018 and 

144# adds many features. See the gitprotocol-v2(5) manual page for details. 

145# As of 2024, Git only implements version 2 during 'git fetch' and still uses 

146# version 0 during 'git push'. 

147GIT_PROTOCOL_VERSIONS = [0, 1, 2] 

148DEFAULT_GIT_PROTOCOL_VERSION_FETCH = 2 

149DEFAULT_GIT_PROTOCOL_VERSION_SEND = 0 

150 

151# Suffix used in the Git protocol to indicate peeled tag references 

152PEELED_TAG_SUFFIX = b"^{}" 

153 

154ZERO_SHA: ObjectID = ObjectID(b"0" * 40) 

155 

156SINGLE_ACK = 0 

157MULTI_ACK = 1 

158MULTI_ACK_DETAILED = 2 

159 

160# pack data 

161SIDE_BAND_CHANNEL_DATA = 1 

162# progress messages 

163SIDE_BAND_CHANNEL_PROGRESS = 2 

164# fatal error message just before stream aborts 

165SIDE_BAND_CHANNEL_FATAL = 3 

166 

167CAPABILITY_ATOMIC = b"atomic" 

168CAPABILITY_DEEPEN_SINCE = b"deepen-since" 

169CAPABILITY_DEEPEN_NOT = b"deepen-not" 

170CAPABILITY_DEEPEN_RELATIVE = b"deepen-relative" 

171CAPABILITY_DELETE_REFS = b"delete-refs" 

172CAPABILITY_INCLUDE_TAG = b"include-tag" 

173CAPABILITY_MULTI_ACK = b"multi_ack" 

174CAPABILITY_MULTI_ACK_DETAILED = b"multi_ack_detailed" 

175CAPABILITY_NO_DONE = b"no-done" 

176CAPABILITY_NO_PROGRESS = b"no-progress" 

177CAPABILITY_OFS_DELTA = b"ofs-delta" 

178CAPABILITY_QUIET = b"quiet" 

179CAPABILITY_REPORT_STATUS = b"report-status" 

180CAPABILITY_SHALLOW = b"shallow" 

181CAPABILITY_SIDE_BAND = b"side-band" 

182CAPABILITY_SIDE_BAND_64K = b"side-band-64k" 

183CAPABILITY_THIN_PACK = b"thin-pack" 

184CAPABILITY_AGENT = b"agent" 

185CAPABILITY_SYMREF = b"symref" 

186CAPABILITY_ALLOW_TIP_SHA1_IN_WANT = b"allow-tip-sha1-in-want" 

187CAPABILITY_ALLOW_REACHABLE_SHA1_IN_WANT = b"allow-reachable-sha1-in-want" 

188CAPABILITY_FETCH = b"fetch" 

189CAPABILITY_FILTER = b"filter" 

190CAPABILITY_OBJECT_FORMAT = b"object-format" 

191CAPABILITY_PACKFILE_URIS = b"packfile-uris" 

192CAPABILITY_PUSH_OPTIONS = b"push-options" 

193 

194# Magic ref that is used to attach capabilities to when 

195# there are no refs. Should always be ste to ZERO_SHA. 

196CAPABILITIES_REF = b"capabilities^{}" 

197 

198COMMON_CAPABILITIES = [ 

199 CAPABILITY_OFS_DELTA, 

200 CAPABILITY_SIDE_BAND, 

201 CAPABILITY_SIDE_BAND_64K, 

202 CAPABILITY_AGENT, 

203 CAPABILITY_NO_PROGRESS, 

204] 

205KNOWN_UPLOAD_CAPABILITIES = set( 

206 [ 

207 *COMMON_CAPABILITIES, 

208 CAPABILITY_THIN_PACK, 

209 CAPABILITY_MULTI_ACK, 

210 CAPABILITY_MULTI_ACK_DETAILED, 

211 CAPABILITY_INCLUDE_TAG, 

212 CAPABILITY_DEEPEN_SINCE, 

213 CAPABILITY_SYMREF, 

214 CAPABILITY_SHALLOW, 

215 CAPABILITY_DEEPEN_NOT, 

216 CAPABILITY_DEEPEN_RELATIVE, 

217 CAPABILITY_ALLOW_TIP_SHA1_IN_WANT, 

218 CAPABILITY_ALLOW_REACHABLE_SHA1_IN_WANT, 

219 CAPABILITY_FETCH, 

220 CAPABILITY_FILTER, 

221 CAPABILITY_PACKFILE_URIS, 

222 ] 

223) 

224KNOWN_RECEIVE_CAPABILITIES = set( 

225 [ 

226 *COMMON_CAPABILITIES, 

227 CAPABILITY_REPORT_STATUS, 

228 CAPABILITY_DELETE_REFS, 

229 CAPABILITY_QUIET, 

230 CAPABILITY_ATOMIC, 

231 CAPABILITY_PUSH_OPTIONS, 

232 ] 

233) 

234 

235DEPTH_INFINITE = 0x7FFFFFFF 

236 

237NAK_LINE = b"NAK\n" 

238 

239 

240def agent_string() -> bytes: 

241 """Generate the agent string for dulwich. 

242 

243 Returns: 

244 Agent string as bytes 

245 """ 

246 return ("dulwich/" + ".".join(map(str, dulwich.__version__))).encode("ascii") 

247 

248 

249def capability_agent() -> bytes: 

250 """Generate the agent capability string. 

251 

252 Returns: 

253 Agent capability with dulwich version 

254 """ 

255 return CAPABILITY_AGENT + b"=" + agent_string() 

256 

257 

258def capability_object_format(fmt: str) -> bytes: 

259 """Generate the object-format capability string. 

260 

261 Args: 

262 fmt: Object format name (e.g., "sha1" or "sha256") 

263 

264 Returns: 

265 Object-format capability with format name 

266 """ 

267 return CAPABILITY_OBJECT_FORMAT + b"=" + fmt.encode("ascii") 

268 

269 

270def capability_symref(from_ref: bytes, to_ref: bytes) -> bytes: 

271 """Generate a symref capability string. 

272 

273 Args: 

274 from_ref: Source reference name 

275 to_ref: Target reference name 

276 

277 Returns: 

278 Symref capability string 

279 """ 

280 return CAPABILITY_SYMREF + b"=" + from_ref + b":" + to_ref 

281 

282 

283def extract_capability_names(capabilities: Iterable[bytes]) -> set[bytes]: 

284 """Extract capability names from a list of capabilities. 

285 

286 Args: 

287 capabilities: List of capability strings 

288 

289 Returns: 

290 Set of capability names 

291 """ 

292 return {parse_capability(c)[0] for c in capabilities} 

293 

294 

295def parse_capability(capability: bytes) -> tuple[bytes, bytes | None]: 

296 """Parse a capability string into name and value. 

297 

298 Args: 

299 capability: Capability string 

300 

301 Returns: 

302 Tuple of (capability_name, capability_value) 

303 """ 

304 parts = capability.split(b"=", 1) 

305 if len(parts) == 1: 

306 return (parts[0], None) 

307 return (parts[0], parts[1]) 

308 

309 

310def symref_capabilities(symrefs: Iterable[tuple[bytes, bytes]]) -> list[bytes]: 

311 """Generate symref capability strings from symref pairs. 

312 

313 Args: 

314 symrefs: Iterable of (from_ref, to_ref) tuples 

315 

316 Returns: 

317 List of symref capability strings 

318 """ 

319 return [capability_symref(*k) for k in symrefs] 

320 

321 

322COMMAND_DEEPEN = b"deepen" 

323COMMAND_DEEPEN_SINCE = b"deepen-since" 

324COMMAND_DEEPEN_NOT = b"deepen-not" 

325COMMAND_SHALLOW = b"shallow" 

326COMMAND_UNSHALLOW = b"unshallow" 

327COMMAND_DONE = b"done" 

328COMMAND_WANT = b"want" 

329COMMAND_HAVE = b"have" 

330COMMAND_FILTER = b"filter" 

331 

332 

333def format_cmd_pkt(cmd: bytes, *args: bytes) -> bytes: 

334 """Format a command packet. 

335 

336 Args: 

337 cmd: Command name 

338 *args: Command arguments 

339 

340 Returns: 

341 Formatted command packet 

342 """ 

343 return cmd + b" " + b"".join([(a + b"\0") for a in args]) 

344 

345 

346def parse_cmd_pkt(line: bytes) -> tuple[bytes, list[bytes]]: 

347 """Parse a command packet. 

348 

349 Args: 

350 line: Command line to parse 

351 

352 Returns: 

353 Tuple of (command, [arguments]) 

354 """ 

355 splice_at = line.find(b" ") 

356 cmd, args = line[:splice_at], line[splice_at + 1 :] 

357 assert args[-1:] == b"\x00" 

358 return cmd, args[:-1].split(b"\0") 

359 

360 

361def pkt_line(data: bytes | None) -> bytes: 

362 """Wrap data in a pkt-line. 

363 

364 Args: 

365 data: The data to wrap, as a str or None. 

366 Returns: The data prefixed with its length in pkt-line format; if data was 

367 None, returns the flush-pkt ('0000'). 

368 """ 

369 if data is None: 

370 return b"0000" 

371 return f"{len(data) + 4:04x}".encode("ascii") + data 

372 

373 

374def pkt_seq(*seq: bytes | None) -> bytes: 

375 """Wrap a sequence of data in pkt-lines. 

376 

377 Args: 

378 seq: An iterable of strings to wrap. 

379 """ 

380 return b"".join([pkt_line(s) for s in seq]) + pkt_line(None) 

381 

382 

383_HEX_DIGITS = frozenset(b"0123456789abcdefABCDEF") 

384 

385 

386def _parse_pkt_line_length(sizestr: bytes) -> int: 

387 """Parse the four-byte length prefix of a pkt-line. 

388 

389 Mirrors git's ``packet_length()``: the prefix must be exactly four 

390 hexadecimal digits. ``int(sizestr, 16)`` on its own also accepts a leading 

391 ``+``/``-``, surrounding whitespace, an ``0x`` prefix and digit-group 

392 underscores, none of which git allows. Accepting a ``-``-prefixed prefix is 

393 the dangerous case: the resulting negative length turns the following 

394 ``read(size - 4)`` into a ``read()`` with a negative argument (which reads 

395 the rest of the stream into memory) and makes the incremental 

396 :class:`PktLineParser` loop without ever consuming its buffer. 

397 

398 Args: 

399 sizestr: The four raw length bytes read from the wire. 

400 Returns: The length encoded by the prefix. 

401 

402 Raises: 

403 GitProtocolError: if the prefix is not four hexadecimal digits. 

404 """ 

405 if len(sizestr) != 4 or not _HEX_DIGITS.issuperset(sizestr): 

406 raise GitProtocolError(f"Invalid pkt-line length prefix: {sizestr!r}") 

407 return int(sizestr, 16) 

408 

409 

410class Protocol: 

411 """Class for interacting with a remote git process over the wire. 

412 

413 Parts of the git wire protocol use 'pkt-lines' to communicate. A pkt-line 

414 consists of the length of the line as a 4-byte hex string, followed by the 

415 payload data. The length includes the 4-byte header. The special line 

416 '0000' indicates the end of a section of input and is called a 'flush-pkt'. 

417 

418 For details on the pkt-line format, see the cgit distribution: 

419 Documentation/technical/protocol-common.txt 

420 """ 

421 

422 def __init__( 

423 self, 

424 read: Callable[[int], bytes], 

425 write: Callable[[bytes], int | None], 

426 close: Callable[[], None] | None = None, 

427 report_activity: Callable[[int, str], None] | None = None, 

428 shutdown_write: Callable[[], None] | None = None, 

429 ) -> None: 

430 """Initialize Protocol. 

431 

432 Args: 

433 read: Function to read bytes from the transport 

434 write: Function to write bytes to the transport 

435 close: Optional function to close the transport 

436 report_activity: Optional function to report activity 

437 shutdown_write: Optional function to half-close the write side of 

438 the transport once the request has been fully sent 

439 """ 

440 self.read = read 

441 self.write = write 

442 self._close = close 

443 self.report_activity = report_activity 

444 self._shutdown_write = shutdown_write 

445 self._readahead: BytesIO | None = None 

446 

447 def shutdown_write(self) -> None: 

448 """Half-close the write side of the transport, if supported. 

449 

450 Signals to the peer that no more data will be sent. On Windows a 

451 server that closes a socket with client data still unread aborts with 

452 an RST rather than a clean FIN; letting it read to EOF avoids that. 

453 """ 

454 if self._shutdown_write is not None: 

455 self._shutdown_write() 

456 self._shutdown_write = None 

457 

458 def close(self) -> None: 

459 """Close the underlying transport if a close function was provided.""" 

460 if self._close: 

461 self._close() 

462 self._close = None # Prevent double-close 

463 

464 def __del__(self) -> None: 

465 """Ensure transport is closed when Protocol is garbage collected.""" 

466 if self._close is not None: 

467 import warnings 

468 

469 warnings.warn( 

470 f"unclosed Protocol {self!r}", 

471 ResourceWarning, 

472 stacklevel=2, 

473 source=self, 

474 ) 

475 try: 

476 self.close() 

477 except Exception: 

478 # Ignore errors during cleanup 

479 pass 

480 

481 def __enter__(self) -> Self: 

482 """Enter context manager.""" 

483 return self 

484 

485 def __exit__( 

486 self, 

487 exc_type: type[BaseException] | None, 

488 exc_val: BaseException | None, 

489 exc_tb: types.TracebackType | None, 

490 ) -> None: 

491 """Exit context manager and close transport.""" 

492 self.close() 

493 

494 def read_pkt_line(self) -> bytes | None: 

495 """Reads a pkt-line from the remote git process. 

496 

497 This method may read from the readahead buffer; see unread_pkt_line. 

498 

499 Returns: The next string from the stream, without the length prefix, or 

500 None for a flush-pkt ('0000') or delim-pkt ('0001'). 

501 """ 

502 if self._readahead is None: 

503 read = self.read 

504 else: 

505 read = self._readahead.read 

506 self._readahead = None 

507 

508 try: 

509 sizestr = read(4) 

510 if not sizestr: 

511 raise HangupException 

512 size = _parse_pkt_line_length(sizestr) 

513 if size == 0 or size == 1: # flush-pkt or delim-pkt 

514 if self.report_activity: 

515 self.report_activity(4, "read") 

516 logger.debug("git< %s", sizestr.decode("ascii")) 

517 return None 

518 if size < 4: 

519 raise GitProtocolError(f"Invalid pkt-line length: {size:04x}") 

520 if self.report_activity: 

521 self.report_activity(size, "read") 

522 pkt_contents = read(size - 4) 

523 except ConnectionResetError as exc: 

524 raise HangupException from exc 

525 except OSError as exc: 

526 raise GitProtocolError(str(exc)) from exc 

527 else: 

528 if len(pkt_contents) + 4 != size: 

529 raise GitProtocolError( 

530 f"Length of pkt read {len(pkt_contents) + 4:04x} does not match length prefix {size:04x}" 

531 ) 

532 # Log the packet contents (truncate if too long for readability) 

533 if len(pkt_contents) > 80: 

534 logger.debug( 

535 "git< %s... (%d bytes)", pkt_contents[:80], len(pkt_contents) 

536 ) 

537 else: 

538 logger.debug("git< %s", pkt_contents) 

539 return pkt_contents 

540 

541 def eof(self) -> bool: 

542 """Test whether the protocol stream has reached EOF. 

543 

544 Note that this refers to the actual stream EOF and not just a 

545 flush-pkt. 

546 

547 Returns: True if the stream is at EOF, False otherwise. 

548 """ 

549 try: 

550 next_line = self.read_pkt_line() 

551 except HangupException: 

552 return True 

553 self.unread_pkt_line(next_line) 

554 return False 

555 

556 def unread_pkt_line(self, data: bytes | None) -> None: 

557 """Unread a single line of data into the readahead buffer. 

558 

559 This method can be used to unread a single pkt-line into a fixed 

560 readahead buffer. 

561 

562 Args: 

563 data: The data to unread, without the length prefix. 

564 

565 Raises: 

566 ValueError: If more than one pkt-line is unread. 

567 """ 

568 if self._readahead is not None: 

569 raise ValueError("Attempted to unread multiple pkt-lines.") 

570 self._readahead = BytesIO(pkt_line(data)) 

571 

572 def read_pkt_seq(self) -> Iterable[bytes]: 

573 """Read a sequence of pkt-lines from the remote git process. 

574 

575 Returns: Yields each line of data up to but not including the next 

576 flush-pkt. 

577 """ 

578 pkt = self.read_pkt_line() 

579 while pkt: 

580 yield pkt 

581 pkt = self.read_pkt_line() 

582 

583 def write_pkt_line(self, line: bytes | None) -> None: 

584 """Sends a pkt-line to the remote git process. 

585 

586 Args: 

587 line: A string containing the data to send, without the length 

588 prefix. 

589 """ 

590 try: 

591 # Log before converting to pkt format 

592 if line is None: 

593 logger.debug("git> 0000") 

594 elif len(line) > 80: 

595 logger.debug("git> %s... (%d bytes)", line[:80], len(line)) 

596 else: 

597 logger.debug("git> %s", line) 

598 

599 line = pkt_line(line) 

600 self.write(line) 

601 if self.report_activity: 

602 self.report_activity(len(line), "write") 

603 except OSError as exc: 

604 raise GitProtocolError(str(exc)) from exc 

605 

606 def write_sideband(self, channel: int, blob: bytes) -> None: 

607 """Write multiplexed data to the sideband. 

608 

609 Args: 

610 channel: An int specifying the channel to write to. 

611 blob: A blob of data (as a string) to send on this channel. 

612 """ 

613 # a pktline can be a max of 65520. a sideband line can therefore be 

614 # 65520-5 = 65515 

615 # WTF: Why have the len in ASCII, but the channel in binary. 

616 while blob: 

617 self.write_pkt_line(bytes(bytearray([channel])) + blob[:65515]) 

618 blob = blob[65515:] 

619 

620 def send_cmd(self, cmd: bytes, *args: bytes) -> None: 

621 """Send a command and some arguments to a git server. 

622 

623 Only used for the TCP git protocol (git://). 

624 

625 Args: 

626 cmd: The remote service to access. 

627 args: List of arguments to send to remove service. 

628 """ 

629 self.write_pkt_line(format_cmd_pkt(cmd, *args)) 

630 

631 def read_cmd(self) -> tuple[bytes, list[bytes]]: 

632 """Read a command and some arguments from the git client. 

633 

634 Only used for the TCP git protocol (git://). 

635 

636 Returns: A tuple of (command, [list of arguments]). 

637 """ 

638 line = self.read_pkt_line() 

639 if line is None: 

640 raise GitProtocolError("Expected command, got flush packet") 

641 return parse_cmd_pkt(line) 

642 

643 

644_RBUFSIZE = 65536 # 64KB buffer for better network I/O performance 

645 

646 

647class ReceivableProtocol(Protocol): 

648 """Variant of Protocol that allows reading up to a size without blocking. 

649 

650 This class has a recv() method that behaves like socket.recv() in addition 

651 to a read() method. 

652 

653 If you want to read n bytes from the wire and block until exactly n bytes 

654 (or EOF) are read, use read(n). If you want to read at most n bytes from 

655 the wire but don't care if you get less, use recv(n). Note that recv(n) 

656 will still block until at least one byte is read. 

657 """ 

658 

659 def __init__( 

660 self, 

661 recv: Callable[[int], bytes], 

662 write: Callable[[bytes], int | None], 

663 close: Callable[[], None] | None = None, 

664 report_activity: Callable[[int, str], None] | None = None, 

665 rbufsize: int = _RBUFSIZE, 

666 ) -> None: 

667 """Initialize ReceivableProtocol. 

668 

669 Args: 

670 recv: Function to receive bytes from the transport 

671 write: Function to write bytes to the transport 

672 close: Optional function to close the transport 

673 report_activity: Optional function to report activity 

674 rbufsize: Read buffer size 

675 """ 

676 super().__init__(self.read, write, close=close, report_activity=report_activity) 

677 self._recv = recv 

678 self._rbuf = BytesIO() 

679 self._rbufsize = rbufsize 

680 

681 def read(self, size: int) -> bytes: 

682 """Read bytes from the socket. 

683 

684 Args: 

685 size: Number of bytes to read 

686 

687 Returns: 

688 Bytes read from socket 

689 """ 

690 # From _fileobj.read in socket.py in the Python 2.6.5 standard library, 

691 # with the following modifications: 

692 # - omit the size <= 0 branch 

693 # - seek back to start rather than 0 in case some buffer has been 

694 # consumed. 

695 # - use SEEK_END instead of the magic number. 

696 # Copyright (c) 2001-2010 Python Software Foundation; All Rights 

697 # Reserved 

698 # Licensed under the Python Software Foundation License. 

699 # TODO: see if buffer is more efficient than cBytesIO. 

700 assert size > 0 

701 

702 # Our use of BytesIO rather than lists of string objects returned by 

703 # recv() minimizes memory usage and fragmentation that occurs when 

704 # rbufsize is large compared to the typical return value of recv(). 

705 buf = self._rbuf 

706 start = buf.tell() 

707 buf.seek(0, SEEK_END) 

708 # buffer may have been partially consumed by recv() 

709 buf_len = buf.tell() - start 

710 if buf_len >= size: 

711 # Already have size bytes in our buffer? Extract and return. 

712 buf.seek(start) 

713 rv = buf.read(size) 

714 self._rbuf = BytesIO() 

715 self._rbuf.write(buf.read()) 

716 self._rbuf.seek(0) 

717 return rv 

718 

719 self._rbuf = BytesIO() # reset _rbuf. we consume it via buf. 

720 while True: 

721 left = size - buf_len 

722 # recv() will malloc the amount of memory given as its 

723 # parameter even though it often returns much less data 

724 # than that. The returned data string is short lived 

725 # as we copy it into a BytesIO and free it. This avoids 

726 # fragmentation issues on many platforms. 

727 data = self._recv(left) 

728 if not data: 

729 break 

730 n = len(data) 

731 if n == size and not buf_len: 

732 # Shortcut. Avoid buffer data copies when: 

733 # - We have no data in our buffer. 

734 # AND 

735 # - Our call to recv returned exactly the 

736 # number of bytes we were asked to read. 

737 return data 

738 if n == left: 

739 buf.write(data) 

740 del data # explicit free 

741 break 

742 assert n <= left, f"_recv({left}) returned {n} bytes" 

743 buf.write(data) 

744 buf_len += n 

745 del data # explicit free 

746 # assert buf_len == buf.tell() 

747 buf.seek(start) 

748 return buf.read() 

749 

750 def recv(self, size: int) -> bytes: 

751 """Receive bytes from the socket with buffering. 

752 

753 Args: 

754 size: Maximum number of bytes to receive 

755 

756 Returns: 

757 Bytes received from socket 

758 """ 

759 assert size > 0 

760 

761 buf = self._rbuf 

762 start = buf.tell() 

763 buf.seek(0, SEEK_END) 

764 buf_len = buf.tell() 

765 buf.seek(start) 

766 

767 left = buf_len - start 

768 if not left: 

769 # only read from the wire if our read buffer is exhausted 

770 data = self._recv(self._rbufsize) 

771 if len(data) == size: 

772 # shortcut: skip the buffer if we read exactly size bytes 

773 return data 

774 buf = BytesIO() 

775 buf.write(data) 

776 buf.seek(0) 

777 del data # explicit free 

778 self._rbuf = buf 

779 return buf.read(size) 

780 

781 

782def extract_capabilities(text: bytes) -> tuple[bytes, list[bytes]]: 

783 """Extract a capabilities list from a string, if present. 

784 

785 Args: 

786 text: String to extract from 

787 Returns: Tuple with text with capabilities removed and list of capabilities 

788 """ 

789 if b"\0" not in text: 

790 return text, [] 

791 text, capabilities = text.rstrip().split(b"\0") 

792 return (text, capabilities.strip().split(b" ")) 

793 

794 

795def extract_want_line_capabilities(text: bytes) -> tuple[bytes, list[bytes]]: 

796 """Extract a capabilities list from a want line, if present. 

797 

798 Note that want lines have capabilities separated from the rest of the line 

799 by a space instead of a null byte. Thus want lines have the form: 

800 

801 want obj-id cap1 cap2 ... 

802 

803 Args: 

804 text: Want line to extract from 

805 Returns: Tuple with text with capabilities removed and list of capabilities 

806 """ 

807 split_text = text.rstrip().split(b" ") 

808 if len(split_text) < 3: 

809 return text, [] 

810 return (b" ".join(split_text[:2]), split_text[2:]) 

811 

812 

813def ack_type(capabilities: Iterable[bytes]) -> int: 

814 """Extract the ack type from a capabilities list.""" 

815 if b"multi_ack_detailed" in capabilities: 

816 return MULTI_ACK_DETAILED 

817 elif b"multi_ack" in capabilities: 

818 return MULTI_ACK 

819 return SINGLE_ACK 

820 

821 

822def find_capability( 

823 capabilities: Iterable[bytes], *capability_names: bytes 

824) -> bytes | None: 

825 """Find a capability value in a list of capabilities. 

826 

827 This function looks for capabilities that may include arguments after an equals sign 

828 and returns only the value part (after the '='). For capabilities without values, 

829 returns the capability name itself. 

830 

831 Args: 

832 capabilities: List of capability strings 

833 capability_names: Capability name(s) to search for 

834 

835 Returns: 

836 The value after '=' if found, or the capability name if no '=', or None if not found 

837 

838 Example: 

839 >>> caps = [b'filter=blob:none', b'agent=git/2.0', b'thin-pack'] 

840 >>> find_capability(caps, b'filter') 

841 b'blob:none' 

842 >>> find_capability(caps, b'thin-pack') 

843 b'thin-pack' 

844 >>> find_capability(caps, b'missing') 

845 None 

846 """ 

847 for cap in capabilities: 

848 for name in capability_names: 

849 if cap == name: 

850 return cap 

851 elif cap.startswith(name + b"="): 

852 return cap[len(name) + 1 :] 

853 return None 

854 

855 

856class BufferedPktLineWriter: 

857 """Writer that wraps its data in pkt-lines and has an independent buffer. 

858 

859 Consecutive calls to write() wrap the data in a pkt-line and then buffers 

860 it until enough lines have been written such that their total length 

861 (including length prefix) reach the buffer size. 

862 """ 

863 

864 def __init__( 

865 self, write: Callable[[bytes], int | None], bufsize: int = 65515 

866 ) -> None: 

867 """Initialize the BufferedPktLineWriter. 

868 

869 Args: 

870 write: A write callback for the underlying writer. 

871 bufsize: The internal buffer size, including length prefixes. 

872 """ 

873 self._write = write 

874 self._bufsize = bufsize 

875 self._wbuf = BytesIO() 

876 self._buflen = 0 

877 

878 def write(self, data: bytes) -> None: 

879 """Write data, wrapping it in a pkt-line.""" 

880 line = pkt_line(data) 

881 line_len = len(line) 

882 over = self._buflen + line_len - self._bufsize 

883 if over >= 0: 

884 start = line_len - over 

885 self._wbuf.write(line[:start]) 

886 self.flush() 

887 else: 

888 start = 0 

889 saved = line[start:] 

890 self._wbuf.write(saved) 

891 self._buflen += len(saved) 

892 

893 def flush(self) -> None: 

894 """Flush all data from the buffer.""" 

895 data = self._wbuf.getvalue() 

896 if data: 

897 self._write(data) 

898 self._len = 0 

899 self._wbuf = BytesIO() 

900 

901 

902class PktLineParser: 

903 """Packet line parser that hands completed packets off to a callback.""" 

904 

905 def __init__(self, handle_pkt: Callable[[bytes | None], None]) -> None: 

906 """Initialize PktLineParser. 

907 

908 Args: 

909 handle_pkt: Callback function to handle completed packets 

910 """ 

911 self.handle_pkt = handle_pkt 

912 self._readahead = BytesIO() 

913 

914 def parse(self, data: bytes) -> None: 

915 """Parse a fragment of data and call back for any completed packets.""" 

916 self._readahead.write(data) 

917 buf = self._readahead.getvalue() 

918 if len(buf) < 4: 

919 return 

920 # Track an offset rather than reslicing buf per pkt-line: slicing copies 

921 # the remainder each time, which makes a single call quadratic in the 

922 # number of lines it contains. 

923 pos = 0 

924 while len(buf) - pos >= 4: 

925 size = _parse_pkt_line_length(buf[pos : pos + 4]) 

926 if size == 0: 

927 self.handle_pkt(None) 

928 pos += 4 

929 elif size < 4: 

930 raise GitProtocolError(f"Invalid pkt-line length: {size:04x}") 

931 elif size <= len(buf) - pos: 

932 self.handle_pkt(buf[pos + 4 : pos + size]) 

933 pos += size 

934 else: 

935 break 

936 self._readahead = BytesIO() 

937 self._readahead.write(buf[pos:]) 

938 

939 def get_tail(self) -> bytes: 

940 """Read back any unused data.""" 

941 return self._readahead.getvalue() 

942 

943 

944def format_capability_line(capabilities: Iterable[bytes]) -> bytes: 

945 """Format a capabilities list for the wire protocol. 

946 

947 Args: 

948 capabilities: List of capability strings 

949 

950 Returns: 

951 Space-separated capabilities as bytes 

952 """ 

953 return b"".join([b" " + c for c in capabilities]) 

954 

955 

956def format_ref_line( 

957 ref: bytes, sha: bytes, capabilities: Sequence[bytes] | None = None 

958) -> bytes: 

959 """Format a ref advertisement line. 

960 

961 Args: 

962 ref: Reference name 

963 sha: SHA hash 

964 capabilities: Optional list of capabilities 

965 

966 Returns: 

967 Formatted ref line 

968 """ 

969 if capabilities is None: 

970 return sha + b" " + ref + b"\n" 

971 else: 

972 return sha + b" " + ref + b"\0" + format_capability_line(capabilities) + b"\n" 

973 

974 

975def format_shallow_line(sha: bytes) -> bytes: 

976 """Format a shallow line. 

977 

978 Args: 

979 sha: SHA to mark as shallow 

980 

981 Returns: 

982 Formatted shallow line 

983 """