Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/docutils/parsers/rst/states.py: 75%

1# $Id: states.py 10401 2026-09-16 16:23:03Z milde $

2# Author: David Goodger <goodger@python.org>

3# Copyright: This module has been placed in the public domain.

5"""

6This is the ``docutils.parsers.rst.states`` module, the core of

7the reStructuredText parser. It defines the following:

9:Classes:

10 - `RSTStateMachine`: reStructuredText parser's entry point.

11 - `NestedStateMachine`: recursive StateMachine.

12 - `RSTState`: reStructuredText State superclass.

13 - `Inliner`: For parsing inline markup.

14 - `Body`: Generic classifier of the first line of a block.

15 - `SpecializedBody`: Superclass for compound element members.

16 - `BulletList`: Second and subsequent bullet_list list_items

17 - `DefinitionList`: Second+ definition_list_items.

18 - `EnumeratedList`: Second+ enumerated_list list_items.

19 - `FieldList`: Second+ fields.

20 - `OptionList`: Second+ option_list_items.

21 - `RFC2822List`: Second+ RFC2822-style fields.

22 - `ExtensionOptions`: Parses directive option fields.

23 - `Explicit`: Second+ explicit markup constructs.

24 - `SubstitutionDef`: For embedded directives in substitution definitions.

25 - `Text`: Classifier of second line of a text block.

26 - `SpecializedText`: Superclass for continuation lines of Text-variants.

27 - `Definition`: Second line of potential definition_list_item.

28 - `Line`: Second line of overlined section title or transition marker.

29 - `Struct`: obsolete, use `types.SimpleNamespace`.

31:Exception classes:

32 - `MarkupError`

33 - `ParserError`

34 - `MarkupMismatch`

36:Functions:

37 - `escape2null()`: Return a string, escape-backslashes converted to nulls.

38 - `unescape()`: Return a string, nulls removed or restored to backslashes.

40:Attributes:

41 - `state_classes`: set of State classes used with `RSTStateMachine`.

43Parser Overview

44===============

46The reStructuredText parser is implemented as a recursive state machine,

47examining its input one line at a time. To understand how the parser works,

48please first become familiar with the `docutils.statemachine` module. In the

49description below, references are made to classes defined in this module;

50please see the individual classes for details.

52Parsing proceeds as follows:

541. The state machine examines each line of input, checking each of the

55 transition patterns of the state `Body`, in order, looking for a match.

56 The implicit transitions (blank lines and indentation) are checked before

57 any others. The 'text' transition is a catch-all (matches anything).

592. The method associated with the matched transition pattern is called.

61 A. Some transition methods are self-contained, appending elements to the

62 document tree (`Body.doctest` parses a doctest block). The parser's

63 current line index is advanced to the end of the element, and parsing

64 continues with step 1.

66 B. Other transition methods trigger the creation of a nested state machine,

67 whose job is to parse a compound construct ('indent' does a block quote,

68 'bullet' does a bullet list, 'overline' does a section [first checking

69 for a valid section header], etc.).

71 - In the case of lists and explicit markup, a one-off state machine is

72 created and run to parse contents of the first item.

74 - A new state machine is created and its initial state is set to the

75 appropriate specialized state (`BulletList` in the case of the

76 'bullet' transition; see `SpecializedBody` for more detail). This

77 state machine is run to parse the compound element (or series of

78 explicit markup elements), and returns as soon as a non-member element

79 is encountered. For example, the `BulletList` state machine ends as

80 soon as it encounters an element which is not a list item of that

81 bullet list. The optional omission of inter-element blank lines is

82 enabled by this nested state machine.

84 - The current line index is advanced to the end of the elements parsed,

85 and parsing continues with step 1.

87 C. The result of the 'text' transition depends on the next line of text.

88 The current state is changed to `Text`, under which the second line is

89 examined. If the second line is:

91 - Indented: The element is a definition list item, and parsing proceeds

92 similarly to step 2.B, using the `DefinitionList` state.

94 - A line of uniform punctuation characters: The element is a section

95 header; again, parsing proceeds as in step 2.B, and `Body` is still

96 used.

98 - Anything else: The element is a paragraph, which is examined for

99 inline markup and appended to the parent element. Processing

100 continues with step 1.

101"""

102

103from __future__ import annotations

104

105__docformat__ = 'reStructuredText'

106

107import re

108from types import FunctionType, MethodType

109from types import SimpleNamespace as Struct

110import warnings

111

112from docutils import nodes, statemachine, utils

113from docutils import ApplicationError, DataError

114from docutils.statemachine import StateMachineWS, StateWS

115from docutils.nodes import fully_normalize_name as normalize_name

116from docutils.nodes import unescape, whitespace_normalize_name

117import docutils.parsers.rst

118from docutils.parsers.rst import directives, languages, tableparser, roles

119from docutils.utils import escape2null, column_width, strip_combining_chars

120from docutils.utils import punctuation_chars, urischemes

121from docutils.utils import split_escaped_whitespace

122from docutils.utils._roman_numerals import (InvalidRomanNumeralError,

123 RomanNumeral)

124from docutils.utils.code_analyzer import tag_code, LexerError

125

126TYPE_CHECKING = False

127if TYPE_CHECKING:

128 from docutils.statemachine import StringList

129

130

131class MarkupError(DataError): pass

132class UnknownInterpretedRoleError(DataError): pass

133class InterpretedRoleNotImplementedError(DataError): pass

134class ParserError(ApplicationError): pass

135class MarkupMismatch(Exception): pass

136

137

138class RSTStateMachine(StateMachineWS):

139

140 """

141 reStructuredText's master StateMachine.

142

143 The entry point to reStructuredText parsing is the `run()` method.

144 """

145 section_level_offset: int = 0

146 """Correction term for section level determination in nested parsing.

147

148 Updated by `RSTState.nested_parse()` and used in

149 `RSTState.check_subsection()` to compensate differences when

150 nested parsing uses a detached base node with a document-wide

151 section title style hierarchy or the current node with a new,

152 independent title style hierarchy.

153 """

154

155 def run(self, input_lines, document, input_offset=0, match_titles=True,

156 inliner=None) -> None:

157 """

158 Parse `input_lines` and modify the `document` node in place.

159

160 Extend `StateMachineWS.run()`: set up parse-global data and

161 run the StateMachine.

162 """

163 self.language = languages.get_language(

164 document.settings.language_code, document.reporter)

165 self.match_titles = match_titles

166 if inliner is None:

167 inliner = Inliner()

168 inliner.init_customizations(document.settings)

169 # A collection of objects to share with nested parsers.

170 # The attributes `reporter`, `section_level`, and

171 # `section_bubble_up_kludge` will be removed in Docutils 2.0

172 self.memo = Struct(document=document,

173 reporter=document.reporter, # ignored

174 language=self.language,

175 title_styles=[],

176 section_level=0, # ignored

177 section_bubble_up_kludge=False, # ignored

178 inliner=inliner)

179 self.document = document

180 self.attach_observer(document.note_source)

181 self.reporter = self.document.reporter

182 self.node = document

183 results = StateMachineWS.run(self, input_lines, input_offset,

184 input_source=document['source'])

185 assert results == [], 'RSTStateMachine.run() results should be empty!'

186 self.node = self.memo = None # remove unneeded references

187

188

189class NestedStateMachine(RSTStateMachine):

190 """

191 StateMachine run from within other StateMachine runs, to parse nested

192 document structures.

193 """

194

195 def __init__(self, state_classes, initial_state,

196 debug=False, parent_state_machine=None) -> None:

197

198 self.parent_state_machine = parent_state_machine

199 """The instance of the parent state machine."""

200

201 super().__init__(state_classes, initial_state, debug)

202

203 def run(self, input_lines, input_offset, memo, node, match_titles=True):

204 """

205 Parse `input_lines` and populate `node`.

206

207 Extend `StateMachineWS.run()`: set up document-wide data.

208 """

209 self.match_titles = match_titles

210 self.memo = memo

211 self.document = memo.document

212 self.attach_observer(self.document.note_source)

213 self.language = memo.language

214 self.reporter = self.document.reporter

215 self.node = node

216 results = StateMachineWS.run(self, input_lines, input_offset)

217 assert results == [], ('NestedStateMachine.run() results should be '

218 'empty!')

219 return results

220

221

222class RSTState(StateWS):

223

224 """

225 reStructuredText State superclass.

226

227 Contains methods used by all State subclasses.

228 """

229

230 nested_sm = NestedStateMachine

231 nested_sm_cache = []

232

233 def __init__(self, state_machine: RSTStateMachine, debug=False) -> None:

234 self.nested_sm_kwargs = {'state_classes': state_classes,

235 'initial_state': 'Body'}

236 StateWS.__init__(self, state_machine, debug)

237

238 def runtime_init(self) -> None:

239 StateWS.runtime_init(self)

240 memo = self.state_machine.memo

241 self.memo = memo

242 self.document = memo.document

243 self.inliner = memo.inliner

244 self.reporter = self.document.reporter

245 # enable the reporter to determine source and source-line

246 if not hasattr(self.reporter, 'get_source_and_line'):

247 self.reporter.get_source_and_line = self.state_machine.get_source_and_line # noqa:E501

248

249 @property

250 def parent(self) -> nodes.Element | None:

251 return self.state_machine.node

252

253 @parent.setter

254 def parent(self, value: nodes.Element):

255 self.state_machine.node = value

256

257 def goto_line(self, abs_line_offset) -> None:

258 """

259 Jump to input line `abs_line_offset`, ignoring jumps past the end.

260 """

261 try:

262 self.state_machine.goto_line(abs_line_offset)

263 except EOFError:

264 pass

265

266 def no_match(self, context, transitions):

267 """

268 Override `StateWS.no_match` to generate a system message.

269

270 This code should never be run.

271 """

272 self.reporter.severe(

273 'Internal error: no transition pattern match. State: "%s"; '

274 'transitions: %s; context: %s; current line: %r.'

275 % (self.__class__.__name__, transitions, context,

276 self.state_machine.line))

277 return context, None, []

278

279 def bof(self, context):

280 """Called at beginning of file."""

281 return [], []

282

283 def nested_parse(self,

284 block: StringList,

285 input_offset: int,

286 node: nodes.Element|None = None,

287 match_titles: bool = False,

288 state_machine_class: StateMachineWS|None = None,

289 state_machine_kwargs: dict|None = None

290 ) -> int:

291 """

292 Parse the input `block` with a nested state-machine rooted at `node`.

293

294 :block:

295 reStructuredText source extract.

296 :input_offset:

297 Line number at start of the block.

298 :node:

299 Base node. Generated nodes will be appended to this node.

300 Default: the "current node" (`self.state_machine.node`).

301 :match_titles:

302 Allow section titles?

303 Caution: With a custom base node, this may lead to an invalid

304 or mixed up document tree. [#]_

305 :state_machine_class:

306 Default: `NestedStateMachine`.

307 :state_machine_kwargs:

308 Keyword arguments for the state-machine instantiation.

309 Default: `self.nested_sm_kwargs`.

310

311 Create a new state-machine instance if required.

312 Return new offset.

313

314 .. [#] See also ``test_parsers/test_rst/test_nested_parsing.py``

315 and Sphinx's `nested_parse_to_nodes()`__.

316

317 __ https://www.sphinx-doc.org/en/master/extdev/utils.html

318 #sphinx.util.parsing.nested_parse_to_nodes

319 """

320 if node is None:

321 node = self.state_machine.node

322 use_default = 0

323 if state_machine_class is None:

324 state_machine_class = self.nested_sm

325 use_default += 1

326 if state_machine_kwargs is None:

327 state_machine_kwargs = self.nested_sm_kwargs

328 use_default += 1

329 my_state_machine = None

330 if use_default == 2:

331 try:

332 # get cached state machine, prevent others from using it

333 my_state_machine = self.nested_sm_cache.pop()

334 except IndexError:

335 pass

336 if not my_state_machine:

337 my_state_machine = state_machine_class(

338 debug=self.debug,

339 parent_state_machine=self.state_machine,

340 **state_machine_kwargs)

341 # Check if we may use sections (with a caveat for custom nodes

342 # that may be dummies to collect children):

343 if (node == self.state_machine.node

344 and not isinstance(node, (nodes.document, nodes.section))):

345 match_titles = False # avoid invalid sections

346 if match_titles:

347 # Compensate mismatch of known title styles and number of

348 # parent sections of the base node if the document wide

349 # title styles are used with a detached base node or

350 # a new list of title styles with the current parent node:

351 l_node = len(node.section_hierarchy())

352 l_start = min(len(self.parent.section_hierarchy()),

353 len(self.memo.title_styles))

354 my_state_machine.section_level_offset = l_start - l_node

355

356 # run the state machine and populate `node`:

357 block_length = len(block)

358 my_state_machine.run(block, input_offset, self.memo,

359 node, match_titles)

360

361 if match_titles:

362 if node == self.state_machine.node:

363 # Pass on the new "current node" to parent state machines:

364 sm = self.state_machine

365 try:

366 while True:

367 sm.node = my_state_machine.node

368 sm = sm.parent_state_machine

369 except AttributeError:

370 pass

371 # clean up

372 new_offset = my_state_machine.abs_line_offset()

373 if use_default == 2:

374 self.nested_sm_cache.append(my_state_machine)

375 else:

376 my_state_machine.unlink()

377 # No `block.parent` implies disconnected -- lines aren't in sync:

378 if block.parent and (len(block) - block_length) != 0:

379 # Adjustment for block if modified in nested parse:

380 self.state_machine.next_line(len(block) - block_length)

381 return new_offset

382

383 def nested_list_parse(self, block, input_offset, node, initial_state,

384 blank_finish,

385 blank_finish_state=None,

386 extra_settings={},

387 match_titles=None, # deprecated, will be removed

388 state_machine_class=None,

389 state_machine_kwargs=None):

390 """

391 Parse the input `block` with a nested state-machine rooted at `node`.

392

393 Create a new StateMachine rooted at `node` and run it over the

394 input `block` (see also `nested_parse()`).

395 Also keep track of optional intermediate blank lines and the

396 required final one.

397

398 Return new offset and a boolean indicating whether there was a

399 blank final line.

400 """

401 if match_titles is not None:

402 warnings.warn('The "match_titles" argument of '

403 'parsers.rst.states.RSTState.nested_list_parse() '

404 'is ignored and will be removed in Docutils 2.0.',

405 PendingDeprecationWarning, stacklevel=2)

406 if state_machine_class is None:

407 state_machine_class = self.nested_sm

408 if state_machine_kwargs is None:

409 state_machine_kwargs = self.nested_sm_kwargs.copy()

410 state_machine_kwargs['initial_state'] = initial_state

411 my_state_machine = state_machine_class(

412 debug=self.debug,

413 parent_state_machine=self.state_machine,

414 **state_machine_kwargs)

415 if blank_finish_state is None:

416 blank_finish_state = initial_state

417 my_state_machine.states[blank_finish_state].blank_finish = blank_finish

418 for key, value in extra_settings.items():

419 setattr(my_state_machine.states[initial_state], key, value)

420 my_state_machine.run(block, input_offset, memo=self.memo, node=node)

421 blank_finish = my_state_machine.states[blank_finish_state].blank_finish

422 my_state_machine.unlink()

423 return my_state_machine.abs_line_offset(), blank_finish

424

425 def section(self, title, source, style, lineno, messages) -> None:

426 """Check for a valid subsection and create one if it checks out."""

427 if self.check_subsection(source, style, lineno):

428 self.new_subsection(title, lineno, messages)

429

430 def check_subsection(self, source, style, lineno) -> bool:

431 """

432 Check for a valid subsection header. Update section data in `memo`.

433

434 When a new section is reached that isn't a subsection of the current

435 section, set `self.parent` to the new section's parent section

436 (or the root node if the new section is a top-level section).

437 """

438 title_styles = self.memo.title_styles

439 parent_sections = self.parent.section_hierarchy()

440 # current section level: (0 root, 1 section, 2 subsection, ...)

441 oldlevel = (len(parent_sections)

442 + self.state_machine.section_level_offset)

443 # new section level:

444 try: # check for existing title style

445 newlevel = title_styles.index(style) + 1

446 except ValueError: # new title style

447 newlevel = len(title_styles) + 1

448 # The new level must not be deeper than an immediate child

449 # of the current level:

450 if newlevel > oldlevel + 1:

451 styles = ' '.join('/'.join(style) for style in title_styles)

452 self.parent += self.reporter.error(

453 'Inconsistent title style:'

454 f' skip from level {oldlevel} to {newlevel}.',

455 nodes.literal_block('', source),

456 nodes.paragraph('', f'Established title styles: {styles}'),

457 line=lineno)

458 return False

459 if newlevel <= oldlevel:

460 # new section is sibling or higher up in the section hierarchy

461 try:

462 new_parent = parent_sections[newlevel-oldlevel-1].parent

463 except IndexError:

464 styles = ' '.join('/'.join(style) for style in title_styles)

465 details = (f'The parent of level {newlevel} sections cannot'

466 ' be reached. The parser is at section level'

467 f' {oldlevel} but the current node has only'

468 f' {len(parent_sections)} parent section(s).'

469 '\nOne reason may be a high level'

470 ' section used in a directive that parses its'

471 ' content into a base node not attached to'

472 ' the document\n(up to Docutils 0.21,'

473 ' these sections were silently dropped).')

474 self.parent += self.reporter.error(

475 f'A level {newlevel} section cannot be used here.',

476 nodes.literal_block('', source),

477 nodes.paragraph('', f'Established title styles: {styles}'),

478 nodes.paragraph('', details),

479 line=lineno)

480 return False

481 self.parent = new_parent

482 self.memo.section_level = newlevel - 1

483 if newlevel > len(title_styles):

484 title_styles.append(style)

485 return True

486

487 def title_inconsistent(self, sourcetext, lineno):

488 # Ignored. Will be removed in Docutils 2.0.

489 error = self.reporter.error(

490 'Title level inconsistent:', nodes.literal_block('', sourcetext),

491 line=lineno)

492 return error

493

494 def new_subsection(self, title, lineno, messages):

495 """Append new subsection to document tree."""

496 section_node = nodes.section()

497 self.parent += section_node

498 textnodes, title_messages = self.inline_text(title, lineno)

499 titlenode = nodes.title(title, '', *textnodes)

500 name = normalize_name(titlenode.astext())

501 section_node['names'].append(name)

502 section_node += titlenode

503 section_node += messages

504 section_node += title_messages

505 self.document.note_implicit_target(section_node, section_node)

506 # Update state:

507 self.parent = section_node

508 self.memo.section_level += 1

509

510 def paragraph(self, lines, lineno):

511 """

512 Return a list (paragraph & messages) & a boolean: literal_block next?

513 """

514 data = '\n'.join(lines).rstrip()

515 if re.search(r'(?<!\\)(\\\\)*::$', data):

516 if len(data) == 2:

517 return [], 1

518 elif data[-3] in ' \n':

519 text = data[:-3].rstrip()

520 else:

521 text = data[:-1]

522 literalnext = 1

523 else:

524 text = data

525 literalnext = 0

526 textnodes, messages = self.inline_text(text, lineno)

527 p = nodes.paragraph(data, '', *textnodes)

528 p.source, p.line = self.state_machine.get_source_and_line(lineno)

529 return [p] + messages, literalnext

530

531 def inline_text(self, text, lineno):

532 """

533 Return 2 lists: nodes (text and inline elements), and system_messages.

534 """

535 nodes, messages = self.inliner.parse(text, lineno,

536 self.memo, self.parent)

537 return nodes, messages

538

539 def unindent_warning(self, node_name):

540 # the actual problem is one line below the current line

541 lineno = self.state_machine.abs_line_number() + 1

542 return self.reporter.warning('%s ends without a blank line; '

543 'unexpected unindent.' % node_name,

544 line=lineno)

545

546

547def build_regexp(definition, compile_patterns=True):

548 """

549 Build, compile and return a regular expression based on `definition`.

550

551 :Parameter: `definition`: a 4-tuple (group name, prefix, suffix, parts),

552 where "parts" is a list of regular expressions and/or regular

553 expression definitions to be joined into an or-group.

554 """

555 name, prefix, suffix, parts = definition

556 part_strings = []

557 for part in parts:

558 if isinstance(part, tuple):

559 part_strings.append(build_regexp(part, None))

560 else:

561 part_strings.append(part)

562 or_group = '|'.join(part_strings)

563 regexp = '%(prefix)s(?P<%(name)s>%(or_group)s)%(suffix)s' % locals()

564 if compile_patterns:

565 return re.compile(regexp)

566 else:

567 return regexp

568

569

570class Inliner:

571

572 """

573 Parse inline markup; call the `parse()` method.

574 """

575

576 def __init__(self) -> None:

577 self.implicit_dispatch = []

578 """List of (pattern, bound method) tuples, used by

579 `self.implicit_inline`."""

580

581 def init_customizations(self, settings) -> None:

582 # lookahead and look-behind expressions for inline markup rules

583 if getattr(settings, 'character_level_inline_markup', False):

584 start_string_prefix = '(^|(?<!\x00))'

585 end_string_suffix = ''

586 else:

587 start_string_prefix = ('(^|(?<=\\s|[%s%s]))' %

588 (punctuation_chars.openers,

589 punctuation_chars.delimiters))

590 end_string_suffix = ('($|(?=\\s|[\x00%s%s%s]))' %

591 (punctuation_chars.closing_delimiters,

592 punctuation_chars.delimiters,

593 punctuation_chars.closers))

594 args = locals().copy()

595 args.update(vars(self.__class__))

596

597 parts = ('initial_inline', start_string_prefix, '',

598 [

599 ('start', '', self.non_whitespace_after, # simple start-strings

600 [r'\*\*', # strong

601 r'\*(?!\*)', # emphasis but not strong

602 r'``', # literal

603 r'_`', # inline internal target

604 r'\|(?!\|)'] # substitution reference

605 ),

606 ('whole', '', end_string_suffix, # whole constructs

607 [ # reference name & end-string

608 r'(?P<refname>%s)(?P<refend>__?)' % self.simplename,

609 ('footnotelabel', r'\[', r'(?P<fnend>\]_)',

610 [r'[0-9]+', # manually numbered

611 r'\#(%s)?' % self.simplename, # auto-numbered (w/ label?)

612 r'\*', # auto-symbol

613 r'(?P<citationlabel>%s)' % self.simplename, # citation ref

614 ]

615 )

616 ]

617 ),

618 ('backquote', # interpreted text or phrase reference

619 '(?P<role>(:%s:)?)' % self.simplename, # optional role

620 self.non_whitespace_after,

621 ['`(?!`)'] # but not literal

622 )

623 ]

624 )

625 self.start_string_prefix = start_string_prefix

626 self.end_string_suffix = end_string_suffix

627 self.parts = parts

628

629 self.patterns = Struct(

630 initial=build_regexp(parts),

631 emphasis=re.compile(self.non_whitespace_escape_before

632 + r'(\*)' + end_string_suffix),

633 strong=re.compile(self.non_whitespace_escape_before

634 + r'(\*\*)' + end_string_suffix),

635 interpreted_or_phrase_ref=re.compile(

636 r"""

637 %(non_unescaped_whitespace_escape_before)s

638 (

639 `

640 (?P<suffix>

641 (?P<role>:%(simplename)s:)?

642 (?P<refend>__?)?

643 )

644 )

645 %(end_string_suffix)s

646 """ % args, re.VERBOSE),

647 embedded_link=re.compile(

648 r"""

649 (

650 (?:[ \n]+|^) # spaces or beginning of line/string

651 < # open bracket

652 %(non_whitespace_after)s

653 (([^<>]|\x00[<>])+) # anything but unescaped angle brackets

654 %(non_whitespace_escape_before)s

655 > # close bracket

656 )

657 $ # end of string

658 """ % args, re.VERBOSE),

659 literal=re.compile(self.non_whitespace_before + '(``)'

660 + end_string_suffix),

661 target=re.compile(self.non_whitespace_escape_before

662 + r'(`)' + end_string_suffix),

663 substitution_ref=re.compile(self.non_whitespace_escape_before

664 + r'(\|_{0,2})'

665 + end_string_suffix),

666 email=re.compile(self.email_pattern % args + '$',

667 re.VERBOSE),

668 uri=re.compile(

669 (r"""

670 %(start_string_prefix)s

671 (?P<whole>

672 (?P<absolute> # absolute URI

673 (?P<scheme> # scheme (http, ftp, mailto)

674 [a-zA-Z][a-zA-Z0-9.+-]*

675 )

676 :

677 (

678 ( # either:

679 (//?)? # hierarchical URI

680 %(uric)s* # URI characters

681 %(uri_end)s # final URI char

682 )

683 ( # optional query

684 \?%(uric)s*

685 %(uri_end)s

686 )?

687 ( # optional fragment

688 \#%(uric)s*

689 %(uri_end)s

690 )?

691 )

692 )

693 | # *OR*

694 (?P<email> # email address

695 """ + self.email_pattern + r"""

696 )

697 )

698 %(end_string_suffix)s

699 """) % args, re.VERBOSE),

700 pep=re.compile(

701 r"""

702 %(start_string_prefix)s

703 (

704 (pep-(?P<pepnum1>\d+)(.txt)?) # reference to source file

705 |

706 (PEP\s+(?P<pepnum2>\d+)) # reference by name

707 )

708 %(end_string_suffix)s""" % args, re.VERBOSE),

709 rfc=re.compile(

710 r"""

711 %(start_string_prefix)s

712 (RFC(-|\s+)?(?P<rfcnum>\d+))

713 %(end_string_suffix)s""" % args, re.VERBOSE))

714

715 self.implicit_dispatch.append((self.patterns.uri,

716 self.standalone_uri))

717 if settings.pep_references:

718 self.implicit_dispatch.append((self.patterns.pep,

719 self.pep_reference))

720 if settings.rfc_references:

721 self.implicit_dispatch.append((self.patterns.rfc,

722 self.rfc_reference))

723

724 def parse(self, text, lineno, memo, parent):

725 # Needs to be refactored for nested inline markup.

726 # Add nested_parse() method?

727 """

728 Return 2 lists: nodes (text and inline elements), and system_messages.

729

730 Using `self.patterns.initial`, a pattern which matches start-strings

731 (emphasis, strong, interpreted, phrase reference, literal,

732 substitution reference, and inline target) and complete constructs

733 (simple reference, footnote reference), search for a candidate. When

734 one is found, check for validity (e.g., not a quoted '*' character).

735 If valid, search for the corresponding end string if applicable, and

736 check it for validity. If not found or invalid, generate a warning

737 and ignore the start-string. Implicit inline markup (e.g. standalone

738 URIs) is found last.

739

740 :text: source string

741 :lineno: absolute line number, cf. `statemachine.get_source_and_line()`

742 """

743 self.document = memo.document

744 self.language = memo.language

745 self.reporter = self.document.reporter

746 self.parent = parent

747 pattern_search = self.patterns.initial.search

748 dispatch = self.dispatch

749 remaining = escape2null(text)

750 processed = []

751 unprocessed = []

752 messages = []

753 while remaining:

754 match = pattern_search(remaining)

755 if match:

756 groups = match.groupdict()

757 method = dispatch[groups['start'] or groups['backquote']

758 or groups['refend'] or groups['fnend']]

759 before, inlines, remaining, sysmessages = method(self, match,

760 lineno)

761 unprocessed.append(before)

762 messages += sysmessages

763 if inlines:

764 processed += self.implicit_inline(''.join(unprocessed),

765 lineno)

766 processed += inlines

767 unprocessed = []

768 else:

769 break

770 remaining = ''.join(unprocessed) + remaining

771 if remaining:

772 processed += self.implicit_inline(remaining, lineno)

773 return processed, messages

774

775 # Inline object recognition

776 # -------------------------

777 # See also init_customizations().

778 non_whitespace_before = r'(?<!\s)'

779 non_whitespace_escape_before = r'(?<![\s\x00])'

780 non_unescaped_whitespace_escape_before = r'(?<!(?<!\x00)[\s\x00])'

781 non_whitespace_after = r'(?!\s)'

782 # Alphanumerics with isolated internal [-._+:] chars (i.e. not 2 together):

783 simplename = r'(?:(?!_)\w)+(?:[-._+:](?:(?!_)\w)+)*'

784 # Valid URI characters (see RFC 2396 & RFC 2732);

785 # final \x00 allows backslash escapes in URIs:

786 uric = r"""[-_.!~*'()[\];/:@&=+$,%a-zA-Z0-9\x00]"""

787 # Delimiter indicating the end of a URI (not part of the URI):

788 uri_end_delim = r"""[>]"""

789 # Last URI character; same as uric but no punctuation:

790 urilast = r"""[_~*/=+a-zA-Z0-9]"""

791 # End of a URI (either 'urilast' or 'uric followed by a

792 # uri_end_delim'):

793 uri_end = r"""(?:%(urilast)s|%(uric)s(?=%(uri_end_delim)s))""" % locals()

794 emailc = r"""[-_!~*'{|}/#?^`&=+$%a-zA-Z0-9\x00]"""

795 email_pattern = r"""

796 %(emailc)s+(?:\.%(emailc)s+)* # name

797 (?<!\x00)@ # at

798 %(emailc)s+(?:\.%(emailc)s*)* # host

799 %(uri_end)s # final URI char

800 """

801

802 def quoted_start(self, match):

803 """Test if inline markup start-string is 'quoted'.

804

805 'Quoted' in this context means the start-string is enclosed in a pair

806 of matching opening/closing delimiters (not necessarily quotes)

807 or at the end of the match.

808 """

809 string = match.string

810 start = match.start()

811 if start == 0: # start-string at beginning of text

812 return False

813 prestart = string[start - 1]

814 try:

815 poststart = string[match.end()]

816 except IndexError: # start-string at end of text

817 return True # not "quoted" but no markup start-string either

818 return punctuation_chars.match_chars(prestart, poststart)

819

820 def inline_obj(self, match, lineno, end_pattern, nodeclass,

821 restore_backslashes=False):

822 string = match.string

823 matchstart = match.start('start')

824 matchend = match.end('start')

825 if self.quoted_start(match):

826 return string[:matchend], [], string[matchend:], [], ''

827 endmatch = end_pattern.search(string[matchend:])

828 if endmatch and endmatch.start(1): # 1 or more chars

829 text = endmatch.string[:endmatch.start(1)]

830 if restore_backslashes:

831 text = unescape(text, True)

832 textend = matchend + endmatch.end(1)

833 rawsource = unescape(string[matchstart:textend], True)

834 node = nodeclass(rawsource, text)

835 return (string[:matchstart], [node],

836 string[textend:], [], endmatch.group(1))

837 role = 'target' if string[matchstart] == '_' else nodeclass.__name__

838 msg = self.reporter.warning(

839 f'Inline {role} start-string without end-string.',

840 line=lineno)

841 text = unescape(string[matchstart:matchend], True)

842 prb = self.problematic(text, text, msg)

843 return string[:matchstart], [prb], string[matchend:], [msg], ''

844

845 def problematic(self, text, rawsource, message):

846 msgid = self.document.set_id(message, self.parent)

847 problematic = nodes.problematic(rawsource, text, refid=msgid)

848 prbid = self.document.set_id(problematic)

849 message.add_backref(prbid)

850 return problematic

851

852 def emphasis(self, match, lineno):

853 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

854 match, lineno, self.patterns.emphasis, nodes.emphasis)

855 return before, inlines, remaining, sysmessages

856

857 def strong(self, match, lineno):

858 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

859 match, lineno, self.patterns.strong, nodes.strong)

860 return before, inlines, remaining, sysmessages

861

862 def interpreted_or_phrase_ref(self, match, lineno):

863 end_pattern = self.patterns.interpreted_or_phrase_ref

864 string = match.string

865 matchstart = match.start('backquote')

866 matchend = match.end('backquote')

867 rolestart = match.start('role')

868 role = match.group('role')

869 position = ''

870 if role:

871 role = role[1:-1]

872 position = 'prefix'

873 elif self.quoted_start(match):

874 return string[:matchend], [], string[matchend:], []

875 endmatch = end_pattern.search(string[matchend:])

876 if endmatch and endmatch.start(1): # 1 or more chars

877 textend = matchend + endmatch.end()

878 if endmatch.group('role'):

879 if role:

880 msg = self.reporter.warning(

881 'Multiple roles in interpreted text (both '

882 'prefix and suffix present; only one allowed).',

883 line=lineno)

884 text = unescape(string[rolestart:textend], True)

885 prb = self.problematic(text, text, msg)

886 return string[:rolestart], [prb], string[textend:], [msg]

887 role = endmatch.group('suffix')[1:-1]

888 position = 'suffix'

889 escaped = endmatch.string[:endmatch.start(1)]

890 rawsource = unescape(string[matchstart:textend], True)

891 if rawsource[-1:] == '_':

892 if role:

893 msg = self.reporter.warning(

894 'Mismatch: both interpreted text role %s and '

895 'reference suffix.' % position, line=lineno)

896 text = unescape(string[rolestart:textend], True)

897 prb = self.problematic(text, text, msg)

898 return string[:rolestart], [prb], string[textend:], [msg]

899 return self.phrase_ref(string[:matchstart], string[textend:],

900 rawsource, escaped)

901 else:

902 rawsource = unescape(string[rolestart:textend], True)

903 nodelist, messages = self.interpreted(rawsource, escaped, role,

904 lineno)

905 return (string[:rolestart], nodelist,

906 string[textend:], messages)

907 msg = self.reporter.warning(

908 'Inline interpreted text or phrase reference start-string '

909 'without end-string.', line=lineno)

910 text = unescape(string[matchstart:matchend], True)

911 prb = self.problematic(text, text, msg)

912 return string[:matchstart], [prb], string[matchend:], [msg]

913

914 def phrase_ref(self, before, after, rawsource, escaped, text=None):

915 # `text` is ignored (since 0.16)

916 match = self.patterns.embedded_link.search(escaped)

917 if match: # embedded <URI> or <alias_>

918 text = escaped[:match.start(0)]

919 unescaped = unescape(text)

920 rawtext = unescape(text, True)

921 aliastext = match.group(2)

922 rawaliastext = unescape(aliastext, True)

923 underscore_escaped = rawaliastext.endswith(r'\_')

924 if (aliastext.endswith('_')

925 and not (underscore_escaped

926 or self.patterns.uri.match(aliastext))):

927 aliastype = 'name'

928 alias = normalize_name(unescape(aliastext[:-1]))

929 target = nodes.target(match.group(1), refname=alias)

930 else:

931 aliastype = 'uri'

932 # remove unescaped whitespace

933 alias_parts = split_escaped_whitespace(match.group(2))

934 alias = ' '.join(''.join(part.split())

935 for part in alias_parts)

936 alias = self.adjust_uri(unescape(alias))

937 if alias.endswith(r'\_'):

938 alias = alias[:-2] + '_'

939 target = nodes.target(match.group(1), refuri=alias)

940 target.referenced = True

941 if not aliastext:

942 raise ApplicationError('problem with embedded link: %r'

943 % aliastext)

944 if not text:

945 text = alias

946 unescaped = unescape(text)

947 rawtext = rawaliastext

948 else:

949 text = escaped

950 unescaped = unescape(text)

951 target = None

952 rawtext = unescape(escaped, True)

953

954 refname = normalize_name(unescaped)

955 reference = nodes.reference(rawsource, text)

956 reference[0].rawsource = rawtext

957

958 node_list = [reference]

959

960 if rawsource[-2:] == '__':

961 if target and (aliastype == 'name'):

962 reference['refname'] = alias

963 self.document.note_refname(reference)

964 # self.document.note_indirect_target(target) # required?

965 elif target and (aliastype == 'uri'):

966 reference['refuri'] = alias

967 else:

968 reference['anonymous'] = True

969 else:

970 if target:

971 target['names'].append(refname)

972 if aliastype == 'name':

973 reference['refname'] = alias

974 self.document.note_indirect_target(target)

975 self.document.note_refname(reference)

976 else:

977 reference['refuri'] = alias

978 # target.note_referenced_by(name=refname)

979 self.document.note_implicit_target(target, self.parent)

980 node_list.append(target)

981 else:

982 reference['refname'] = refname

983 self.document.note_refname(reference)

984 return before, node_list, after, []

985

986 def adjust_uri(self, uri):

987 match = self.patterns.email.match(uri)

988 if match:

989 return 'mailto:' + uri

990 else:

991 return uri

992

993 def interpreted(self, rawsource, text, role, lineno):

994 role_fn, messages = roles.role(role, self.language, lineno,

995 self.reporter)

996 if role_fn:

997 nodes, messages2 = role_fn(role, rawsource, text, lineno, self)

998 return nodes, messages + messages2

999 else:

1000 msg = self.reporter.error(

1001 'Unknown interpreted text role "%s".' % role,

1002 line=lineno)

1003 return ([self.problematic(rawsource, rawsource, msg)],

1004 messages + [msg])

1005

1006 def literal(self, match, lineno):

1007 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

1008 match, lineno, self.patterns.literal, nodes.literal,

1009 restore_backslashes=True)

1010 return before, inlines, remaining, sysmessages

1011

1012 def inline_internal_target(self, match, lineno):

1013 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

1014 match, lineno, self.patterns.target, nodes.inline)

1015 if inlines and isinstance(inlines[0], nodes.inline):

1016 assert len(inlines) == 1

1017 inline_target = inlines[0]

1018 name = normalize_name(inline_target.astext())

1019 inline_target['names'].append(name)

1020 self.document.note_explicit_target(inline_target, self.parent)

1021 return before, inlines, remaining, sysmessages

1022

1023 def substitution_reference(self, match, lineno):

1024 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

1025 match, lineno, self.patterns.substitution_ref,

1026 nodes.substitution_reference)

1027 if len(inlines) == 1:

1028 subref_node = inlines[0]

1029 if isinstance(subref_node, nodes.substitution_reference):

1030 subref_text = subref_node.astext()

1031 self.document.note_substitution_ref(subref_node, subref_text)

1032 if endstring[-1:] == '_':

1033 reference_node = nodes.reference(

1034 '|%s%s' % (subref_text, endstring), '')

1035 if endstring[-2:] == '__':

1036 reference_node['anonymous'] = True

1037 else:

1038 reference_node['refname'] = normalize_name(subref_text)

1039 self.document.note_refname(reference_node)

1040 reference_node += subref_node

1041 inlines = [reference_node]

1042 return before, inlines, remaining, sysmessages

1043

1044 def footnote_reference(self, match, lineno):

1045 """

1046 Handles `nodes.footnote_reference` and `nodes.citation_reference`

1047 elements.

1048 """

1049 label = match.group('footnotelabel')

1050 refname = normalize_name(label)

1051 string = match.string

1052 before = string[:match.start('whole')]

1053 remaining = string[match.end('whole'):]

1054 if match.group('citationlabel'):

1055 refnode = nodes.citation_reference('[%s]_' % label,

1056 refname=refname)

1057 refnode += nodes.Text(label)

1058 self.document.note_citation_ref(refnode)

1059 else:

1060 refnode = nodes.footnote_reference('[%s]_' % label)

1061 if refname[0] == '#':

1062 refname = refname[1:]

1063 refnode['auto'] = 1

1064 self.document.note_autofootnote_ref(refnode)

1065 elif refname == '*':

1066 refname = ''

1067 refnode['auto'] = '*'

1068 self.document.note_symbol_footnote_ref(

1069 refnode)

1070 else:

1071 refnode += nodes.Text(label)

1072 if refname:

1073 refnode['refname'] = refname

1074 self.document.note_footnote_ref(refnode)

1075 if utils.get_trim_footnote_ref_space(self.document.settings):

1076 before = before.rstrip()

1077 return before, [refnode], remaining, []

1078

1079 def reference(self, match, lineno, anonymous=False):

1080 referencename = match.group('refname')

1081 refname = normalize_name(referencename)

1082 referencenode = nodes.reference(

1083 referencename + match.group('refend'), referencename)

1084 referencenode[0].rawsource = referencename

1085 if anonymous:

1086 referencenode['anonymous'] = True

1087 else:

1088 referencenode['refname'] = refname

1089 self.document.note_refname(referencenode)

1090 string = match.string

1091 matchstart = match.start('whole')

1092 matchend = match.end('whole')

1093 return string[:matchstart], [referencenode], string[matchend:], []

1094

1095 def anonymous_reference(self, match, lineno):

1096 return self.reference(match, lineno, anonymous=True)

1097

1098 def standalone_uri(self, match, lineno):

1099 if (not match.group('scheme')

1100 or match.group('scheme').lower() in urischemes.schemes):

1101 if match.group('email'):

1102 addscheme = 'mailto:'

1103 else:

1104 addscheme = ''

1105 text = match.group('whole')

1106 refuri = addscheme + unescape(text)

1107 reference = nodes.reference(unescape(text, True), text,

1108 refuri=refuri)

1109 return [reference]

1110 else: # not a valid scheme

1111 raise MarkupMismatch

1112

1113 def pep_reference(self, match, lineno):

1114 text = match.group(0)

1115 if text.startswith('pep-'):

1116 pepnum = int(unescape(match.group('pepnum1')))

1117 elif text.startswith('PEP'):

1118 pepnum = int(unescape(match.group('pepnum2')))

1119 else:

1120 raise MarkupMismatch

1121 ref = (self.document.settings.pep_base_url

1122 + self.document.settings.pep_file_url_template % pepnum)

1123 return [nodes.reference(unescape(text, True), text, refuri=ref)]

1124

1125 rfc_url = 'rfc%d/'

1126

1127 def rfc_reference(self, match, lineno):

1128 text = match.group(0)

1129 if text.startswith('RFC'):

1130 rfcnum = int(unescape(match.group('rfcnum')))

1131 ref = self.document.settings.rfc_base_url + self.rfc_url % rfcnum

1132 else:

1133 raise MarkupMismatch

1134 return [nodes.reference(unescape(text, True), text, refuri=ref)]

1135

1136 def implicit_inline(self, text, lineno):

1137 """

1138 Check each of the patterns in `self.implicit_dispatch` for a match,

1139 and dispatch to the stored method for the pattern. Recursively check

1140 the text before and after the match. Return a list of `nodes.Text`

1141 and inline element nodes.

1142 """

1143 if not text:

1144 return []

1145 for pattern, method in self.implicit_dispatch:

1146 match = pattern.search(text)

1147 if match:

1148 try:

1149 # Must recurse on strings before *and* after the match;

1150 # there may be multiple patterns.

1151 return (self.implicit_inline(text[:match.start()], lineno)

1152 + method(match, lineno)

1153 + self.implicit_inline(text[match.end():], lineno))

1154 except MarkupMismatch:

1155 pass

1156 return [nodes.Text(text)]

1157

1158 dispatch = {'*': emphasis,

1159 '**': strong,

1160 '`': interpreted_or_phrase_ref,

1161 '``': literal,

1162 '_`': inline_internal_target,

1163 ']_': footnote_reference,

1164 '|': substitution_reference,

1165 '_': reference,

1166 '__': anonymous_reference}

1167

1168

1169def _loweralpha_to_int(s, _zero=(ord('a')-1)):

1170 return ord(s) - _zero

1171

1172

1173def _upperalpha_to_int(s, _zero=(ord('A')-1)):

1174 return ord(s) - _zero

1175

1176

1177class Body(RSTState):

1178

1179 """

1180 Generic classifier of the first line of a block.

1181 """

1182

1183 double_width_pad_char = tableparser.TableParser.double_width_pad_char

1184 """Padding character for East Asian double-width text."""

1185

1186 enum = Struct()

1187 """Enumerated list parsing information."""

1188

1189 enum.formatinfo = {

1190 'parens': Struct(prefix='(', suffix=')', start=1, end=-1),

1191 'rparen': Struct(prefix='', suffix=')', start=0, end=-1),

1192 'period': Struct(prefix='', suffix='.', start=0, end=-1)}

1193 enum.formats = enum.formatinfo.keys()

1194 enum.sequences = ['arabic', 'loweralpha', 'upperalpha',

1195 'lowerroman', 'upperroman'] # ORDERED!

1196 enum.sequencepats = {'arabic': '[0-9]+',

1197 'loweralpha': '[a-z]',

1198 'upperalpha': '[A-Z]',

1199 'lowerroman': '[ivxlcdm]+',

1200 'upperroman': '[IVXLCDM]+'}

1201 enum.converters = {'arabic': int,

1202 'loweralpha': _loweralpha_to_int,

1203 'upperalpha': _upperalpha_to_int,

1204 'lowerroman': RomanNumeral.from_string,

1205 'upperroman': RomanNumeral.from_string}

1206

1207 enum.sequenceregexps = {}

1208 for sequence in enum.sequences:

1209 enum.sequenceregexps[sequence] = re.compile(

1210 enum.sequencepats[sequence] + '$')

1211

1212 grid_table_top_pat = re.compile(r'\+-[-+]+-\+ *$')

1213 """Matches the top (& bottom) of a full table)."""

1214

1215 simple_table_top_pat = re.compile('=+( +=+)+ *$')

1216 """Matches the top of a simple table."""

1217

1218 simple_table_border_pat = re.compile('=+[ =]*$')

1219 """Matches the bottom & header bottom of a simple table."""

1220

1221 pats = {}

1222 """Fragments of patterns used by transitions."""

1223

1224 pats['nonalphanum7bit'] = '[!-/:-@[-`{-~]'

1225 pats['alpha'] = '[a-zA-Z]'

1226 pats['alphanum'] = '[a-zA-Z0-9]'

1227 pats['alphanumplus'] = '[a-zA-Z0-9_-]'

1228 pats['enum'] = ('(%(arabic)s|%(loweralpha)s|%(upperalpha)s|%(lowerroman)s'

1229 '|%(upperroman)s|#)' % enum.sequencepats)

1230 pats['optname'] = '%(alphanum)s%(alphanumplus)s*' % pats

1231 # @@@ Loosen up the pattern? Allow Unicode?

1232 pats['optarg'] = '(%(alpha)s%(alphanumplus)s*|<[^<>]+>)' % pats

1233 pats['shortopt'] = r'(-|\+)%(alphanum)s( ?%(optarg)s)?' % pats

1234 pats['longopt'] = r'(--|/)%(optname)s([ =]%(optarg)s)?' % pats

1235 pats['option'] = r'(%(shortopt)s|%(longopt)s)' % pats

1236

1237 for format in enum.formats:

1238 pats[format] = '(?P<%s>%s%s%s)' % (

1239 format, re.escape(enum.formatinfo[format].prefix),

1240 pats['enum'], re.escape(enum.formatinfo[format].suffix))

1241

1242 patterns = {

1243 'bullet': '[-+*\u2022\u2023\u2043]( +|$)',

1244 'enumerator': r'(%(parens)s|%(rparen)s|%(period)s)( +|$)' % pats,

1245 'field_marker': r':(?![: ])([^:\\]|\\.|:(?!([ `]|$)))*(?<! ):( +|$)',

1246 'option_marker': r'%(option)s(, %(option)s)*( +| ?$)' % pats,

1247 'doctest': r'>>>( +|$)',

1248 'line_block': r'\|( +|$)',

1249 'grid_table_top': grid_table_top_pat,

1250 'simple_table_top': simple_table_top_pat,

1251 'explicit_markup': r'\.\.( +|$)',

1252 'anonymous': r'__( +|$)',

1253 'line': r'(%(nonalphanum7bit)s)\1* *$' % pats,

1254 'text': r''}

1255 initial_transitions = (

1256 'bullet',

1257 'enumerator',

1258 'field_marker',

1259 'option_marker',

1260 'doctest',

1261 'line_block',

1262 'grid_table_top',

1263 'simple_table_top',

1264 'explicit_markup',

1265 'anonymous',

1266 'line',

1267 'text')

1268

1269 def indent(self, match, context, next_state):

1270 """Block quote."""

1271 (indented, indent, line_offset, blank_finish

1272 ) = self.state_machine.get_indented()

1273 elements = self.block_quote(indented, line_offset)

1274 self.parent += elements

1275 if not blank_finish:

1276 self.parent += self.unindent_warning('Block quote')

1277 return context, next_state, []

1278

1279 def block_quote(self, indented, line_offset):

1280 elements = []

1281 while indented:

1282 blockquote = nodes.block_quote(rawsource='\n'.join(indented))

1283 (blockquote.source, blockquote.line

1284 ) = self.state_machine.get_source_and_line(line_offset+1)

1285 (blockquote_lines,

1286 attribution_lines,

1287 attribution_offset,

1288 indented,

1289 new_line_offset) = self.split_attribution(indented, line_offset)

1290 self.nested_parse(blockquote_lines, line_offset, blockquote)

1291 elements.append(blockquote)

1292 if attribution_lines:

1293 attribution, messages = self.parse_attribution(

1294 attribution_lines, line_offset+attribution_offset)

1295 blockquote += attribution

1296 elements += messages

1297 line_offset = new_line_offset

1298 while indented and not indented[0]:

1299 indented = indented[1:]

1300 line_offset += 1

1301 return elements

1302

1303 # U+2014 is an em-dash:

1304 attribution_pattern = re.compile('(---?(?!-)|\u2014) *(?=[^ \\n])')

1305

1306 def split_attribution(self, indented, line_offset):

1307 """

1308 Check for a block quote attribution and split it off:

1309

1310 * First line after a blank line must begin with a dash ("--", "---",

1311 em-dash; matches `self.attribution_pattern`).

1312 * Every line after that must have consistent indentation.

1313 * Attributions must be preceded by block quote content.

1314

1315 Return a tuple of: (block quote content lines, attribution lines,

1316 attribution offset, remaining indented lines, remaining lines offset).

1317 """

1318 blank = None

1319 nonblank_seen = False

1320 for i in range(len(indented)):

1321 line = indented[i].rstrip()

1322 if line:

1323 if nonblank_seen and blank == i - 1: # last line blank

1324 match = self.attribution_pattern.match(line)

1325 if match:

1326 attribution_end, indent = self.check_attribution(

1327 indented, i)

1328 if attribution_end:

1329 a_lines = indented[i:attribution_end]

1330 a_lines.trim_left(match.end(), end=1)

1331 a_lines.trim_left(indent, start=1)

1332 return (indented[:i], a_lines,

1333 i, indented[attribution_end:],

1334 line_offset + attribution_end)

1335 nonblank_seen = True

1336 else:

1337 blank = i

1338 else:

1339 return indented, None, None, None, None

1340

1341 def check_attribution(self, indented, attribution_start):

1342 """

1343 Check attribution shape.

1344 Return the index past the end of the attribution, and the indent.

1345 """

1346 indent = None

1347 i = attribution_start + 1

1348 for i in range(attribution_start + 1, len(indented)):

1349 line = indented[i].rstrip()

1350 if not line:

1351 break

1352 if indent is None:

1353 indent = len(line) - len(line.lstrip())

1354 elif len(line) - len(line.lstrip()) != indent:

1355 return None, None # bad shape; not an attribution

1356 else:

1357 # return index of line after last attribution line:

1358 i += 1

1359 return i, (indent or 0)

1360

1361 def parse_attribution(self, indented, line_offset):

1362 text = '\n'.join(indented).rstrip()

1363 lineno = 1 + line_offset # line_offset is zero-based

1364 textnodes, messages = self.inline_text(text, lineno)

1365 node = nodes.attribution(text, '', *textnodes)

1366 node.source, node.line = self.state_machine.get_source_and_line(lineno)

1367 return node, messages

1368

1369 def bullet(self, match, context, next_state):

1370 """Bullet list item."""

1371 ul = nodes.bullet_list()

1372 ul.source, ul.line = self.state_machine.get_source_and_line()

1373 self.parent += ul

1374 ul['bullet'] = match.string[0]

1375 i, blank_finish = self.list_item(match.end())

1376 ul += i

1377 offset = self.state_machine.line_offset + 1 # next line

1378 new_line_offset, blank_finish = self.nested_list_parse(

1379 self.state_machine.input_lines[offset:],

1380 input_offset=self.state_machine.abs_line_offset() + 1,

1381 node=ul, initial_state='BulletList',

1382 blank_finish=blank_finish)

1383 self.goto_line(new_line_offset)

1384 if not blank_finish:

1385 self.parent += self.unindent_warning('Bullet list')

1386 return [], next_state, []

1387

1388 def list_item(self, indent):

1389 src, srcline = self.state_machine.get_source_and_line()

1390 if self.state_machine.line[indent:]:

1391 indented, line_offset, blank_finish = (

1392 self.state_machine.get_known_indented(indent))

1393 else:

1394 indented, indent, line_offset, blank_finish = (

1395 self.state_machine.get_first_known_indented(indent))

1396 listitem = nodes.list_item('\n'.join(indented))

1397 listitem.source, listitem.line = src, srcline

1398 if indented:

1399 self.nested_parse(indented, input_offset=line_offset,

1400 node=listitem)

1401 return listitem, blank_finish

1402

1403 def enumerator(self, match, context, next_state):

1404 """Enumerated List Item"""

1405 format, sequence, text, ordinal = self.parse_enumerator(match)

1406 if not self.is_enumerated_list_item(ordinal, sequence, format):

1407 raise statemachine.TransitionCorrection('text')

1408 enumlist = nodes.enumerated_list()

1409 (enumlist.source,

1410 enumlist.line) = self.state_machine.get_source_and_line()

1411 self.parent += enumlist

1412 if sequence == '#':

1413 enumlist['enumtype'] = 'arabic'

1414 else:

1415 enumlist['enumtype'] = sequence

1416 enumlist['prefix'] = self.enum.formatinfo[format].prefix

1417 enumlist['suffix'] = self.enum.formatinfo[format].suffix

1418 if ordinal != 1:

1419 enumlist['start'] = ordinal

1420 msg = self.reporter.info(

1421 'Enumerated list start value not ordinal-1: "%s" (ordinal %s)'

1422 % (text, ordinal), base_node=enumlist)

1423 self.parent += msg

1424 listitem, blank_finish = self.list_item(match.end())

1425 enumlist += listitem

1426 offset = self.state_machine.line_offset + 1 # next line

1427 newline_offset, blank_finish = self.nested_list_parse(

1428 self.state_machine.input_lines[offset:],

1429 input_offset=self.state_machine.abs_line_offset() + 1,

1430 node=enumlist, initial_state='EnumeratedList',

1431 blank_finish=blank_finish,

1432 extra_settings={'lastordinal': ordinal,

1433 'format': format,

1434 'auto': sequence == '#'})

1435 self.goto_line(newline_offset)

1436 if not blank_finish:

1437 self.parent += self.unindent_warning('Enumerated list')

1438 return [], next_state, []

1439

1440 def parse_enumerator(self, match, expected_sequence=None):

1441 """

1442 Analyze an enumerator and return the results.

1443

1444 :Return:

1445 - the enumerator format ('period', 'parens', or 'rparen'),

1446 - the sequence used ('arabic', 'loweralpha', 'upperroman', etc.),

1447 - the text of the enumerator, stripped of formatting, and

1448 - the ordinal value of the enumerator ('a' -> 1, 'ii' -> 2, etc.;

1449 ``None`` is returned for invalid enumerator text).

1450

1451 The enumerator format has already been determined by the regular

1452 expression match. If `expected_sequence` is given, that sequence is

1453 tried first. If not, we check for Roman numeral 1. This way,

1454 single-character Roman numerals (which are also alphabetical) can be

1455 matched. If no sequence has been matched, all sequences are checked in

1456 order.

1457 """

1458 groupdict = match.groupdict()

1459 sequence = ''

1460 for format in self.enum.formats:

1461 if groupdict[format]: # was this the format matched?

1462 break # yes; keep `format`

1463 else: # shouldn't happen

1464 raise ParserError('enumerator format not matched')

1465 text = groupdict[format][self.enum.formatinfo[format].start # noqa: E203,E501

1466 : self.enum.formatinfo[format].end]

1467 if text == '#':

1468 sequence = '#'

1469 elif expected_sequence:

1470 try:

1471 if self.enum.sequenceregexps[expected_sequence].match(text):

1472 sequence = expected_sequence

1473 except KeyError: # shouldn't happen

1474 raise ParserError('unknown enumerator sequence: %s'

1475 % sequence)

1476 elif text == 'i':

1477 sequence = 'lowerroman'

1478 elif text == 'I':

1479 sequence = 'upperroman'

1480 if not sequence:

1481 for sequence in self.enum.sequences:

1482 if self.enum.sequenceregexps[sequence].match(text):

1483 break

1484 else: # shouldn't happen

1485 raise ParserError('enumerator sequence not matched')

1486 if sequence == '#':

1487 ordinal = 1

1488 else:

1489 try:

1490 ordinal = int(self.enum.converters[sequence](text))

1491 except InvalidRomanNumeralError:

1492 ordinal = None

1493 return format, sequence, text, ordinal

1494

1495 def is_enumerated_list_item(self, ordinal, sequence, format):

1496 """

1497 Check validity based on the ordinal value and the second line.

1498

1499 Return true if the ordinal is valid and the second line is blank,

1500 indented, or starts with the next enumerator or an auto-enumerator.

1501 """

1502 if ordinal is None:

1503 return None

1504 try:

1505 next_line = self.state_machine.next_line()

1506 except EOFError: # end of input lines

1507 self.state_machine.previous_line()

1508 return 1

1509 else:

1510 self.state_machine.previous_line()

1511 if not next_line[:1].strip(): # blank or indented

1512 return 1

1513 result = self.make_enumerator(ordinal + 1, sequence, format)

1514 if result:

1515 next_enumerator, auto_enumerator = result

1516 try:

1517 if next_line.startswith((next_enumerator, auto_enumerator)):

1518 return 1

1519 except TypeError:

1520 pass

1521 return None

1522

1523 def make_enumerator(self, ordinal, sequence, format):

1524 """

1525 Construct and return the next enumerated list item marker, and an

1526 auto-enumerator ("#" instead of the regular enumerator).

1527

1528 Return ``None`` for invalid (out of range) ordinals.

1529 """

1530 if sequence == '#':

1531 enumerator = '#'

1532 elif sequence == 'arabic':

1533 enumerator = str(ordinal)

1534 else:

1535 if sequence.endswith('alpha'):

1536 if ordinal > 26:

1537 return None

1538 enumerator = chr(ordinal + ord('a') - 1)

1539 elif sequence.endswith('roman'):

1540 try:

1541 enumerator = RomanNumeral(ordinal).to_uppercase()

1542 except TypeError:

1543 return None

1544 else: # shouldn't happen

1545 raise ParserError('unknown enumerator sequence: "%s"'

1546 % sequence)

1547 if sequence.startswith('lower'):

1548 enumerator = enumerator.lower()

1549 elif sequence.startswith('upper'):

1550 enumerator = enumerator.upper()

1551 else: # shouldn't happen

1552 raise ParserError('unknown enumerator sequence: "%s"'

1553 % sequence)

1554 formatinfo = self.enum.formatinfo[format]

1555 next_enumerator = (formatinfo.prefix + enumerator + formatinfo.suffix

1556 + ' ')

1557 auto_enumerator = formatinfo.prefix + '#' + formatinfo.suffix + ' '

1558 return next_enumerator, auto_enumerator

1559

1560 def field_marker(self, match, context, next_state):

1561 """Field list item."""

1562 field_list = nodes.field_list()

1563 self.parent += field_list

1564 field, blank_finish = self.field(match)

1565 field_list += field

1566 offset = self.state_machine.line_offset + 1 # next line

1567 newline_offset, blank_finish = self.nested_list_parse(

1568 self.state_machine.input_lines[offset:],

1569 input_offset=self.state_machine.abs_line_offset() + 1,

1570 node=field_list, initial_state='FieldList',

1571 blank_finish=blank_finish)

1572 self.goto_line(newline_offset)

1573 if not blank_finish:

1574 self.parent += self.unindent_warning('Field list')

1575 return [], next_state, []

1576

1577 def field(self, match):

1578 name = self.parse_field_marker(match)

1579 src, srcline = self.state_machine.get_source_and_line()

1580 lineno = self.state_machine.abs_line_number()

1581 (indented, indent, line_offset, blank_finish

1582 ) = self.state_machine.get_first_known_indented(match.end())

1583 field_node = nodes.field()

1584 field_node.source = src

1585 field_node.line = srcline

1586 name_nodes, name_messages = self.inline_text(name, lineno)

1587 field_node += nodes.field_name(name, '', *name_nodes)

1588 field_body = nodes.field_body('\n'.join(indented), *name_messages)

1589 field_node += field_body

1590 if indented:

1591 self.parse_field_body(indented, line_offset, field_body)

1592 return field_node, blank_finish

1593

1594 def parse_field_marker(self, match):

1595 """Extract & return field name from a field marker match."""

1596 field = match.group()[1:] # strip off leading ':'

1597 field = field[:field.rfind(':')] # strip off trailing ':' etc.

1598 return field

1599

1600 def parse_field_body(self, indented, offset, node) -> None:

1601 self.nested_parse(indented, input_offset=offset, node=node)

1602

1603 def option_marker(self, match, context, next_state):

1604 """Option list item."""

1605 optionlist = nodes.option_list()

1606 (optionlist.source, optionlist.line

1607 ) = self.state_machine.get_source_and_line()

1608 try:

1609 listitem, blank_finish = self.option_list_item(match)

1610 except MarkupError as error:

1611 # This shouldn't happen; pattern won't match.

1612 msg = self.reporter.error('Invalid option list marker: %s'

1613 % error)

1614 self.parent += msg

1615 (indented, indent, line_offset, blank_finish

1616 ) = self.state_machine.get_first_known_indented(match.end())

1617 elements = self.block_quote(indented, line_offset)

1618 self.parent += elements

1619 if not blank_finish:

1620 self.parent += self.unindent_warning('Option list')

1621 return [], next_state, []

1622 self.parent += optionlist

1623 optionlist += listitem

1624 offset = self.state_machine.line_offset + 1 # next line

1625 newline_offset, blank_finish = self.nested_list_parse(

1626 self.state_machine.input_lines[offset:],

1627 input_offset=self.state_machine.abs_line_offset() + 1,

1628 node=optionlist, initial_state='OptionList',

1629 blank_finish=blank_finish)

1630 self.goto_line(newline_offset)

1631 if not blank_finish:

1632 self.parent += self.unindent_warning('Option list')

1633 return [], next_state, []

1634

1635 def option_list_item(self, match):

1636 offset = self.state_machine.abs_line_offset()

1637 options = self.parse_option_marker(match)

1638 (indented, indent, line_offset, blank_finish

1639 ) = self.state_machine.get_first_known_indented(match.end())

1640 if not indented: # not an option list item

1641 self.goto_line(offset)

1642 raise statemachine.TransitionCorrection('text')

1643 option_group = nodes.option_group('', *options)

1644 description = nodes.description('\n'.join(indented))

1645 option_list_item = nodes.option_list_item('', option_group,

1646 description)

1647 if indented:

1648 self.nested_parse(indented, input_offset=line_offset,

1649 node=description)

1650 return option_list_item, blank_finish

1651

1652 def parse_option_marker(self, match):

1653 """

1654 Return a list of `node.option` and `node.option_argument` objects,

1655 parsed from an option marker match.

1656

1657 :Exception: `MarkupError` for invalid option markers.

1658 """

1659 optlist = []

1660 # split at ", ", except inside < > (complex arguments)

1661 optionstrings = re.split(r', (?![^<]*>)', match.group().rstrip())

1662 for optionstring in optionstrings:

1663 tokens = optionstring.split()

1664 delimiter = ' '

1665 firstopt = tokens[0].split('=', 1)

1666 if len(firstopt) > 1:

1667 # "--opt=value" form

1668 tokens[:1] = firstopt

1669 delimiter = '='

1670 elif (len(tokens[0]) > 2

1671 and ((tokens[0].startswith('-')

1672 and not tokens[0].startswith('--'))

1673 or tokens[0].startswith('+'))):

1674 # "-ovalue" form

1675 tokens[:1] = [tokens[0][:2], tokens[0][2:]]

1676 delimiter = ''

1677 if len(tokens) > 1 and (tokens[1].startswith('<')

1678 and tokens[-1].endswith('>')):

1679 # "-o <value1 value2>" form; join all values into one token

1680 tokens[1:] = [' '.join(tokens[1:])]

1681 if 0 < len(tokens) <= 2:

1682 option = nodes.option(optionstring)

1683 option += nodes.option_string(tokens[0], tokens[0])

1684 if len(tokens) > 1:

1685 option += nodes.option_argument(tokens[1], tokens[1],

1686 delimiter=delimiter)

1687 optlist.append(option)

1688 else:

1689 raise MarkupError(

1690 'wrong number of option tokens (=%s), should be 1 or 2: '

1691 '"%s"' % (len(tokens), optionstring))

1692 return optlist

1693

1694 def doctest(self, match, context, next_state):

1695 line = self.document.current_line

1696 data = self.state_machine.get_text_block()

1697 try:

1698 n = tag_code(data, 'pycon', self.document.settings,

1699 options={'class': ['doctest']})

1700 except LexerError as error:

1701 n = self.reporter.warning(error)

1702 n.line = line

1703 self.parent += n

1704 return [], next_state, []

1705

1706 def line_block(self, match, context, next_state):

1707 """First line of a line block."""

1708 block = nodes.line_block()

1709 self.parent += block

1710 lineno = self.state_machine.abs_line_number()

1711 (block.source,

1712 block.line) = self.state_machine.get_source_and_line(lineno)

1713 line, messages, blank_finish = self.line_block_line(match, lineno)

1714 block += line

1715 self.parent += messages

1716 if not blank_finish:

1717 offset = self.state_machine.line_offset + 1 # next line

1718 new_line_offset, blank_finish = self.nested_list_parse(

1719 self.state_machine.input_lines[offset:],

1720 input_offset=self.state_machine.abs_line_offset() + 1,

1721 node=block, initial_state='LineBlock',

1722 blank_finish=False)

1723 self.goto_line(new_line_offset)

1724 if not blank_finish:

1725 self.parent += self.reporter.warning(

1726 'Line block ends without a blank line.',

1727 line=lineno+1)

1728 if len(block):

1729 if block[0].indent is None:

1730 block[0].indent = 0

1731 self.nest_line_block_lines(block)

1732 return [], next_state, []

1733

1734 def line_block_line(self, match, lineno):

1735 """Return one line element of a line_block."""

1736 (indented, indent, line_offset, blank_finish

1737 ) = self.state_machine.get_first_known_indented(match.end(),

1738 until_blank=True)

1739 text = '\n'.join(indented)

1740 text_nodes, messages = self.inline_text(text, lineno)

1741 line = nodes.line(text, '', *text_nodes)

1742 (line.source,

1743 line.line) = self.state_machine.get_source_and_line(lineno)

1744 if match.string.rstrip() != '|': # not empty

1745 line.indent = len(match.group(1)) - 1

1746 return line, messages, blank_finish

1747

1748 def nest_line_block_lines(self, block) -> None:

1749 for index in range(1, len(block)):

1750 if block[index].indent is None:

1751 block[index].indent = block[index - 1].indent

1752 self.nest_line_block_segment(block)

1753

1754 def nest_line_block_segment(self, block) -> None:

1755 indents = [item.indent for item in block]

1756 least = min(indents)

1757 new_items = []

1758 new_block = nodes.line_block()

1759 for item in block:

1760 if item.indent > least:

1761 new_block.append(item)

1762 else:

1763 if len(new_block):

1764 self.nest_line_block_segment(new_block)

1765 new_items.append(new_block)

1766 new_block = nodes.line_block()

1767 new_items.append(item)

1768 if len(new_block):

1769 self.nest_line_block_segment(new_block)

1770 new_items.append(new_block)

1771 block[:] = new_items

1772

1773 def grid_table_top(self, match, context, next_state):

1774 """Top border of a full table."""

1775 return self.table_top(match, context, next_state,

1776 self.isolate_grid_table,

1777 tableparser.GridTableParser)

1778

1779 def simple_table_top(self, match, context, next_state):

1780 """Top border of a simple table."""

1781 return self.table_top(match, context, next_state,

1782 self.isolate_simple_table,

1783 tableparser.SimpleTableParser)

1784

1785 def table_top(self, match, context, next_state,

1786 isolate_function, parser_class):

1787 """Top border of a generic table."""

1788 nodelist, blank_finish = self.table(isolate_function, parser_class)

1789 self.parent += nodelist

1790 if not blank_finish:

1791 msg = self.reporter.warning(

1792 'Blank line required after table.',

1793 line=self.state_machine.abs_line_number()+1)

1794 self.parent += msg

1795 return [], next_state, []

1796

1797 def table(self, isolate_function, parser_class):

1798 """Parse a table."""

1799 block, messages, blank_finish = isolate_function()

1800 if block:

1801 try:

1802 parser = parser_class()

1803 tabledata = parser.parse(block)

1804 tableline = (self.state_machine.abs_line_number() - len(block)

1805 + 1)

1806 table = self.build_table(tabledata, tableline)

1807 nodelist = [table] + messages

1808 except tableparser.TableMarkupError as err:

1809 nodelist = self.malformed_table(block, ' '.join(err.args),

1810 offset=err.offset) + messages

1811 else:

1812 nodelist = messages

1813 return nodelist, blank_finish

1814

1815 def isolate_grid_table(self):

1816 messages = []

1817 blank_finish = True

1818 try:

1819 block = self.state_machine.get_text_block(flush_left=True)

1820 except statemachine.UnexpectedIndentationError as err:

1821 block, src, srcline = err.args

1822 messages.append(self.reporter.error('Unexpected indentation.',

1823 source=src, line=srcline))

1824 blank_finish = False

1825 block.disconnect()

1826 # for East Asian chars:

1827 block.pad_double_width(self.double_width_pad_char)

1828 width = len(block[0].strip())

1829 for i in range(len(block)):

1830 block[i] = block[i].strip()

1831 if block[i][0] not in '+|': # check left edge

1832 blank_finish = False

1833 self.state_machine.previous_line(len(block) - i)

1834 del block[i:]

1835 break

1836 if not self.grid_table_top_pat.match(block[-1]): # find bottom

1837 # from second-last to third line of table:

1838 for i in range(len(block) - 2, 1, -1):

1839 if self.grid_table_top_pat.match(block[i]):

1840 self.state_machine.previous_line(len(block) - i + 1)

1841 del block[i+1:]

1842 blank_finish = False

1843 break

1844 else:

1845 detail = 'Bottom border missing or corrupt.'

1846 messages.extend(self.malformed_table(block, detail, i))

1847 return [], messages, blank_finish

1848 for i in range(len(block)): # check right edge

1849 if len(strip_combining_chars(block[i])

1850 ) != width or block[i][-1] not in '+|':

1851 detail = 'Right border not aligned or missing.'

1852 messages.extend(self.malformed_table(block, detail, i))

1853 return [], messages, blank_finish

1854 return block, messages, blank_finish

1855

1856 def isolate_simple_table(self):

1857 start = self.state_machine.line_offset

1858 lines = self.state_machine.input_lines

1859 limit = len(lines) - 1

1860 toplen = len(lines[start].strip())

1861 pattern_match = self.simple_table_border_pat.match

1862 found = 0

1863 found_at = None

1864 i = start + 1

1865 while i <= limit:

1866 line = lines[i]

1867 match = pattern_match(line)

1868 if match:

1869 if len(line.strip()) != toplen:

1870 self.state_machine.next_line(i - start)

1871 messages = self.malformed_table(

1872 lines[start:i+1], 'Bottom border or header rule does '

1873 'not match top border.', i-start)

1874 return [], messages, i == limit or not lines[i+1].strip()

1875 found += 1

1876 found_at = i

1877 if found == 2 or i == limit or not lines[i+1].strip():

1878 end = i

1879 break

1880 i += 1

1881 else: # reached end of input_lines

1882 details = 'No bottom table border found'

1883 if found:

1884 details += ' or no blank line after table bottom'

1885 self.state_machine.next_line(found_at - start)

1886 block = lines[start:found_at+1]

1887 else:

1888 self.state_machine.next_line(i - start - 1)

1889 block = lines[start:]

1890 messages = self.malformed_table(block, details + '.')

1891 return [], messages, not found

1892 self.state_machine.next_line(end - start)

1893 block = lines[start:end+1]

1894 # for East Asian chars:

1895 block.pad_double_width(self.double_width_pad_char)

1896 return block, [], end == limit or not lines[end+1].strip()

1897

1898 def malformed_table(self, block, detail='', offset=0):

1899 block.replace(self.double_width_pad_char, '')

1900 data = '\n'.join(block)

1901 message = 'Malformed table.'

1902 startline = self.state_machine.abs_line_number() - len(block) + 1

1903 if detail:

1904 message += '\n' + detail

1905 error = self.reporter.error(message, nodes.literal_block(data, data),

1906 line=startline+offset)

1907 return [error]

1908

1909 def build_table(self, tabledata, tableline, stub_columns=0, widths=None):

1910 colwidths, headrows, bodyrows = tabledata

1911 table = nodes.table()

1912 (table.source,

1913 table.line) = self.state_machine.get_source_and_line(tableline)

1914 if widths == 'auto':

1915 table['classes'] += ['colwidths-auto']

1916 elif widths: # "grid" or list of integers

1917 table['classes'] += ['colwidths-given']

1918 tgroup = nodes.tgroup(cols=len(colwidths))

1919 table += tgroup

1920 for colwidth in colwidths:

1921 colspec = nodes.colspec(colwidth=str(colwidth))

1922 if stub_columns:

1923 colspec.attributes['stub'] = True

1924 stub_columns -= 1

1925 tgroup += colspec

1926 if headrows:

1927 thead = nodes.thead()

1928 tgroup += thead

1929 for row in headrows:

1930 thead += self.build_table_row(row, tableline)

1931 tbody = nodes.tbody()

1932 tgroup += tbody

1933 for row in bodyrows:

1934 tbody += self.build_table_row(row, tableline)

1935 return table

1936

1937 def build_table_row(self, rowdata, tableline):

1938 row = nodes.row()

1939 for cell in rowdata:

1940 if cell is None:

1941 continue

1942 morerows, morecols, offset, cellblock = cell

1943 attributes = {}

1944 if morerows:

1945 attributes['morerows'] = morerows

1946 if morecols:

1947 attributes['morecols'] = morecols

1948 entry = nodes.entry(**attributes)

1949 row += entry

1950 if ''.join(cellblock):

1951 self.nested_parse(cellblock, input_offset=tableline+offset-1,

1952 node=entry)

1953 return row

1954

1955 explicit = Struct()

1956 """Patterns and constants used for explicit markup recognition."""

1957

1958 explicit.patterns = Struct(

1959 target=re.compile(r"""

1960 (

1961 _ # anonymous target

1962 | # *OR*

1963 (?!_) # no underscore at the beginning

1964 (?P<quote>`?) # optional open quote

1965 (?![ `]) # first char. not space or

1966 # backquote

1967 (?P<name> # reference name

1968 .+?

1969 )

1970 %(non_whitespace_escape_before)s

1971 (?P=quote) # close quote if open quote used

1972 )

1973 (?<!(?<!\x00):) # no unescaped colon at end

1974 %(non_whitespace_escape_before)s

1975 [ ]? # optional space

1976 : # end of reference name

1977 ([ ]+|$) # followed by whitespace

1978 """ % vars(Inliner), re.VERBOSE),

1979 reference=re.compile(r"""

1980 (

1981 (?P<simple>%(simplename)s)_

1982 | # *OR*

1983 ` # open backquote

1984 (?![ ]) # not space

1985 (?P<phrase>.+?) # hyperlink phrase

1986 %(non_whitespace_escape_before)s

1987 `_ # close backquote,

1988 # reference mark

1989 )

1990 $ # end of string

1991 """ % vars(Inliner), re.VERBOSE),

1992 substitution=re.compile(r"""

1993 (

1994 (?![ ]) # first char. not space

1995 (?P<name>.+?) # substitution text

1996 %(non_whitespace_escape_before)s

1997 \| # close delimiter

1998 )

1999 ([ ]+|$) # followed by whitespace

2000 """ % vars(Inliner),

2001 re.VERBOSE),)

2002

2003 def footnote(self, match):

2004 src, srcline = self.state_machine.get_source_and_line()

2005 (indented, indent, offset, blank_finish

2006 ) = self.state_machine.get_first_known_indented(match.end())

2007 label = match.group(1)

2008 name = normalize_name(label)

2009 footnote = nodes.footnote('\n'.join(indented))

2010 footnote.source = src

2011 footnote.line = srcline

2012 if name[0] == '#': # auto-numbered

2013 name = name[1:] # autonumber label

2014 footnote['auto'] = 1

2015 if name:

2016 footnote['names'].append(name)

2017 self.document.note_autofootnote(footnote)

2018 elif name == '*': # auto-symbol

2019 name = ''

2020 footnote['auto'] = '*'

2021 self.document.note_symbol_footnote(footnote)

2022 else: # manually numbered

2023 footnote += nodes.label('', label)

2024 footnote['names'].append(name)

2025 self.document.note_footnote(footnote)

2026 if name:

2027 self.document.note_explicit_target(footnote, footnote)

2028 else:

2029 self.document.set_id(footnote, footnote)

2030 if indented:

2031 self.nested_parse(indented, input_offset=offset, node=footnote)

2032 else:

2033 footnote += self.reporter.warning('Footnote content expected.')

2034 return [footnote], blank_finish

2035

2036 def citation(self, match):

2037 src, srcline = self.state_machine.get_source_and_line()

2038 (indented, indent, offset, blank_finish

2039 ) = self.state_machine.get_first_known_indented(match.end())

2040 label = match.group(1)

2041 name = normalize_name(label)

2042 citation = nodes.citation('\n'.join(indented))

2043 citation.source = src

2044 citation.line = srcline

2045 citation += nodes.label('', label)

2046 citation['names'].append(name)

2047 self.document.note_citation(citation)

2048 self.document.note_explicit_target(citation, citation)

2049 if indented:

2050 self.nested_parse(indented, input_offset=offset, node=citation)

2051 else:

2052 citation += self.reporter.warning('Citation content expected.')

2053 return [citation], blank_finish

2054

2055 def hyperlink_target(self, match):

2056 pattern = self.explicit.patterns.target

2057 lineno = self.state_machine.abs_line_number()

2058 (block, indent, offset, blank_finish

2059 ) = self.state_machine.get_first_known_indented(

2060 match.end(), until_blank=True, strip_indent=False)

2061 blocktext = match.string[:match.end()] + '\n'.join(block)

2062 block = [escape2null(line) for line in block]

2063 escaped = block[0]

2064 blockindex = 0

2065 while True:

2066 targetmatch = pattern.match(escaped)

2067 if targetmatch:

2068 break

2069 blockindex += 1

2070 try:

2071 escaped += block[blockindex]

2072 except IndexError:

2073 raise MarkupError('malformed hyperlink target.')

2074 del block[:blockindex]

2075 block[0] = (block[0] + ' ')[targetmatch.end()-len(escaped)-1:].strip()

2076 target = self.make_target(block, blocktext, lineno,

2077 targetmatch.group('name'))

2078 return [target], blank_finish

2079

2080 def make_target(self, block, block_text, lineno, target_name):

2081 target_type, data = self.parse_target(block, block_text, lineno)

2082 if target_type == 'refname':

2083 target = nodes.target(block_text, '', refname=normalize_name(data))

2084 self.add_target(target_name, '', target, lineno)

2085 self.document.note_indirect_target(target)

2086 return target

2087 elif target_type == 'refuri':

2088 target = nodes.target(block_text, '')

2089 self.add_target(target_name, data, target, lineno)

2090 return target

2091 else:

2092 return data

2093

2094 def parse_target(self, block, block_text, lineno):

2095 """

2096 Determine the type of reference of a target.

2097

2098 :Return: A 2-tuple, one of:

2099

2100 - 'refname' and the indirect reference name

2101 - 'refuri' and the URI

2102 - 'malformed' and a system_message node

2103 """

2104 if block and block[-1].strip()[-1:] == '_': # possible indirect target

2105 reference = ' '.join(line.strip() for line in block)

2106 refname = self.is_reference(reference)

2107 if refname:

2108 return 'refname', refname

2109 ref_parts = split_escaped_whitespace(' '.join(block))

2110 reference = ' '.join(''.join(unescape(part).split())

2111 for part in ref_parts)

2112 return 'refuri', reference

2113

2114 def is_reference(self, reference):

2115 match = self.explicit.patterns.reference.match(

2116 whitespace_normalize_name(reference))

2117 if not match:

2118 return None

2119 return unescape(match.group('simple') or match.group('phrase'))

2120

2121 def add_target(self, targetname, refuri, target, lineno):

2122 target.line = lineno

2123 if targetname:

2124 name = normalize_name(unescape(targetname))

2125 target['names'].append(name)

2126 if refuri:

2127 uri = self.inliner.adjust_uri(refuri)

2128 if uri:

2129 target['refuri'] = uri

2130 else:

2131 raise ApplicationError('problem with URI: %r' % refuri)

2132 self.document.note_explicit_target(target, self.parent)

2133 else: # anonymous target

2134 if refuri:

2135 target['refuri'] = refuri

2136 target['anonymous'] = True

2137 self.document.note_anonymous_target(target)

2138

2139 def substitution_def(self, match):

2140 pattern = self.explicit.patterns.substitution

2141 src, srcline = self.state_machine.get_source_and_line()

2142 (block, indent, offset, blank_finish

2143 ) = self.state_machine.get_first_known_indented(match.end(),

2144 strip_indent=False)

2145 blocktext = (match.string[:match.end()] + '\n'.join(block))

2146 block.disconnect()

2147 escaped = escape2null(block[0].rstrip())

2148 blockindex = 0

2149 while True:

2150 subdefmatch = pattern.match(escaped)

2151 if subdefmatch:

2152 break

2153 blockindex += 1

2154 try:

2155 escaped = escaped + ' ' + escape2null(

2156 block[blockindex].strip())

2157 except IndexError:

2158 raise MarkupError('malformed substitution definition.')

2159 del block[:blockindex] # strip out the substitution marker

2160 start = subdefmatch.end()-len(escaped)-1

2161 block[0] = (block[0].strip() + ' ')[start:-1]

2162 if not block[0]:

2163 del block[0]

2164 offset += 1

2165 while block and not block[-1].strip():

2166 block.pop()

2167 subname = subdefmatch.group('name')

2168 substitution_node = nodes.substitution_definition(blocktext)

2169 substitution_node.source = src

2170 substitution_node.line = srcline

2171 if not block:

2172 msg = self.reporter.warning(

2173 'Substitution definition "%s" missing contents.' % subname,

2174 nodes.literal_block(blocktext, blocktext),

2175 source=src, line=srcline)

2176 return [msg], blank_finish

2177 block[0] = block[0].strip()

2178 substitution_node['names'].append(

2179 nodes.whitespace_normalize_name(subname))

2180 new_abs_offset, blank_finish = self.nested_list_parse(

2181 block, input_offset=offset, node=substitution_node,

2182 initial_state='SubstitutionDef', blank_finish=blank_finish)

2183 i = 0

2184 for node in substitution_node[:]:

2185 if not (isinstance(node, nodes.Inline)

2186 or isinstance(node, nodes.Text)):

2187 self.parent += substitution_node[i]

2188 del substitution_node[i]

2189 else:

2190 i += 1

2191 for node in substitution_node.findall(nodes.Element,

2192 include_self=False):

2193 if isinstance(node, nodes.problematic):

2194 msg = self.reporter.error(

2195 'Problematic content in substitution definition',

2196 nodes.literal_block('', blocktext),

2197 source=src, line=srcline)

2198 msg.append(nodes.block_quote(

2199 '', nodes.paragraph('', '', *substitution_node.children)))

2200 return [msg], blank_finish

2201 illegal = self.disallowed_inside_substitution_definitions(node)

2202 if illegal:

2203 msg = self.reporter.error(f'{illegal} are not supported in '

2204 'a substitution definition.',

2205 nodes.literal_block('', blocktext),

2206 source=src, line=srcline)

2207 return [msg], blank_finish

2208 if len(substitution_node) == 0:

2209 msg = self.reporter.warning(

2210 'Substitution definition "%s" empty or invalid.' % subname,

2211 nodes.literal_block(blocktext, blocktext),

2212 source=src, line=srcline)

2213 return [msg], blank_finish

2214 self.document.note_substitution_def(

2215 substitution_node, subname, self.parent)

2216 return [substitution_node], blank_finish

2217

2218 def disallowed_inside_substitution_definitions(self, node) -> str:

2219 if isinstance(node, nodes.reference) and node.get('anonymous'):

2220 return 'Anonymous references'

2221 if isinstance(node, nodes.footnote_reference) and node.get('auto'):

2222 return 'References to auto-numbered and auto-symbol footnotes'

2223 if node['names'] or node['ids']:

2224 return 'Targets (names and identifiers)'

2225 else:

2226 return ''

2227

2228 def directive(self, match, **option_presets):

2229 """Returns a 2-tuple: list of nodes, and a "blank finish" boolean."""

2230 type_name = match.group(1)

2231 directive_class, messages = directives.directive(

2232 type_name, self.memo.language, self.document)

2233 self.parent += messages

2234 if directive_class:

2235 return self.run_directive(

2236 directive_class, match, type_name, option_presets)

2237 else:

2238 return self.unknown_directive(type_name)

2239

2240 def run_directive(self, directive, match, type_name, option_presets):

2241 """

2242 Parse a directive then run its directive function.

2243

2244 Parameters:

2245

2246 - `directive`: The class implementing the directive. Must be

2247 a subclass of `rst.Directive`.

2248

2249 - `match`: A regular expression match object which matched the first

2250 line of the directive.

2251

2252 - `type_name`: The directive name, as used in the source text.

2253

2254 - `option_presets`: A dictionary of preset options, defaults for the

2255 directive options. Currently, only an "alt" option is passed by

2256 substitution definitions (value: the substitution name), which may

2257 be used by an embedded image directive.

2258

2259 Returns a 2-tuple: list of nodes, and a "blank finish" boolean.

2260 """

2261 if isinstance(directive, (FunctionType, MethodType)):

2262 from docutils.parsers.rst import convert_directive_function

2263 directive = convert_directive_function(directive)

2264 lineno = self.state_machine.abs_line_number()

2265 initial_line_offset = self.state_machine.line_offset

2266 (indented, indent, line_offset, blank_finish

2267 ) = self.state_machine.get_first_known_indented(match.end(),

2268 strip_top=0)

2269 block_text = '\n'.join(self.state_machine.input_lines[

2270 initial_line_offset : self.state_machine.line_offset + 1]) # noqa: E203,E501

2271 try:

2272 arguments, options, content, content_offset = (

2273 self.parse_directive_block(indented, line_offset,

2274 directive, option_presets))

2275 except MarkupError as detail:

2276 error = self.reporter.error(

2277 'Error in "%s" directive:\n%s.' % (type_name,

2278 ' '.join(detail.args)),

2279 nodes.literal_block(block_text, block_text), line=lineno)

2280 return [error], blank_finish

2281 directive_instance = directive(

2282 type_name, arguments, options, content, lineno,

2283 content_offset, block_text, self, self.state_machine)

2284 try:

2285 result = directive_instance.run()

2286 except docutils.parsers.rst.DirectiveError as error:

2287 msg_node = self.reporter.system_message(error.level, error.msg,

2288 line=lineno)

2289 msg_node += nodes.literal_block(block_text, block_text)

2290 result = [msg_node]

2291 assert isinstance(result, list), \

2292 'Directive "%s" must return a list of nodes.' % type_name

2293 for i in range(len(result)):

2294 assert isinstance(result[i], nodes.Node), \

2295 ('Directive "%s" returned non-Node object (index %s): %r'

2296 % (type_name, i, result[i]))

2297 return (result,

2298 blank_finish or self.state_machine.is_next_line_blank())

2299

2300 def parse_directive_block(self, indented, line_offset, directive,

2301 option_presets):

2302 option_spec = directive.option_spec

2303 has_content = directive.has_content

2304 if indented and not indented[0].strip():

2305 indented.trim_start()

2306 line_offset += 1

2307 while indented and not indented[-1].strip():

2308 indented.trim_end()

2309 if indented and (directive.required_arguments

2310 or directive.optional_arguments

2311 or option_spec):

2312 for i, line in enumerate(indented):

2313 if not line.strip():

2314 break

2315 else:

2316 i += 1

2317 arg_block = indented[:i]

2318 content = indented[i+1:]

2319 content_offset = line_offset + i + 1

2320 else:

2321 content = indented

2322 content_offset = line_offset

2323 arg_block = []

2324 if option_spec:

2325 options, arg_block = self.parse_directive_options(

2326 option_presets, option_spec, arg_block)

2327 else:

2328 options = {}

2329 if arg_block and not (directive.required_arguments

2330 or directive.optional_arguments):

2331 if options and content:

2332 self.reporter.info(

2333 'Directive content before and after options.',

2334 nodes.literal_block('', '\n'.join(indented)),

2335 line=content_offset)

2336 content = arg_block + indented[i:]

2337 content_offset = line_offset

2338 arg_block = []

2339 while content and not content[0].strip():

2340 content.trim_start()

2341 content_offset += 1

2342 if directive.required_arguments or directive.optional_arguments:

2343 arguments = self.parse_directive_arguments(

2344 directive, arg_block)

2345 else:

2346 arguments = []

2347 if content and not has_content:

2348 raise MarkupError('no content permitted')

2349 return arguments, options, content, content_offset

2350

2351 def parse_directive_options(self, option_presets, option_spec, arg_block):

2352 options = option_presets.copy()

2353 for i, line in enumerate(arg_block):

2354 if re.match(Body.patterns['field_marker'], line):

2355 opt_block = arg_block[i:]

2356 arg_block = arg_block[:i]

2357 break

2358 else:

2359 opt_block = []

2360 if opt_block:

2361 success, data = self.parse_extension_options(option_spec,

2362 opt_block)

2363 if success: # data is a dict of options

2364 options.update(data)

2365 else: # data is an error string

2366 raise MarkupError(data)

2367 return options, arg_block

2368

2369 def parse_directive_arguments(self, directive, arg_block):

2370 required = directive.required_arguments

2371 optional = directive.optional_arguments

2372 arg_text = '\n'.join(arg_block)

2373 arguments = arg_text.split()

2374 if len(arguments) < required:

2375 raise MarkupError('%s argument(s) required, %s supplied'

2376 % (required, len(arguments)))

2377 elif len(arguments) > required + optional:

2378 if directive.final_argument_whitespace:

2379 arguments = arg_text.split(None, required + optional - 1)

2380 else:

2381 raise MarkupError(

2382 'maximum %s argument(s) allowed, %s supplied'

2383 % (required + optional, len(arguments)))

2384 return arguments

2385

2386 def parse_extension_options(self, option_spec, datalines):

2387 """

2388 Parse `datalines` for a field list containing extension options

2389 matching `option_spec`.

2390

2391 :Parameters:

2392 - `option_spec`: a mapping of option name to conversion

2393 function, which should raise an exception on bad input.

2394 - `datalines`: a list of input strings.

2395

2396 :Return:

2397 - Success value, 1 or 0.

2398 - An option dictionary on success, an error string on failure.

2399 """

2400 node = nodes.field_list()

2401 newline_offset, blank_finish = self.nested_list_parse(

2402 datalines, 0, node, initial_state='ExtensionOptions',

2403 blank_finish=True)

2404 if newline_offset != len(datalines): # incomplete parse of block

2405 return 0, 'invalid option block'

2406 try:

2407 options = utils.extract_extension_options(node, option_spec)

2408 except KeyError as detail:

2409 return 0, 'unknown option: "%s"' % detail.args[0]

2410 except (ValueError, TypeError) as detail:

2411 return 0, 'invalid option value: %s' % ' '.join(detail.args)

2412 except utils.ExtensionOptionError as detail:

2413 return 0, 'invalid option data: %s' % ' '.join(detail.args)

2414 if blank_finish:

2415 return 1, options

2416 else:

2417 return 0, 'option data incompletely parsed'

2418

2419 def unknown_directive(self, type_name):

2420 lineno = self.state_machine.abs_line_number()

2421 (indented, indent, offset, blank_finish

2422 ) = self.state_machine.get_first_known_indented(0, strip_indent=False)

2423 text = '\n'.join(indented)

2424 error = self.reporter.error('Unknown directive type "%s".' % type_name,

2425 nodes.literal_block(text, text),

2426 line=lineno)

2427 return [error], blank_finish

2428

2429 def comment(self, match):

2430 if self.state_machine.is_next_line_blank():

2431 first_comment_line = match.string[match.end():]

2432 if not first_comment_line.strip(): # empty comment

2433 return [nodes.comment()], True # "A tiny but practical wart."

2434 if first_comment_line.startswith('end of inclusion from "'):

2435 # cf. parsers.rst.directives.misc.Include

2436 self.document.include_log.pop()

2437 return [], True

2438 (indented, indent, offset, blank_finish

2439 ) = self.state_machine.get_first_known_indented(match.end())

2440 while indented and not indented[-1].strip():

2441 indented.trim_end()

2442 text = '\n'.join(indented)

2443 return [nodes.comment(text, text)], blank_finish

2444

2445 explicit.constructs = [

2446 (footnote,

2447 re.compile(r"""

2448 \.\.[ ]+ # explicit markup start

2449 \[

2450 ( # footnote label:

2451 [0-9]+ # manually numbered footnote

2452 | # *OR*

2453 \# # anonymous auto-numbered footnote

2454 | # *OR*

2455 \#%s # auto-number ed?) footnote label

2456 | # *OR*

2457 \* # auto-symbol footnote

2458 )

2459 \]

2460 ([ ]+|$) # whitespace or end of line

2461 """ % Inliner.simplename, re.VERBOSE)),

2462 (citation,

2463 re.compile(r"""

2464 \.\.[ ]+ # explicit markup start

2465 \[(%s)\] # citation label

2466 ([ ]+|$) # whitespace or end of line

2467 """ % Inliner.simplename, re.VERBOSE)),

2468 (hyperlink_target,

2469 re.compile(r"""

2470 \.\.[ ]+ # explicit markup start

2471 _ # target indicator

2472 (?![ ]|$) # first char. not space or EOL

2473 """, re.VERBOSE)),

2474 (substitution_def,

2475 re.compile(r"""

2476 \.\.[ ]+ # explicit markup start

2477 \| # substitution indicator

2478 (?![ ]|$) # first char. not space or EOL

2479 """, re.VERBOSE)),

2480 (directive,

2481 re.compile(r"""

2482 \.\.[ ]+ # explicit markup start

2483 (%s) # directive name

2484 [ ]? # optional space

2485 :: # directive delimiter

2486 ([ ]+|$) # whitespace or end of line

2487 """ % Inliner.simplename, re.VERBOSE))]

2488

2489 def explicit_markup(self, match, context, next_state):

2490 """Footnotes, hyperlink targets, directives, comments."""

2491 nodelist, blank_finish = self.explicit_construct(match)

2492 self.parent += nodelist

2493 self.explicit_list(blank_finish)

2494 return [], next_state, []

2495

2496 def explicit_construct(self, match):

2497 """Determine which explicit construct this is, parse & return it."""

2498 errors = []

2499 for method, pattern in self.explicit.constructs:

2500 expmatch = pattern.match(match.string)

2501 if expmatch:

2502 try:

2503 return method(self, expmatch)

2504 except MarkupError as error:

2505 lineno = self.state_machine.abs_line_number()

2506 message = ' '.join(error.args)

2507 errors.append(self.reporter.warning(message, line=lineno))

2508 break

2509 nodelist, blank_finish = self.comment(match)

2510 return nodelist + errors, blank_finish

2511

2512 def explicit_list(self, blank_finish) -> None:

2513 """

2514 Create a nested state machine for a series of explicit markup

2515 constructs (including anonymous hyperlink targets).

2516 """

2517 offset = self.state_machine.line_offset + 1 # next line

2518 newline_offset, blank_finish = self.nested_list_parse(

2519 self.state_machine.input_lines[offset:],

2520 input_offset=self.state_machine.abs_line_offset() + 1,

2521 node=self.parent, initial_state='Explicit',

2522 blank_finish=blank_finish)

2523 self.goto_line(newline_offset)

2524 if not blank_finish:

2525 self.parent += self.unindent_warning('Explicit markup')

2526

2527 def anonymous(self, match, context, next_state):

2528 """Anonymous hyperlink targets."""

2529 nodelist, blank_finish = self.anonymous_target(match)

2530 self.parent += nodelist

2531 self.explicit_list(blank_finish)

2532 return [], next_state, []

2533

2534 def anonymous_target(self, match):

2535 lineno = self.state_machine.abs_line_number()

2536 (block, indent, offset, blank_finish

2537 ) = self.state_machine.get_first_known_indented(match.end(),

2538 until_blank=True)

2539 blocktext = match.string[:match.end()] + '\n'.join(block)

2540 block = [escape2null(line) for line in block]

2541 target = self.make_target(block, blocktext, lineno, '')

2542 return [target], blank_finish

2543

2544 def line(self, match, context, next_state):

2545 """Section title overline or transition marker."""

2546 if self.state_machine.match_titles:

2547 return [match.string], 'Line', []

2548 elif match.string.strip() == '::':

2549 raise statemachine.TransitionCorrection('text')

2550 elif len(match.string.strip()) < 4:

2551 msg = self.reporter.info(

2552 'Unexpected possible title overline or transition.\n'

2553 "Treating it as ordinary text because it's so short.",

2554 line=self.state_machine.abs_line_number())

2555 self.parent += msg

2556 raise statemachine.TransitionCorrection('text')

2557 else:

2558 blocktext = self.state_machine.line

2559 msg = self.reporter.error(

2560 'Unexpected section title or transition.',

2561 nodes.literal_block(blocktext, blocktext),

2562 line=self.state_machine.abs_line_number())

2563 self.parent += msg

2564 return [], next_state, []

2565

2566 def text(self, match, context, next_state):

2567 """Titles, definition lists, paragraphs."""

2568 return [match.string], 'Text', []

2569

2570

2571class RFC2822Body(Body):

2572

2573 """

2574 RFC2822 headers are only valid as the first constructs in documents. As

2575 soon as anything else appears, the `Body` state should take over.

2576 """

2577

2578 patterns = Body.patterns.copy() # can't modify the original

2579 patterns['rfc2822'] = r'[!-9;-~]+:( +|$)'

2580 initial_transitions = [(name, 'Body')

2581 for name in Body.initial_transitions]

2582 initial_transitions.insert(-1, ('rfc2822', 'Body')) # just before 'text'

2583

2584 def rfc2822(self, match, context, next_state):

2585 """RFC2822-style field list item."""

2586 fieldlist = nodes.field_list(classes=['rfc2822'])

2587 self.parent += fieldlist

2588 field, blank_finish = self.rfc2822_field(match)

2589 fieldlist += field

2590 offset = self.state_machine.line_offset + 1 # next line

2591 newline_offset, blank_finish = self.nested_list_parse(

2592 self.state_machine.input_lines[offset:],

2593 input_offset=self.state_machine.abs_line_offset() + 1,

2594 node=fieldlist, initial_state='RFC2822List',

2595 blank_finish=blank_finish)

2596 self.goto_line(newline_offset)

2597 if not blank_finish:

2598 self.parent += self.unindent_warning(

2599 'RFC2822-style field list')

2600 return [], next_state, []

2601

2602 def rfc2822_field(self, match):

2603 name = match.string[:match.string.find(':')]

2604 (indented, indent, line_offset, blank_finish

2605 ) = self.state_machine.get_first_known_indented(match.end(),

2606 until_blank=True)

2607 fieldnode = nodes.field()

2608 fieldnode += nodes.field_name(name, name)

2609 fieldbody = nodes.field_body('\n'.join(indented))

2610 fieldnode += fieldbody

2611 if indented:

2612 self.nested_parse(indented, input_offset=line_offset,

2613 node=fieldbody)

2614 return fieldnode, blank_finish

2615

2616

2617class SpecializedBody(Body):

2618

2619 """

2620 Superclass for second and subsequent compound element members. Compound

2621 elements are lists and list-like constructs.

2622

2623 All transition methods are disabled (redefined as `invalid_input`).

2624 Override individual methods in subclasses to re-enable.

2625

2626 For example, once an initial bullet list item, say, is recognized, the

2627 `BulletList` subclass takes over, with a "bullet_list" node as its

2628 container. Upon encountering the initial bullet list item, `Body.bullet`

2629 calls its ``self.nested_list_parse`` (`RSTState.nested_list_parse`), which

2630 starts up a nested parsing session with `BulletList` as the initial state.

2631 Only the ``bullet`` transition method is enabled in `BulletList`; as long

2632 as only bullet list items are encountered, they are parsed and inserted

2633 into the container. The first construct which is *not* a bullet list item

2634 triggers the `invalid_input` method, which ends the nested parse and

2635 closes the container. `BulletList` needs to recognize input that is

2636 invalid in the context of a bullet list, which means everything *other

2637 than* bullet list items, so it inherits the transition list created in

2638 `Body`.

2639 """

2640

2641 def invalid_input(self, match=None, context=None, next_state=None):

2642 """Not a compound element member. Abort this state machine."""

2643 self.state_machine.previous_line() # back up so parent SM can reassess

2644 raise EOFError

2645

2646 indent = invalid_input

2647 bullet = invalid_input

2648 enumerator = invalid_input

2649 field_marker = invalid_input

2650 option_marker = invalid_input

2651 doctest = invalid_input

2652 line_block = invalid_input

2653 grid_table_top = invalid_input

2654 simple_table_top = invalid_input

2655 explicit_markup = invalid_input

2656 anonymous = invalid_input

2657 line = invalid_input

2658 text = invalid_input

2659

2660

2661class BulletList(SpecializedBody):

2662

2663 """Second and subsequent bullet_list list_items."""

2664

2665 def bullet(self, match, context, next_state):

2666 """Bullet list item."""

2667 if match.string[0] != self.parent['bullet']:

2668 # different bullet: new list

2669 self.invalid_input()

2670 listitem, blank_finish = self.list_item(match.end())

2671 self.parent += listitem

2672 self.blank_finish = blank_finish

2673 return [], next_state, []

2674

2675

2676class DefinitionList(SpecializedBody):

2677

2678 """Second and subsequent definition_list_items."""

2679

2680 def text(self, match, context, next_state):

2681 """Definition lists."""

2682 return [match.string], 'Definition', []

2683

2684

2685class EnumeratedList(SpecializedBody):

2686

2687 """Second and subsequent enumerated_list list_items."""

2688

2689 def enumerator(self, match, context, next_state):

2690 """Enumerated list item."""

2691 format, sequence, text, ordinal = self.parse_enumerator(

2692 match, self.parent['enumtype'])

2693 if (format != self.format

2694 or (sequence != '#' and (sequence != self.parent['enumtype']

2695 or self.auto

2696 or ordinal != (self.lastordinal + 1)))

2697 or not self.is_enumerated_list_item(ordinal, sequence, format)):

2698 # different enumeration: new list

2699 self.invalid_input()

2700 if sequence == '#':

2701 self.auto = 1

2702 listitem, blank_finish = self.list_item(match.end())

2703 self.parent += listitem

2704 self.blank_finish = blank_finish

2705 self.lastordinal = ordinal

2706 return [], next_state, []

2707

2708

2709class FieldList(SpecializedBody):

2710

2711 """Second and subsequent field_list fields."""

2712

2713 def field_marker(self, match, context, next_state):

2714 """Field list field."""

2715 field, blank_finish = self.field(match)

2716 self.parent += field

2717 self.blank_finish = blank_finish

2718 return [], next_state, []

2719

2720

2721class OptionList(SpecializedBody):

2722

2723 """Second and subsequent option_list option_list_items."""

2724

2725 def option_marker(self, match, context, next_state):

2726 """Option list item."""

2727 try:

2728 option_list_item, blank_finish = self.option_list_item(match)

2729 except MarkupError:

2730 self.invalid_input()

2731 self.parent += option_list_item

2732 self.blank_finish = blank_finish

2733 return [], next_state, []

2734

2735

2736class RFC2822List(SpecializedBody, RFC2822Body):

2737

2738 """Second and subsequent RFC2822-style field_list fields."""

2739

2740 patterns = RFC2822Body.patterns

2741 initial_transitions = RFC2822Body.initial_transitions

2742

2743 def rfc2822(self, match, context, next_state):

2744 """RFC2822-style field list item."""

2745 field, blank_finish = self.rfc2822_field(match)

2746 self.parent += field

2747 self.blank_finish = blank_finish

2748 return [], 'RFC2822List', []

2749

2750 blank = SpecializedBody.invalid_input

2751

2752

2753class ExtensionOptions(FieldList):

2754

2755 """

2756 Parse field_list fields for extension options.

2757

2758 No nested parsing is done (including inline markup parsing).

2759 """

2760

2761 def parse_field_body(self, indented, offset, node) -> None:

2762 """Override `Body.parse_field_body` for simpler parsing."""

2763 lines = []

2764 for line in list(indented) + ['']:

2765 if line.strip():

2766 lines.append(line)

2767 elif lines:

2768 text = '\n'.join(lines)

2769 node += nodes.paragraph(text, text)

2770 lines = []

2771

2772

2773class LineBlock(SpecializedBody):

2774

2775 """Second and subsequent lines of a line_block."""

2776

2777 blank = SpecializedBody.invalid_input

2778

2779 def line_block(self, match, context, next_state):

2780 """New line of line block."""

2781 lineno = self.state_machine.abs_line_number()

2782 line, messages, blank_finish = self.line_block_line(match, lineno)

2783 self.parent += line

2784 self.parent.parent += messages

2785 self.blank_finish = blank_finish

2786 return [], next_state, []

2787

2788

2789class Explicit(SpecializedBody):

2790

2791 """Second and subsequent explicit markup construct."""

2792

2793 def explicit_markup(self, match, context, next_state):

2794 """Footnotes, hyperlink targets, directives, comments."""

2795 nodelist, blank_finish = self.explicit_construct(match)

2796 self.parent += nodelist

2797 self.blank_finish = blank_finish

2798 return [], next_state, []

2799

2800 def anonymous(self, match, context, next_state):

2801 """Anonymous hyperlink targets."""

2802 nodelist, blank_finish = self.anonymous_target(match)

2803 self.parent += nodelist

2804 self.blank_finish = blank_finish

2805 return [], next_state, []

2806

2807 blank = SpecializedBody.invalid_input

2808

2809

2810class SubstitutionDef(Body):

2811

2812 """

2813 Parser for the contents of a substitution_definition element.

2814 """

2815

2816 patterns = {

2817 'embedded_directive': re.compile(r'(%s)::( +|$)'

2818 % Inliner.simplename),

2819 'text': r''}

2820 initial_transitions = ['embedded_directive', 'text']

2821

2822 def embedded_directive(self, match, context, next_state):

2823 nodelist, blank_finish = self.directive(match,

2824 alt=self.parent['names'][0])

2825 self.parent += nodelist

2826 if not self.state_machine.at_eof():

2827 self.blank_finish = blank_finish

2828 raise EOFError

2829

2830 def text(self, match, context, next_state):

2831 if not self.state_machine.at_eof():

2832 self.blank_finish = self.state_machine.is_next_line_blank()

2833 raise EOFError

2834

2835

2836class Text(RSTState):

2837

2838 """

2839 Classifier of second line of a text block.

2840

2841 Could be a paragraph, a definition list item, or a title.

2842 """

2843

2844 patterns = {'underline': Body.patterns['line'],

2845 'text': r''}

2846 initial_transitions = [('underline', 'Body'), ('text', 'Body')]

2847

2848 def blank(self, match, context, next_state):

2849 """End of paragraph."""

2850 # NOTE: self.paragraph returns [node, system_message(s)], literalnext

2851 paragraph, literalnext = self.paragraph(

2852 context, self.state_machine.abs_line_number() - 1)

2853 self.parent += paragraph

2854 if literalnext:

2855 self.parent += self.literal_block()

2856 return [], 'Body', []

2857

2858 def eof(self, context):

2859 if context:

2860 self.blank(None, context, None)

2861 return []

2862

2863 def indent(self, match, context, next_state):

2864 """Definition list item."""

2865 dl = nodes.definition_list()

2866 # the definition list starts on the line before the indent:

2867 lineno = self.state_machine.abs_line_number() - 1

2868 dl.source, dl.line = self.state_machine.get_source_and_line(lineno)

2869 dl_item, blank_finish = self.definition_list_item(context)

2870 dl += dl_item

2871 self.parent += dl

2872 offset = self.state_machine.line_offset + 1 # next line

2873 newline_offset, blank_finish = self.nested_list_parse(

2874 self.state_machine.input_lines[offset:],

2875 input_offset=self.state_machine.abs_line_offset() + 1,

2876 node=dl, initial_state='DefinitionList',

2877 blank_finish=blank_finish, blank_finish_state='Definition')

2878 self.goto_line(newline_offset)

2879 if not blank_finish:

2880 self.parent += self.unindent_warning('Definition list')

2881 return [], 'Body', []

2882

2883 def underline(self, match, context, next_state):

2884 """Section title."""

2885 lineno = self.state_machine.abs_line_number()

2886 title = context[0].rstrip()

2887 underline = match.string.rstrip()

2888 source = title + '\n' + underline

2889 messages = []

2890 if column_width(title) > len(underline):

2891 if len(underline) < 4:

2892 if self.state_machine.match_titles:

2893 msg = self.reporter.info(

2894 'Possible title underline, too short for the title.\n'

2895 "Treating it as ordinary text because it's so short.",

2896 line=lineno)

2897 self.parent += msg

2898 raise statemachine.TransitionCorrection('text')

2899 else:

2900 blocktext = context[0] + '\n' + self.state_machine.line

2901 msg = self.reporter.warning(

2902 'Title underline too short.',

2903 nodes.literal_block(blocktext, blocktext),

2904 line=lineno)

2905 messages.append(msg)

2906 if not self.state_machine.match_titles:

2907 blocktext = context[0] + '\n' + self.state_machine.line

2908 # We need get_source_and_line() here to report correctly

2909 src, srcline = self.state_machine.get_source_and_line()

2910 # TODO: why is abs_line_number() == srcline+1

2911 # if the error is in a table (try with test_tables.py)?

2912 # print("get_source_and_line", srcline)

2913 # print("abs_line_number", self.state_machine.abs_line_number())

2914 msg = self.reporter.error(

2915 'Unexpected section title.',

2916 nodes.literal_block(blocktext, blocktext),

2917 source=src, line=srcline)

2918 self.parent += messages

2919 self.parent += msg

2920 return [], next_state, []

2921 style = underline[0]

2922 context[:] = []

2923 self.section(title, source, style, lineno - 1, messages)

2924 return [], next_state, []

2925

2926 def text(self, match, context, next_state):

2927 """Paragraph."""

2928 startline = self.state_machine.abs_line_number() - 1

2929 msg = None

2930 try:

2931 block = self.state_machine.get_text_block(flush_left=True)

2932 except statemachine.UnexpectedIndentationError as err:

2933 block, src, srcline = err.args

2934 msg = self.reporter.error('Unexpected indentation.',

2935 source=src, line=srcline)

2936 lines = context + list(block)

2937 paragraph, literalnext = self.paragraph(lines, startline)

2938 self.parent += paragraph

2939 self.parent += msg

2940 if literalnext:

2941 try:

2942 self.state_machine.next_line()

2943 except EOFError:

2944 pass

2945 self.parent += self.literal_block()

2946 return [], next_state, []

2947

2948 def literal_block(self):

2949 """Return a list of nodes."""

2950 (indented, indent, offset, blank_finish

2951 ) = self.state_machine.get_indented()

2952 while indented and not indented[-1].strip():

2953 indented.trim_end()

2954 if not indented:

2955 return self.quoted_literal_block()

2956 data = '\n'.join(indented)

2957 literal_block = nodes.literal_block(data, data)

2958 (literal_block.source,

2959 literal_block.line) = self.state_machine.get_source_and_line(offset+1)

2960 nodelist = [literal_block]

2961 if not blank_finish:

2962 nodelist.append(self.unindent_warning('Literal block'))

2963 return nodelist

2964

2965 def quoted_literal_block(self):

2966 abs_line_offset = self.state_machine.abs_line_offset()

2967 offset = self.state_machine.line_offset

2968 parent_node = nodes.Element()

2969 new_abs_offset = self.nested_parse(

2970 self.state_machine.input_lines[offset:],

2971 input_offset=abs_line_offset, node=parent_node, match_titles=False,

2972 state_machine_kwargs={'state_classes': (QuotedLiteralBlock,),

2973 'initial_state': 'QuotedLiteralBlock'})

2974 self.goto_line(new_abs_offset)

2975 return parent_node.children

2976

2977 def definition_list_item(self, termline):

2978 # the parser is already on the second (indented) line:

2979 dd_lineno = self.state_machine.abs_line_number()

2980 dt_lineno = dd_lineno - 1

2981 (indented, indent, line_offset, blank_finish

2982 ) = self.state_machine.get_indented()

2983 dl_item = nodes.definition_list_item(

2984 '\n'.join(termline + list(indented)))

2985 (dl_item.source,

2986 dl_item.line) = self.state_machine.get_source_and_line(dt_lineno)

2987 dt_nodes, messages = self.term(termline, dt_lineno)

2988 dl_item += dt_nodes

2989 dd = nodes.definition('', *messages)

2990 dd.source, dd.line = self.state_machine.get_source_and_line(dd_lineno)

2991 dl_item += dd

2992 if termline[0][-2:] == '::':

2993 dd += self.reporter.info(

2994 'Blank line missing before literal block (after the "::")? '

2995 'Interpreted as a definition list item.',

2996 line=dd_lineno)

2997 # TODO: drop a definition if it is an empty comment to allow

2998 # definition list items with several terms?

2999 # https://sourceforge.net/p/docutils/feature-requests/60/

3000 self.nested_parse(indented, input_offset=line_offset, node=dd)

3001 return dl_item, blank_finish

3002

3003 classifier_delimiter = re.compile(' +: +')

3004

3005 def term(self, lines, lineno):

3006 """Return a definition_list's term and optional classifiers."""

3007 assert len(lines) == 1

3008 text_nodes, messages = self.inline_text(lines[0], lineno)

3009 dt = nodes.term(lines[0])

3010 dt.source, dt.line = self.state_machine.get_source_and_line(lineno)

3011 node_list = [dt]

3012 for i in range(len(text_nodes)):

3013 node = text_nodes[i]

3014 if isinstance(node, nodes.Text):

3015 parts = self.classifier_delimiter.split(node)

3016 if len(parts) == 1:

3017 node_list[-1] += node

3018 else:

3019 text = parts[0].rstrip()

3020 textnode = nodes.Text(text)

3021 node_list[-1] += textnode

3022 node_list += [nodes.classifier(unescape(part, True), part)

3023 for part in parts[1:]]

3024 else:

3025 node_list[-1] += node

3026 return node_list, messages

3027

3028

3029class SpecializedText(Text):

3030

3031 """

3032 Superclass for second and subsequent lines of Text-variants.

3033

3034 All transition methods are disabled. Override individual methods in

3035 subclasses to re-enable.

3036 """

3037

3038 def eof(self, context):

3039 """Incomplete construct."""

3040 return []

3041

3042 def invalid_input(self, match=None, context=None, next_state=None):

3043 """Not a compound element member. Abort this state machine."""

3044 raise EOFError

3045

3046 blank = invalid_input

3047 indent = invalid_input

3048 underline = invalid_input

3049 text = invalid_input

3050

3051

3052class Definition(SpecializedText):

3053

3054 """Second line of potential definition_list_item."""

3055

3056 def eof(self, context):

3057 """Not a definition."""

3058 self.state_machine.previous_line(2) # so parent SM can reassess

3059 return []

3060

3061 def indent(self, match, context, next_state):

3062 """Definition list item."""

3063 dl_item, blank_finish = self.definition_list_item(context)

3064 self.parent += dl_item

3065 self.blank_finish = blank_finish

3066 return [], 'DefinitionList', []

3067

3068

3069class Line(SpecializedText):

3070

3071 """

3072 Second line of over- & underlined section title or transition marker.

3073 """

3074

3075 eofcheck = 1 # ignored, will be removed in Docutils 2.0.

3076

3077 def eof(self, context):

3078 """Transition marker at end of section or document."""

3079 marker = context[0].strip()

3080 if len(marker) < 4:

3081 self.state_correction(context)

3082 src, srcline = self.state_machine.get_source_and_line()

3083 # lineno = self.state_machine.abs_line_number() - 1

3084 transition = nodes.transition(rawsource=context[0])

3085 transition.source = src

3086 transition.line = srcline - 1

3087 # transition.line = lineno

3088 self.parent += transition

3089 return []

3090

3091 def blank(self, match, context, next_state):

3092 """Transition marker."""

3093 src, srcline = self.state_machine.get_source_and_line()

3094 marker = context[0].strip()

3095 if len(marker) < 4:

3096 self.state_correction(context)

3097 transition = nodes.transition(rawsource=marker)

3098 transition.source = src

3099 transition.line = srcline - 1

3100 self.parent += transition

3101 return [], 'Body', []

3102

3103 def text(self, match, context, next_state):

3104 """Potential over- & underlined title."""

3105 lineno = self.state_machine.abs_line_number() - 1

3106 overline = context[0]

3107 title = match.string

3108 underline = ''

3109 try:

3110 underline = self.state_machine.next_line()

3111 except EOFError:

3112 blocktext = overline + '\n' + title

3113 if len(overline.rstrip()) < 4:

3114 self.short_overline(context, blocktext, lineno, 2)

3115 else:

3116 msg = self.reporter.error(

3117 'Incomplete section title.',

3118 nodes.literal_block(blocktext, blocktext),

3119 line=lineno)

3120 self.parent += msg

3121 return [], 'Body', []

3122 source = '%s\n%s\n%s' % (overline, title, underline)

3123 overline = overline.rstrip()

3124 underline = underline.rstrip()

3125 if not self.transitions['underline'][0].match(underline):

3126 blocktext = overline + '\n' + title + '\n' + underline

3127 if len(overline.rstrip()) < 4:

3128 self.short_overline(context, blocktext, lineno, 2)

3129 else:

3130 msg = self.reporter.error(

3131 'Missing matching underline for section title overline.',

3132 nodes.literal_block(source, source),

3133 line=lineno)

3134 self.parent += msg

3135 return [], 'Body', []

3136 elif overline != underline:

3137 blocktext = overline + '\n' + title + '\n' + underline

3138 if len(overline.rstrip()) < 4:

3139 self.short_overline(context, blocktext, lineno, 2)

3140 else:

3141 msg = self.reporter.error(

3142 'Title overline & underline mismatch.',

3143 nodes.literal_block(source, source),

3144 line=lineno)

3145 self.parent += msg

3146 return [], 'Body', []

3147 title = title.rstrip()

3148 messages = []

3149 if column_width(title) > len(overline):

3150 blocktext = overline + '\n' + title + '\n' + underline

3151 if len(overline.rstrip()) < 4:

3152 self.short_overline(context, blocktext, lineno, 2)

3153 else:

3154 msg = self.reporter.warning(

3155 'Title overline too short.',

3156 nodes.literal_block(source, source),

3157 line=lineno)

3158 messages.append(msg)

3159 style = (overline[0], underline[0])

3160 self.section(title.lstrip(), source, style, lineno + 1, messages)

3161 return [], 'Body', []

3162

3163 indent = text # indented title

3164

3165 def underline(self, match, context, next_state):

3166 overline = context[0]

3167 blocktext = overline + '\n' + self.state_machine.line

3168 lineno = self.state_machine.abs_line_number() - 1

3169 if len(overline.rstrip()) < 4:

3170 self.short_overline(context, blocktext, lineno, 1)

3171 msg = self.reporter.error(

3172 'Invalid section title or transition marker.',

3173 nodes.literal_block(blocktext, blocktext),

3174 line=lineno)

3175 self.parent += msg

3176 return [], 'Body', []

3177

3178 def short_overline(self, context, blocktext, lineno, lines=1) -> None:

3179 msg = self.reporter.info(

3180 'Possible incomplete section title.\nTreating the overline as '

3181 "ordinary text because it's so short.",

3182 line=lineno)

3183 self.parent += msg

3184 self.state_correction(context, lines)

3185

3186 def state_correction(self, context, lines=1):

3187 self.state_machine.previous_line(lines)

3188 context[:] = []

3189 raise statemachine.StateCorrection('Body', 'text')

3190

3191

3192class QuotedLiteralBlock(RSTState):

3193

3194 """

3195 Nested parse handler for quoted (unindented) literal blocks.

3196

3197 Special-purpose. Not for inclusion in `state_classes`.

3198 """

3199

3200 patterns = {'initial_quoted': r'(%(nonalphanum7bit)s)' % Body.pats,

3201 'text': r''}

3202 initial_transitions = ('initial_quoted', 'text')

3203

3204 def __init__(self, state_machine, debug=False) -> None:

3205 RSTState.__init__(self, state_machine, debug)

3206 self.messages = []

3207 self.initial_lineno = None

3208

3209 def blank(self, match, context, next_state):

3210 if context:

3211 raise EOFError

3212 else:

3213 return context, next_state, []

3214

3215 def eof(self, context):

3216 if context:

3217 src, srcline = self.state_machine.get_source_and_line(

3218 self.initial_lineno)

3219 text = '\n'.join(context)

3220 literal_block = nodes.literal_block(text, text)

3221 literal_block.source = src

3222 literal_block.line = srcline

3223 self.parent += literal_block

3224 else:

3225 self.parent += self.reporter.warning(

3226 'Literal block expected; none found.',

3227 line=self.state_machine.abs_line_number()

3228 ) # src not available, statemachine.input_lines is empty

3229 self.state_machine.previous_line()

3230 self.parent += self.messages

3231 return []

3232

3233 def indent(self, match, context, next_state):

3234 assert context, ('QuotedLiteralBlock.indent: context should not '

3235 'be empty!')

3236 self.messages.append(

3237 self.reporter.error('Unexpected indentation.',

3238 line=self.state_machine.abs_line_number()))

3239 self.state_machine.previous_line()

3240 raise EOFError

3241

3242 def initial_quoted(self, match, context, next_state):

3243 """Match arbitrary quote character on the first line only."""

3244 self.remove_transition('initial_quoted')

3245 quote = match.string[0]

3246 pattern = re.compile(re.escape(quote))

3247 # New transition matches consistent quotes only:

3248 self.add_transition('quoted',

3249 (pattern, self.quoted, self.__class__.__name__))

3250 self.initial_lineno = self.state_machine.abs_line_number()

3251 return [match.string], next_state, []

3252

3253 def quoted(self, match, context, next_state):

3254 """Match consistent quotes on subsequent lines."""

3255 context.append(match.string)

3256 return context, next_state, []

3257

3258 def text(self, match, context, next_state):

3259 if context:

3260 self.messages.append(

3261 self.reporter.error('Inconsistent literal block quoting.',

3262 line=self.state_machine.abs_line_number()))

3263 self.state_machine.previous_line()

3264 raise EOFError

3265

3266

3267state_classes = (Body, BulletList, DefinitionList, EnumeratedList, FieldList,

3268 OptionList, LineBlock, ExtensionOptions, Explicit, Text,

3269 Definition, Line, SubstitutionDef, RFC2822Body, RFC2822List)

3270"""Standard set of State classes used to start `RSTStateMachine`."""