Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/docutils/parsers/rst/states.py: 76%

1# $Id: states.py 10391 2026-07-22 20:59:24Z milde $

2# Author: David Goodger <goodger@python.org>

3# Copyright: This module has been placed in the public domain.

5"""

6This is the ``docutils.parsers.rst.states`` module, the core of

7the reStructuredText parser. It defines the following:

9:Classes:

10 - `RSTStateMachine`: reStructuredText parser's entry point.

11 - `NestedStateMachine`: recursive StateMachine.

12 - `RSTState`: reStructuredText State superclass.

13 - `Inliner`: For parsing inline markup.

14 - `Body`: Generic classifier of the first line of a block.

15 - `SpecializedBody`: Superclass for compound element members.

16 - `BulletList`: Second and subsequent bullet_list list_items

17 - `DefinitionList`: Second+ definition_list_items.

18 - `EnumeratedList`: Second+ enumerated_list list_items.

19 - `FieldList`: Second+ fields.

20 - `OptionList`: Second+ option_list_items.

21 - `RFC2822List`: Second+ RFC2822-style fields.

22 - `ExtensionOptions`: Parses directive option fields.

23 - `Explicit`: Second+ explicit markup constructs.

24 - `SubstitutionDef`: For embedded directives in substitution definitions.

25 - `Text`: Classifier of second line of a text block.

26 - `SpecializedText`: Superclass for continuation lines of Text-variants.

27 - `Definition`: Second line of potential definition_list_item.

28 - `Line`: Second line of overlined section title or transition marker.

29 - `Struct`: obsolete, use `types.SimpleNamespace`.

31:Exception classes:

32 - `MarkupError`

33 - `ParserError`

34 - `MarkupMismatch`

36:Functions:

37 - `escape2null()`: Return a string, escape-backslashes converted to nulls.

38 - `unescape()`: Return a string, nulls removed or restored to backslashes.

40:Attributes:

41 - `state_classes`: set of State classes used with `RSTStateMachine`.

43Parser Overview

44===============

46The reStructuredText parser is implemented as a recursive state machine,

47examining its input one line at a time. To understand how the parser works,

48please first become familiar with the `docutils.statemachine` module. In the

49description below, references are made to classes defined in this module;

50please see the individual classes for details.

52Parsing proceeds as follows:

541. The state machine examines each line of input, checking each of the

55 transition patterns of the state `Body`, in order, looking for a match.

56 The implicit transitions (blank lines and indentation) are checked before

57 any others. The 'text' transition is a catch-all (matches anything).

592. The method associated with the matched transition pattern is called.

61 A. Some transition methods are self-contained, appending elements to the

62 document tree (`Body.doctest` parses a doctest block). The parser's

63 current line index is advanced to the end of the element, and parsing

64 continues with step 1.

66 B. Other transition methods trigger the creation of a nested state machine,

67 whose job is to parse a compound construct ('indent' does a block quote,

68 'bullet' does a bullet list, 'overline' does a section [first checking

69 for a valid section header], etc.).

71 - In the case of lists and explicit markup, a one-off state machine is

72 created and run to parse contents of the first item.

74 - A new state machine is created and its initial state is set to the

75 appropriate specialized state (`BulletList` in the case of the

76 'bullet' transition; see `SpecializedBody` for more detail). This

77 state machine is run to parse the compound element (or series of

78 explicit markup elements), and returns as soon as a non-member element

79 is encountered. For example, the `BulletList` state machine ends as

80 soon as it encounters an element which is not a list item of that

81 bullet list. The optional omission of inter-element blank lines is

82 enabled by this nested state machine.

84 - The current line index is advanced to the end of the elements parsed,

85 and parsing continues with step 1.

87 C. The result of the 'text' transition depends on the next line of text.

88 The current state is changed to `Text`, under which the second line is

89 examined. If the second line is:

91 - Indented: The element is a definition list item, and parsing proceeds

92 similarly to step 2.B, using the `DefinitionList` state.

94 - A line of uniform punctuation characters: The element is a section

95 header; again, parsing proceeds as in step 2.B, and `Body` is still

96 used.

98 - Anything else: The element is a paragraph, which is examined for

99 inline markup and appended to the parent element. Processing

100 continues with step 1.

101"""

102

103from __future__ import annotations

104

105__docformat__ = 'reStructuredText'

106

107import re

108from types import FunctionType, MethodType

109from types import SimpleNamespace as Struct

110import warnings

111

112from docutils import nodes, statemachine, utils

113from docutils import ApplicationError, DataError

114from docutils.statemachine import StateMachineWS, StateWS

115from docutils.nodes import fully_normalize_name as normalize_name

116from docutils.nodes import unescape, whitespace_normalize_name

117import docutils.parsers.rst

118from docutils.parsers.rst import directives, languages, tableparser, roles

119from docutils.utils import escape2null, column_width, strip_combining_chars

120from docutils.utils import punctuation_chars, urischemes

121from docutils.utils import split_escaped_whitespace

122from docutils.utils._roman_numerals import (InvalidRomanNumeralError,

123 RomanNumeral)

124

125TYPE_CHECKING = False

126if TYPE_CHECKING:

127 from docutils.statemachine import StringList

128

129

130class MarkupError(DataError): pass

131class UnknownInterpretedRoleError(DataError): pass

132class InterpretedRoleNotImplementedError(DataError): pass

133class ParserError(ApplicationError): pass

134class MarkupMismatch(Exception): pass

135

136

137class RSTStateMachine(StateMachineWS):

138

139 """

140 reStructuredText's master StateMachine.

141

142 The entry point to reStructuredText parsing is the `run()` method.

143 """

144 section_level_offset: int = 0

145 """Correction term for section level determination in nested parsing.

146

147 Updated by `RSTState.nested_parse()` and used in

148 `RSTState.check_subsection()` to compensate differences when

149 nested parsing uses a detached base node with a document-wide

150 section title style hierarchy or the current node with a new,

151 independent title style hierarchy.

152 """

153

154 def run(self, input_lines, document, input_offset=0, match_titles=True,

155 inliner=None) -> None:

156 """

157 Parse `input_lines` and modify the `document` node in place.

158

159 Extend `StateMachineWS.run()`: set up parse-global data and

160 run the StateMachine.

161 """

162 self.language = languages.get_language(

163 document.settings.language_code, document.reporter)

164 self.match_titles = match_titles

165 if inliner is None:

166 inliner = Inliner()

167 inliner.init_customizations(document.settings)

168 # A collection of objects to share with nested parsers.

169 # The attributes `reporter`, `section_level`, and

170 # `section_bubble_up_kludge` will be removed in Docutils 2.0

171 self.memo = Struct(document=document,

172 reporter=document.reporter, # ignored

173 language=self.language,

174 title_styles=[],

175 section_level=0, # ignored

176 section_bubble_up_kludge=False, # ignored

177 inliner=inliner)

178 self.document = document

179 self.attach_observer(document.note_source)

180 self.reporter = self.document.reporter

181 self.node = document

182 results = StateMachineWS.run(self, input_lines, input_offset,

183 input_source=document['source'])

184 assert results == [], 'RSTStateMachine.run() results should be empty!'

185 self.node = self.memo = None # remove unneeded references

186

187

188class NestedStateMachine(RSTStateMachine):

189 """

190 StateMachine run from within other StateMachine runs, to parse nested

191 document structures.

192 """

193

194 def __init__(self, state_classes, initial_state,

195 debug=False, parent_state_machine=None) -> None:

196

197 self.parent_state_machine = parent_state_machine

198 """The instance of the parent state machine."""

199

200 super().__init__(state_classes, initial_state, debug)

201

202 def run(self, input_lines, input_offset, memo, node, match_titles=True):

203 """

204 Parse `input_lines` and populate `node`.

205

206 Extend `StateMachineWS.run()`: set up document-wide data.

207 """

208 self.match_titles = match_titles

209 self.memo = memo

210 self.document = memo.document

211 self.attach_observer(self.document.note_source)

212 self.language = memo.language

213 self.reporter = self.document.reporter

214 self.node = node

215 results = StateMachineWS.run(self, input_lines, input_offset)

216 assert results == [], ('NestedStateMachine.run() results should be '

217 'empty!')

218 return results

219

220

221class RSTState(StateWS):

222

223 """

224 reStructuredText State superclass.

225

226 Contains methods used by all State subclasses.

227 """

228

229 nested_sm = NestedStateMachine

230 nested_sm_cache = []

231

232 def __init__(self, state_machine: RSTStateMachine, debug=False) -> None:

233 self.nested_sm_kwargs = {'state_classes': state_classes,

234 'initial_state': 'Body'}

235 StateWS.__init__(self, state_machine, debug)

236

237 def runtime_init(self) -> None:

238 StateWS.runtime_init(self)

239 memo = self.state_machine.memo

240 self.memo = memo

241 self.document = memo.document

242 self.inliner = memo.inliner

243 self.reporter = self.document.reporter

244 # enable the reporter to determine source and source-line

245 if not hasattr(self.reporter, 'get_source_and_line'):

246 self.reporter.get_source_and_line = self.state_machine.get_source_and_line # noqa:E501

247

248 @property

249 def parent(self) -> nodes.Element | None:

250 return self.state_machine.node

251

252 @parent.setter

253 def parent(self, value: nodes.Element):

254 self.state_machine.node = value

255

256 def goto_line(self, abs_line_offset) -> None:

257 """

258 Jump to input line `abs_line_offset`, ignoring jumps past the end.

259 """

260 try:

261 self.state_machine.goto_line(abs_line_offset)

262 except EOFError:

263 pass

264

265 def no_match(self, context, transitions):

266 """

267 Override `StateWS.no_match` to generate a system message.

268

269 This code should never be run.

270 """

271 self.reporter.severe(

272 'Internal error: no transition pattern match. State: "%s"; '

273 'transitions: %s; context: %s; current line: %r.'

274 % (self.__class__.__name__, transitions, context,

275 self.state_machine.line))

276 return context, None, []

277

278 def bof(self, context):

279 """Called at beginning of file."""

280 return [], []

281

282 def nested_parse(self,

283 block: StringList,

284 input_offset: int,

285 node: nodes.Element|None = None,

286 match_titles: bool = False,

287 state_machine_class: StateMachineWS|None = None,

288 state_machine_kwargs: dict|None = None

289 ) -> int:

290 """

291 Parse the input `block` with a nested state-machine rooted at `node`.

292

293 :block:

294 reStructuredText source extract.

295 :input_offset:

296 Line number at start of the block.

297 :node:

298 Base node. Generated nodes will be appended to this node.

299 Default: the "current node" (`self.state_machine.node`).

300 :match_titles:

301 Allow section titles?

302 Caution: With a custom base node, this may lead to an invalid

303 or mixed up document tree. [#]_

304 :state_machine_class:

305 Default: `NestedStateMachine`.

306 :state_machine_kwargs:

307 Keyword arguments for the state-machine instantiation.

308 Default: `self.nested_sm_kwargs`.

309

310 Create a new state-machine instance if required.

311 Return new offset.

312

313 .. [#] See also ``test_parsers/test_rst/test_nested_parsing.py``

314 and Sphinx's `nested_parse_to_nodes()`__.

315

316 __ https://www.sphinx-doc.org/en/master/extdev/utils.html

317 #sphinx.util.parsing.nested_parse_to_nodes

318 """

319 if node is None:

320 node = self.state_machine.node

321 use_default = 0

322 if state_machine_class is None:

323 state_machine_class = self.nested_sm

324 use_default += 1

325 if state_machine_kwargs is None:

326 state_machine_kwargs = self.nested_sm_kwargs

327 use_default += 1

328 my_state_machine = None

329 if use_default == 2:

330 try:

331 # get cached state machine, prevent others from using it

332 my_state_machine = self.nested_sm_cache.pop()

333 except IndexError:

334 pass

335 if not my_state_machine:

336 my_state_machine = state_machine_class(

337 debug=self.debug,

338 parent_state_machine=self.state_machine,

339 **state_machine_kwargs)

340 # Check if we may use sections (with a caveat for custom nodes

341 # that may be dummies to collect children):

342 if (node == self.state_machine.node

343 and not isinstance(node, (nodes.document, nodes.section))):

344 match_titles = False # avoid invalid sections

345 if match_titles:

346 # Compensate mismatch of known title styles and number of

347 # parent sections of the base node if the document wide

348 # title styles are used with a detached base node or

349 # a new list of title styles with the current parent node:

350 l_node = len(node.section_hierarchy())

351 l_start = min(len(self.parent.section_hierarchy()),

352 len(self.memo.title_styles))

353 my_state_machine.section_level_offset = l_start - l_node

354

355 # run the state machine and populate `node`:

356 block_length = len(block)

357 my_state_machine.run(block, input_offset, self.memo,

358 node, match_titles)

359

360 if match_titles:

361 if node == self.state_machine.node:

362 # Pass on the new "current node" to parent state machines:

363 sm = self.state_machine

364 try:

365 while True:

366 sm.node = my_state_machine.node

367 sm = sm.parent_state_machine

368 except AttributeError:

369 pass

370 # clean up

371 new_offset = my_state_machine.abs_line_offset()

372 if use_default == 2:

373 self.nested_sm_cache.append(my_state_machine)

374 else:

375 my_state_machine.unlink()

376 # No `block.parent` implies disconnected -- lines aren't in sync:

377 if block.parent and (len(block) - block_length) != 0:

378 # Adjustment for block if modified in nested parse:

379 self.state_machine.next_line(len(block) - block_length)

380 return new_offset

381

382 def nested_list_parse(self, block, input_offset, node, initial_state,

383 blank_finish,

384 blank_finish_state=None,

385 extra_settings={},

386 match_titles=None, # deprecated, will be removed

387 state_machine_class=None,

388 state_machine_kwargs=None):

389 """

390 Parse the input `block` with a nested state-machine rooted at `node`.

391

392 Create a new StateMachine rooted at `node` and run it over the

393 input `block` (see also `nested_parse()`).

394 Also keep track of optional intermediate blank lines and the

395 required final one.

396

397 Return new offset and a boolean indicating whether there was a

398 blank final line.

399 """

400 if match_titles is not None:

401 warnings.warn('The "match_titles" argument of '

402 'parsers.rst.states.RSTState.nested_list_parse() '

403 'is ignored and will be removed in Docutils 2.0.',

404 PendingDeprecationWarning, stacklevel=2)

405 if state_machine_class is None:

406 state_machine_class = self.nested_sm

407 if state_machine_kwargs is None:

408 state_machine_kwargs = self.nested_sm_kwargs.copy()

409 state_machine_kwargs['initial_state'] = initial_state

410 my_state_machine = state_machine_class(

411 debug=self.debug,

412 parent_state_machine=self.state_machine,

413 **state_machine_kwargs)

414 if blank_finish_state is None:

415 blank_finish_state = initial_state

416 my_state_machine.states[blank_finish_state].blank_finish = blank_finish

417 for key, value in extra_settings.items():

418 setattr(my_state_machine.states[initial_state], key, value)

419 my_state_machine.run(block, input_offset, memo=self.memo, node=node)

420 blank_finish = my_state_machine.states[blank_finish_state].blank_finish

421 my_state_machine.unlink()

422 return my_state_machine.abs_line_offset(), blank_finish

423

424 def section(self, title, source, style, lineno, messages) -> None:

425 """Check for a valid subsection and create one if it checks out."""

426 if self.check_subsection(source, style, lineno):

427 self.new_subsection(title, lineno, messages)

428

429 def check_subsection(self, source, style, lineno) -> bool:

430 """

431 Check for a valid subsection header. Update section data in `memo`.

432

433 When a new section is reached that isn't a subsection of the current

434 section, set `self.parent` to the new section's parent section

435 (or the root node if the new section is a top-level section).

436 """

437 title_styles = self.memo.title_styles

438 parent_sections = self.parent.section_hierarchy()

439 # current section level: (0 root, 1 section, 2 subsection, ...)

440 oldlevel = (len(parent_sections)

441 + self.state_machine.section_level_offset)

442 # new section level:

443 try: # check for existing title style

444 newlevel = title_styles.index(style) + 1

445 except ValueError: # new title style

446 newlevel = len(title_styles) + 1

447 # The new level must not be deeper than an immediate child

448 # of the current level:

449 if newlevel > oldlevel + 1:

450 styles = ' '.join('/'.join(style) for style in title_styles)

451 self.parent += self.reporter.error(

452 'Inconsistent title style:'

453 f' skip from level {oldlevel} to {newlevel}.',

454 nodes.literal_block('', source),

455 nodes.paragraph('', f'Established title styles: {styles}'),

456 line=lineno)

457 return False

458 if newlevel <= oldlevel:

459 # new section is sibling or higher up in the section hierarchy

460 try:

461 new_parent = parent_sections[newlevel-oldlevel-1].parent

462 except IndexError:

463 styles = ' '.join('/'.join(style) for style in title_styles)

464 details = (f'The parent of level {newlevel} sections cannot'

465 ' be reached. The parser is at section level'

466 f' {oldlevel} but the current node has only'

467 f' {len(parent_sections)} parent section(s).'

468 '\nOne reason may be a high level'

469 ' section used in a directive that parses its'

470 ' content into a base node not attached to'

471 ' the document\n(up to Docutils 0.21,'

472 ' these sections were silently dropped).')

473 self.parent += self.reporter.error(

474 f'A level {newlevel} section cannot be used here.',

475 nodes.literal_block('', source),

476 nodes.paragraph('', f'Established title styles: {styles}'),

477 nodes.paragraph('', details),

478 line=lineno)

479 return False

480 self.parent = new_parent

481 self.memo.section_level = newlevel - 1

482 if newlevel > len(title_styles):

483 title_styles.append(style)

484 return True

485

486 def title_inconsistent(self, sourcetext, lineno):

487 # Ignored. Will be removed in Docutils 2.0.

488 error = self.reporter.error(

489 'Title level inconsistent:', nodes.literal_block('', sourcetext),

490 line=lineno)

491 return error

492

493 def new_subsection(self, title, lineno, messages):

494 """Append new subsection to document tree."""

495 section_node = nodes.section()

496 self.parent += section_node

497 textnodes, title_messages = self.inline_text(title, lineno)

498 titlenode = nodes.title(title, '', *textnodes)

499 name = normalize_name(titlenode.astext())

500 section_node['names'].append(name)

501 section_node += titlenode

502 section_node += messages

503 section_node += title_messages

504 self.document.note_implicit_target(section_node, section_node)

505 # Update state:

506 self.parent = section_node

507 self.memo.section_level += 1

508

509 def paragraph(self, lines, lineno):

510 """

511 Return a list (paragraph & messages) & a boolean: literal_block next?

512 """

513 data = '\n'.join(lines).rstrip()

514 if re.search(r'(?<!\\)(\\\\)*::$', data):

515 if len(data) == 2:

516 return [], 1

517 elif data[-3] in ' \n':

518 text = data[:-3].rstrip()

519 else:

520 text = data[:-1]

521 literalnext = 1

522 else:

523 text = data

524 literalnext = 0

525 textnodes, messages = self.inline_text(text, lineno)

526 p = nodes.paragraph(data, '', *textnodes)

527 p.source, p.line = self.state_machine.get_source_and_line(lineno)

528 return [p] + messages, literalnext

529

530 def inline_text(self, text, lineno):

531 """

532 Return 2 lists: nodes (text and inline elements), and system_messages.

533 """

534 nodes, messages = self.inliner.parse(text, lineno,

535 self.memo, self.parent)

536 return nodes, messages

537

538 def unindent_warning(self, node_name):

539 # the actual problem is one line below the current line

540 lineno = self.state_machine.abs_line_number() + 1

541 return self.reporter.warning('%s ends without a blank line; '

542 'unexpected unindent.' % node_name,

543 line=lineno)

544

545

546def build_regexp(definition, compile_patterns=True):

547 """

548 Build, compile and return a regular expression based on `definition`.

549

550 :Parameter: `definition`: a 4-tuple (group name, prefix, suffix, parts),

551 where "parts" is a list of regular expressions and/or regular

552 expression definitions to be joined into an or-group.

553 """

554 name, prefix, suffix, parts = definition

555 part_strings = []

556 for part in parts:

557 if isinstance(part, tuple):

558 part_strings.append(build_regexp(part, None))

559 else:

560 part_strings.append(part)

561 or_group = '|'.join(part_strings)

562 regexp = '%(prefix)s(?P<%(name)s>%(or_group)s)%(suffix)s' % locals()

563 if compile_patterns:

564 return re.compile(regexp)

565 else:

566 return regexp

567

568

569class Inliner:

570

571 """

572 Parse inline markup; call the `parse()` method.

573 """

574

575 def __init__(self) -> None:

576 self.implicit_dispatch = []

577 """List of (pattern, bound method) tuples, used by

578 `self.implicit_inline`."""

579

580 def init_customizations(self, settings) -> None:

581 # lookahead and look-behind expressions for inline markup rules

582 if getattr(settings, 'character_level_inline_markup', False):

583 start_string_prefix = '(^|(?<!\x00))'

584 end_string_suffix = ''

585 else:

586 start_string_prefix = ('(^|(?<=\\s|[%s%s]))' %

587 (punctuation_chars.openers,

588 punctuation_chars.delimiters))

589 end_string_suffix = ('($|(?=\\s|[\x00%s%s%s]))' %

590 (punctuation_chars.closing_delimiters,

591 punctuation_chars.delimiters,

592 punctuation_chars.closers))

593 args = locals().copy()

594 args.update(vars(self.__class__))

595

596 parts = ('initial_inline', start_string_prefix, '',

597 [

598 ('start', '', self.non_whitespace_after, # simple start-strings

599 [r'\*\*', # strong

600 r'\*(?!\*)', # emphasis but not strong

601 r'``', # literal

602 r'_`', # inline internal target

603 r'\|(?!\|)'] # substitution reference

604 ),

605 ('whole', '', end_string_suffix, # whole constructs

606 [ # reference name & end-string

607 r'(?P<refname>%s)(?P<refend>__?)' % self.simplename,

608 ('footnotelabel', r'\[', r'(?P<fnend>\]_)',

609 [r'[0-9]+', # manually numbered

610 r'\#(%s)?' % self.simplename, # auto-numbered (w/ label?)

611 r'\*', # auto-symbol

612 r'(?P<citationlabel>%s)' % self.simplename, # citation ref

613 ]

614 )

615 ]

616 ),

617 ('backquote', # interpreted text or phrase reference

618 '(?P<role>(:%s:)?)' % self.simplename, # optional role

619 self.non_whitespace_after,

620 ['`(?!`)'] # but not literal

621 )

622 ]

623 )

624 self.start_string_prefix = start_string_prefix

625 self.end_string_suffix = end_string_suffix

626 self.parts = parts

627

628 self.patterns = Struct(

629 initial=build_regexp(parts),

630 emphasis=re.compile(self.non_whitespace_escape_before

631 + r'(\*)' + end_string_suffix),

632 strong=re.compile(self.non_whitespace_escape_before

633 + r'(\*\*)' + end_string_suffix),

634 interpreted_or_phrase_ref=re.compile(

635 r"""

636 %(non_unescaped_whitespace_escape_before)s

637 (

638 `

639 (?P<suffix>

640 (?P<role>:%(simplename)s:)?

641 (?P<refend>__?)?

642 )

643 )

644 %(end_string_suffix)s

645 """ % args, re.VERBOSE),

646 embedded_link=re.compile(

647 r"""

648 (

649 (?:[ \n]+|^) # spaces or beginning of line/string

650 < # open bracket

651 %(non_whitespace_after)s

652 (([^<>]|\x00[<>])+) # anything but unescaped angle brackets

653 %(non_whitespace_escape_before)s

654 > # close bracket

655 )

656 $ # end of string

657 """ % args, re.VERBOSE),

658 literal=re.compile(self.non_whitespace_before + '(``)'

659 + end_string_suffix),

660 target=re.compile(self.non_whitespace_escape_before

661 + r'(`)' + end_string_suffix),

662 substitution_ref=re.compile(self.non_whitespace_escape_before

663 + r'(\|_{0,2})'

664 + end_string_suffix),

665 email=re.compile(self.email_pattern % args + '$',

666 re.VERBOSE),

667 uri=re.compile(

668 (r"""

669 %(start_string_prefix)s

670 (?P<whole>

671 (?P<absolute> # absolute URI

672 (?P<scheme> # scheme (http, ftp, mailto)

673 [a-zA-Z][a-zA-Z0-9.+-]*

674 )

675 :

676 (

677 ( # either:

678 (//?)? # hierarchical URI

679 %(uric)s* # URI characters

680 %(uri_end)s # final URI char

681 )

682 ( # optional query

683 \?%(uric)s*

684 %(uri_end)s

685 )?

686 ( # optional fragment

687 \#%(uric)s*

688 %(uri_end)s

689 )?

690 )

691 )

692 | # *OR*

693 (?P<email> # email address

694 """ + self.email_pattern + r"""

695 )

696 )

697 %(end_string_suffix)s

698 """) % args, re.VERBOSE),

699 pep=re.compile(

700 r"""

701 %(start_string_prefix)s

702 (

703 (pep-(?P<pepnum1>\d+)(.txt)?) # reference to source file

704 |

705 (PEP\s+(?P<pepnum2>\d+)) # reference by name

706 )

707 %(end_string_suffix)s""" % args, re.VERBOSE),

708 rfc=re.compile(

709 r"""

710 %(start_string_prefix)s

711 (RFC(-|\s+)?(?P<rfcnum>\d+))

712 %(end_string_suffix)s""" % args, re.VERBOSE))

713

714 self.implicit_dispatch.append((self.patterns.uri,

715 self.standalone_uri))

716 if settings.pep_references:

717 self.implicit_dispatch.append((self.patterns.pep,

718 self.pep_reference))

719 if settings.rfc_references:

720 self.implicit_dispatch.append((self.patterns.rfc,

721 self.rfc_reference))

722

723 def parse(self, text, lineno, memo, parent):

724 # Needs to be refactored for nested inline markup.

725 # Add nested_parse() method?

726 """

727 Return 2 lists: nodes (text and inline elements), and system_messages.

728

729 Using `self.patterns.initial`, a pattern which matches start-strings

730 (emphasis, strong, interpreted, phrase reference, literal,

731 substitution reference, and inline target) and complete constructs

732 (simple reference, footnote reference), search for a candidate. When

733 one is found, check for validity (e.g., not a quoted '*' character).

734 If valid, search for the corresponding end string if applicable, and

735 check it for validity. If not found or invalid, generate a warning

736 and ignore the start-string. Implicit inline markup (e.g. standalone

737 URIs) is found last.

738

739 :text: source string

740 :lineno: absolute line number, cf. `statemachine.get_source_and_line()`

741 """

742 self.document = memo.document

743 self.language = memo.language

744 self.reporter = self.document.reporter

745 self.parent = parent

746 pattern_search = self.patterns.initial.search

747 dispatch = self.dispatch

748 remaining = escape2null(text)

749 processed = []

750 unprocessed = []

751 messages = []

752 while remaining:

753 match = pattern_search(remaining)

754 if match:

755 groups = match.groupdict()

756 method = dispatch[groups['start'] or groups['backquote']

757 or groups['refend'] or groups['fnend']]

758 before, inlines, remaining, sysmessages = method(self, match,

759 lineno)

760 unprocessed.append(before)

761 messages += sysmessages

762 if inlines:

763 processed += self.implicit_inline(''.join(unprocessed),

764 lineno)

765 processed += inlines

766 unprocessed = []

767 else:

768 break

769 remaining = ''.join(unprocessed) + remaining

770 if remaining:

771 processed += self.implicit_inline(remaining, lineno)

772 return processed, messages

773

774 # Inline object recognition

775 # -------------------------

776 # See also init_customizations().

777 non_whitespace_before = r'(?<!\s)'

778 non_whitespace_escape_before = r'(?<![\s\x00])'

779 non_unescaped_whitespace_escape_before = r'(?<!(?<!\x00)[\s\x00])'

780 non_whitespace_after = r'(?!\s)'

781 # Alphanumerics with isolated internal [-._+:] chars (i.e. not 2 together):

782 simplename = r'(?:(?!_)\w)+(?:[-._+:](?:(?!_)\w)+)*'

783 # Valid URI characters (see RFC 2396 & RFC 2732);

784 # final \x00 allows backslash escapes in URIs:

785 uric = r"""[-_.!~*'()[\];/:@&=+$,%a-zA-Z0-9\x00]"""

786 # Delimiter indicating the end of a URI (not part of the URI):

787 uri_end_delim = r"""[>]"""

788 # Last URI character; same as uric but no punctuation:

789 urilast = r"""[_~*/=+a-zA-Z0-9]"""

790 # End of a URI (either 'urilast' or 'uric followed by a

791 # uri_end_delim'):

792 uri_end = r"""(?:%(urilast)s|%(uric)s(?=%(uri_end_delim)s))""" % locals()

793 emailc = r"""[-_!~*'{|}/#?^`&=+$%a-zA-Z0-9\x00]"""

794 email_pattern = r"""

795 %(emailc)s+(?:\.%(emailc)s+)* # name

796 (?<!\x00)@ # at

797 %(emailc)s+(?:\.%(emailc)s*)* # host

798 %(uri_end)s # final URI char

799 """

800

801 def quoted_start(self, match):

802 """Test if inline markup start-string is 'quoted'.

803

804 'Quoted' in this context means the start-string is enclosed in a pair

805 of matching opening/closing delimiters (not necessarily quotes)

806 or at the end of the match.

807 """

808 string = match.string

809 start = match.start()

810 if start == 0: # start-string at beginning of text

811 return False

812 prestart = string[start - 1]

813 try:

814 poststart = string[match.end()]

815 except IndexError: # start-string at end of text

816 return True # not "quoted" but no markup start-string either

817 return punctuation_chars.match_chars(prestart, poststart)

818

819 def inline_obj(self, match, lineno, end_pattern, nodeclass,

820 restore_backslashes=False):

821 string = match.string

822 matchstart = match.start('start')

823 matchend = match.end('start')

824 if self.quoted_start(match):

825 return string[:matchend], [], string[matchend:], [], ''

826 endmatch = end_pattern.search(string[matchend:])

827 if endmatch and endmatch.start(1): # 1 or more chars

828 text = endmatch.string[:endmatch.start(1)]

829 if restore_backslashes:

830 text = unescape(text, True)

831 textend = matchend + endmatch.end(1)

832 rawsource = unescape(string[matchstart:textend], True)

833 node = nodeclass(rawsource, text)

834 return (string[:matchstart], [node],

835 string[textend:], [], endmatch.group(1))

836 role = 'target' if string[matchstart] == '_' else nodeclass.__name__

837 msg = self.reporter.warning(

838 f'Inline {role} start-string without end-string.',

839 line=lineno)

840 text = unescape(string[matchstart:matchend], True)

841 prb = self.problematic(text, text, msg)

842 return string[:matchstart], [prb], string[matchend:], [msg], ''

843

844 def problematic(self, text, rawsource, message):

845 msgid = self.document.set_id(message, self.parent)

846 problematic = nodes.problematic(rawsource, text, refid=msgid)

847 prbid = self.document.set_id(problematic)

848 message.add_backref(prbid)

849 return problematic

850

851 def emphasis(self, match, lineno):

852 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

853 match, lineno, self.patterns.emphasis, nodes.emphasis)

854 return before, inlines, remaining, sysmessages

855

856 def strong(self, match, lineno):

857 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

858 match, lineno, self.patterns.strong, nodes.strong)

859 return before, inlines, remaining, sysmessages

860

861 def interpreted_or_phrase_ref(self, match, lineno):

862 end_pattern = self.patterns.interpreted_or_phrase_ref

863 string = match.string

864 matchstart = match.start('backquote')

865 matchend = match.end('backquote')

866 rolestart = match.start('role')

867 role = match.group('role')

868 position = ''

869 if role:

870 role = role[1:-1]

871 position = 'prefix'

872 elif self.quoted_start(match):

873 return string[:matchend], [], string[matchend:], []

874 endmatch = end_pattern.search(string[matchend:])

875 if endmatch and endmatch.start(1): # 1 or more chars

876 textend = matchend + endmatch.end()

877 if endmatch.group('role'):

878 if role:

879 msg = self.reporter.warning(

880 'Multiple roles in interpreted text (both '

881 'prefix and suffix present; only one allowed).',

882 line=lineno)

883 text = unescape(string[rolestart:textend], True)

884 prb = self.problematic(text, text, msg)

885 return string[:rolestart], [prb], string[textend:], [msg]

886 role = endmatch.group('suffix')[1:-1]

887 position = 'suffix'

888 escaped = endmatch.string[:endmatch.start(1)]

889 rawsource = unescape(string[matchstart:textend], True)

890 if rawsource[-1:] == '_':

891 if role:

892 msg = self.reporter.warning(

893 'Mismatch: both interpreted text role %s and '

894 'reference suffix.' % position, line=lineno)

895 text = unescape(string[rolestart:textend], True)

896 prb = self.problematic(text, text, msg)

897 return string[:rolestart], [prb], string[textend:], [msg]

898 return self.phrase_ref(string[:matchstart], string[textend:],

899 rawsource, escaped)

900 else:

901 rawsource = unescape(string[rolestart:textend], True)

902 nodelist, messages = self.interpreted(rawsource, escaped, role,

903 lineno)

904 return (string[:rolestart], nodelist,

905 string[textend:], messages)

906 msg = self.reporter.warning(

907 'Inline interpreted text or phrase reference start-string '

908 'without end-string.', line=lineno)

909 text = unescape(string[matchstart:matchend], True)

910 prb = self.problematic(text, text, msg)

911 return string[:matchstart], [prb], string[matchend:], [msg]

912

913 def phrase_ref(self, before, after, rawsource, escaped, text=None):

914 # `text` is ignored (since 0.16)

915 match = self.patterns.embedded_link.search(escaped)

916 if match: # embedded <URI> or <alias_>

917 text = escaped[:match.start(0)]

918 unescaped = unescape(text)

919 rawtext = unescape(text, True)

920 aliastext = match.group(2)

921 rawaliastext = unescape(aliastext, True)

922 underscore_escaped = rawaliastext.endswith(r'\_')

923 if (aliastext.endswith('_')

924 and not (underscore_escaped

925 or self.patterns.uri.match(aliastext))):

926 aliastype = 'name'

927 alias = normalize_name(unescape(aliastext[:-1]))

928 target = nodes.target(match.group(1), refname=alias)

929 else:

930 aliastype = 'uri'

931 # remove unescaped whitespace

932 alias_parts = split_escaped_whitespace(match.group(2))

933 alias = ' '.join(''.join(part.split())

934 for part in alias_parts)

935 alias = self.adjust_uri(unescape(alias))

936 if alias.endswith(r'\_'):

937 alias = alias[:-2] + '_'

938 target = nodes.target(match.group(1), refuri=alias)

939 target.referenced = True

940 if not aliastext:

941 raise ApplicationError('problem with embedded link: %r'

942 % aliastext)

943 if not text:

944 text = alias

945 unescaped = unescape(text)

946 rawtext = rawaliastext

947 else:

948 text = escaped

949 unescaped = unescape(text)

950 target = None

951 rawtext = unescape(escaped, True)

952

953 refname = normalize_name(unescaped)

954 reference = nodes.reference(rawsource, text)

955 reference[0].rawsource = rawtext

956

957 node_list = [reference]

958

959 if rawsource[-2:] == '__':

960 if target and (aliastype == 'name'):

961 reference['refname'] = alias

962 self.document.note_refname(reference)

963 # self.document.note_indirect_target(target) # required?

964 elif target and (aliastype == 'uri'):

965 reference['refuri'] = alias

966 else:

967 reference['anonymous'] = True

968 else:

969 if target:

970 target['names'].append(refname)

971 if aliastype == 'name':

972 reference['refname'] = alias

973 self.document.note_indirect_target(target)

974 self.document.note_refname(reference)

975 else:

976 reference['refuri'] = alias

977 # target.note_referenced_by(name=refname)

978 self.document.note_implicit_target(target, self.parent)

979 node_list.append(target)

980 else:

981 reference['refname'] = refname

982 self.document.note_refname(reference)

983 return before, node_list, after, []

984

985 def adjust_uri(self, uri):

986 match = self.patterns.email.match(uri)

987 if match:

988 return 'mailto:' + uri

989 else:

990 return uri

991

992 def interpreted(self, rawsource, text, role, lineno):

993 role_fn, messages = roles.role(role, self.language, lineno,

994 self.reporter)

995 if role_fn:

996 nodes, messages2 = role_fn(role, rawsource, text, lineno, self)

997 return nodes, messages + messages2

998 else:

999 msg = self.reporter.error(

1000 'Unknown interpreted text role "%s".' % role,

1001 line=lineno)

1002 return ([self.problematic(rawsource, rawsource, msg)],

1003 messages + [msg])

1004

1005 def literal(self, match, lineno):

1006 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

1007 match, lineno, self.patterns.literal, nodes.literal,

1008 restore_backslashes=True)

1009 return before, inlines, remaining, sysmessages

1010

1011 def inline_internal_target(self, match, lineno):

1012 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

1013 match, lineno, self.patterns.target, nodes.inline)

1014 if inlines and isinstance(inlines[0], nodes.inline):

1015 assert len(inlines) == 1

1016 inline_target = inlines[0]

1017 name = normalize_name(inline_target.astext())

1018 inline_target['names'].append(name)

1019 self.document.note_explicit_target(inline_target, self.parent)

1020 return before, inlines, remaining, sysmessages

1021

1022 def substitution_reference(self, match, lineno):

1023 before, inlines, remaining, sysmessages, endstring = self.inline_obj(

1024 match, lineno, self.patterns.substitution_ref,

1025 nodes.substitution_reference)

1026 if len(inlines) == 1:

1027 subref_node = inlines[0]

1028 if isinstance(subref_node, nodes.substitution_reference):

1029 subref_text = subref_node.astext()

1030 self.document.note_substitution_ref(subref_node, subref_text)

1031 if endstring[-1:] == '_':

1032 reference_node = nodes.reference(

1033 '|%s%s' % (subref_text, endstring), '')

1034 if endstring[-2:] == '__':

1035 reference_node['anonymous'] = True

1036 else:

1037 reference_node['refname'] = normalize_name(subref_text)

1038 self.document.note_refname(reference_node)

1039 reference_node += subref_node

1040 inlines = [reference_node]

1041 return before, inlines, remaining, sysmessages

1042

1043 def footnote_reference(self, match, lineno):

1044 """

1045 Handles `nodes.footnote_reference` and `nodes.citation_reference`

1046 elements.

1047 """

1048 label = match.group('footnotelabel')

1049 refname = normalize_name(label)

1050 string = match.string

1051 before = string[:match.start('whole')]

1052 remaining = string[match.end('whole'):]

1053 if match.group('citationlabel'):

1054 refnode = nodes.citation_reference('[%s]_' % label,

1055 refname=refname)

1056 refnode += nodes.Text(label)

1057 self.document.note_citation_ref(refnode)

1058 else:

1059 refnode = nodes.footnote_reference('[%s]_' % label)

1060 if refname[0] == '#':

1061 refname = refname[1:]

1062 refnode['auto'] = 1

1063 self.document.note_autofootnote_ref(refnode)

1064 elif refname == '*':

1065 refname = ''

1066 refnode['auto'] = '*'

1067 self.document.note_symbol_footnote_ref(

1068 refnode)

1069 else:

1070 refnode += nodes.Text(label)

1071 if refname:

1072 refnode['refname'] = refname

1073 self.document.note_footnote_ref(refnode)

1074 if utils.get_trim_footnote_ref_space(self.document.settings):

1075 before = before.rstrip()

1076 return before, [refnode], remaining, []

1077

1078 def reference(self, match, lineno, anonymous=False):

1079 referencename = match.group('refname')

1080 refname = normalize_name(referencename)

1081 referencenode = nodes.reference(

1082 referencename + match.group('refend'), referencename)

1083 referencenode[0].rawsource = referencename

1084 if anonymous:

1085 referencenode['anonymous'] = True

1086 else:

1087 referencenode['refname'] = refname

1088 self.document.note_refname(referencenode)

1089 string = match.string

1090 matchstart = match.start('whole')

1091 matchend = match.end('whole')

1092 return string[:matchstart], [referencenode], string[matchend:], []

1093

1094 def anonymous_reference(self, match, lineno):

1095 return self.reference(match, lineno, anonymous=True)

1096

1097 def standalone_uri(self, match, lineno):

1098 if (not match.group('scheme')

1099 or match.group('scheme').lower() in urischemes.schemes):

1100 if match.group('email'):

1101 addscheme = 'mailto:'

1102 else:

1103 addscheme = ''

1104 text = match.group('whole')

1105 refuri = addscheme + unescape(text)

1106 reference = nodes.reference(unescape(text, True), text,

1107 refuri=refuri)

1108 return [reference]

1109 else: # not a valid scheme

1110 raise MarkupMismatch

1111

1112 def pep_reference(self, match, lineno):

1113 text = match.group(0)

1114 if text.startswith('pep-'):

1115 pepnum = int(unescape(match.group('pepnum1')))

1116 elif text.startswith('PEP'):

1117 pepnum = int(unescape(match.group('pepnum2')))

1118 else:

1119 raise MarkupMismatch

1120 ref = (self.document.settings.pep_base_url

1121 + self.document.settings.pep_file_url_template % pepnum)

1122 return [nodes.reference(unescape(text, True), text, refuri=ref)]

1123

1124 rfc_url = 'rfc%d.html'

1125

1126 def rfc_reference(self, match, lineno):

1127 text = match.group(0)

1128 if text.startswith('RFC'):

1129 rfcnum = int(unescape(match.group('rfcnum')))

1130 ref = self.document.settings.rfc_base_url + self.rfc_url % rfcnum

1131 else:

1132 raise MarkupMismatch

1133 return [nodes.reference(unescape(text, True), text, refuri=ref)]

1134

1135 def implicit_inline(self, text, lineno):

1136 """

1137 Check each of the patterns in `self.implicit_dispatch` for a match,

1138 and dispatch to the stored method for the pattern. Recursively check

1139 the text before and after the match. Return a list of `nodes.Text`

1140 and inline element nodes.

1141 """

1142 if not text:

1143 return []

1144 for pattern, method in self.implicit_dispatch:

1145 match = pattern.search(text)

1146 if match:

1147 try:

1148 # Must recurse on strings before *and* after the match;

1149 # there may be multiple patterns.

1150 return (self.implicit_inline(text[:match.start()], lineno)

1151 + method(match, lineno)

1152 + self.implicit_inline(text[match.end():], lineno))

1153 except MarkupMismatch:

1154 pass

1155 return [nodes.Text(text)]

1156

1157 dispatch = {'*': emphasis,

1158 '**': strong,

1159 '`': interpreted_or_phrase_ref,

1160 '``': literal,

1161 '_`': inline_internal_target,

1162 ']_': footnote_reference,

1163 '|': substitution_reference,

1164 '_': reference,

1165 '__': anonymous_reference}

1166

1167

1168def _loweralpha_to_int(s, _zero=(ord('a')-1)):

1169 return ord(s) - _zero

1170

1171

1172def _upperalpha_to_int(s, _zero=(ord('A')-1)):

1173 return ord(s) - _zero

1174

1175

1176class Body(RSTState):

1177

1178 """

1179 Generic classifier of the first line of a block.

1180 """

1181

1182 double_width_pad_char = tableparser.TableParser.double_width_pad_char

1183 """Padding character for East Asian double-width text."""

1184

1185 enum = Struct()

1186 """Enumerated list parsing information."""

1187

1188 enum.formatinfo = {

1189 'parens': Struct(prefix='(', suffix=')', start=1, end=-1),

1190 'rparen': Struct(prefix='', suffix=')', start=0, end=-1),

1191 'period': Struct(prefix='', suffix='.', start=0, end=-1)}

1192 enum.formats = enum.formatinfo.keys()

1193 enum.sequences = ['arabic', 'loweralpha', 'upperalpha',

1194 'lowerroman', 'upperroman'] # ORDERED!

1195 enum.sequencepats = {'arabic': '[0-9]+',

1196 'loweralpha': '[a-z]',

1197 'upperalpha': '[A-Z]',

1198 'lowerroman': '[ivxlcdm]+',

1199 'upperroman': '[IVXLCDM]+'}

1200 enum.converters = {'arabic': int,

1201 'loweralpha': _loweralpha_to_int,

1202 'upperalpha': _upperalpha_to_int,

1203 'lowerroman': RomanNumeral.from_string,

1204 'upperroman': RomanNumeral.from_string}

1205

1206 enum.sequenceregexps = {}

1207 for sequence in enum.sequences:

1208 enum.sequenceregexps[sequence] = re.compile(

1209 enum.sequencepats[sequence] + '$')

1210

1211 grid_table_top_pat = re.compile(r'\+-[-+]+-\+ *$')

1212 """Matches the top (& bottom) of a full table)."""

1213

1214 simple_table_top_pat = re.compile('=+( +=+)+ *$')

1215 """Matches the top of a simple table."""

1216

1217 simple_table_border_pat = re.compile('=+[ =]*$')

1218 """Matches the bottom & header bottom of a simple table."""

1219

1220 pats = {}

1221 """Fragments of patterns used by transitions."""

1222

1223 pats['nonalphanum7bit'] = '[!-/:-@[-`{-~]'

1224 pats['alpha'] = '[a-zA-Z]'

1225 pats['alphanum'] = '[a-zA-Z0-9]'

1226 pats['alphanumplus'] = '[a-zA-Z0-9_-]'

1227 pats['enum'] = ('(%(arabic)s|%(loweralpha)s|%(upperalpha)s|%(lowerroman)s'

1228 '|%(upperroman)s|#)' % enum.sequencepats)

1229 pats['optname'] = '%(alphanum)s%(alphanumplus)s*' % pats

1230 # @@@ Loosen up the pattern? Allow Unicode?

1231 pats['optarg'] = '(%(alpha)s%(alphanumplus)s*|<[^<>]+>)' % pats

1232 pats['shortopt'] = r'(-|\+)%(alphanum)s( ?%(optarg)s)?' % pats

1233 pats['longopt'] = r'(--|/)%(optname)s([ =]%(optarg)s)?' % pats

1234 pats['option'] = r'(%(shortopt)s|%(longopt)s)' % pats

1235

1236 for format in enum.formats:

1237 pats[format] = '(?P<%s>%s%s%s)' % (

1238 format, re.escape(enum.formatinfo[format].prefix),

1239 pats['enum'], re.escape(enum.formatinfo[format].suffix))

1240

1241 patterns = {

1242 'bullet': '[-+*\u2022\u2023\u2043]( +|$)',

1243 'enumerator': r'(%(parens)s|%(rparen)s|%(period)s)( +|$)' % pats,

1244 'field_marker': r':(?![: ])([^:\\]|\\.|:(?!([ `]|$)))*(?<! ):( +|$)',

1245 'option_marker': r'%(option)s(, %(option)s)*( +| ?$)' % pats,

1246 'doctest': r'>>>( +|$)',

1247 'line_block': r'\|( +|$)',

1248 'grid_table_top': grid_table_top_pat,

1249 'simple_table_top': simple_table_top_pat,

1250 'explicit_markup': r'\.\.( +|$)',

1251 'anonymous': r'__( +|$)',

1252 'line': r'(%(nonalphanum7bit)s)\1* *$' % pats,

1253 'text': r''}

1254 initial_transitions = (

1255 'bullet',

1256 'enumerator',

1257 'field_marker',

1258 'option_marker',

1259 'doctest',

1260 'line_block',

1261 'grid_table_top',

1262 'simple_table_top',

1263 'explicit_markup',

1264 'anonymous',

1265 'line',

1266 'text')

1267

1268 def indent(self, match, context, next_state):

1269 """Block quote."""

1270 (indented, indent, line_offset, blank_finish

1271 ) = self.state_machine.get_indented()

1272 elements = self.block_quote(indented, line_offset)

1273 self.parent += elements

1274 if not blank_finish:

1275 self.parent += self.unindent_warning('Block quote')

1276 return context, next_state, []

1277

1278 def block_quote(self, indented, line_offset):

1279 elements = []

1280 while indented:

1281 blockquote = nodes.block_quote(rawsource='\n'.join(indented))

1282 (blockquote.source, blockquote.line

1283 ) = self.state_machine.get_source_and_line(line_offset+1)

1284 (blockquote_lines,

1285 attribution_lines,

1286 attribution_offset,

1287 indented,

1288 new_line_offset) = self.split_attribution(indented, line_offset)

1289 self.nested_parse(blockquote_lines, line_offset, blockquote)

1290 elements.append(blockquote)

1291 if attribution_lines:

1292 attribution, messages = self.parse_attribution(

1293 attribution_lines, line_offset+attribution_offset)

1294 blockquote += attribution

1295 elements += messages

1296 line_offset = new_line_offset

1297 while indented and not indented[0]:

1298 indented = indented[1:]

1299 line_offset += 1

1300 return elements

1301

1302 # U+2014 is an em-dash:

1303 attribution_pattern = re.compile('(---?(?!-)|\u2014) *(?=[^ \\n])')

1304

1305 def split_attribution(self, indented, line_offset):

1306 """

1307 Check for a block quote attribution and split it off:

1308

1309 * First line after a blank line must begin with a dash ("--", "---",

1310 em-dash; matches `self.attribution_pattern`).

1311 * Every line after that must have consistent indentation.

1312 * Attributions must be preceded by block quote content.

1313

1314 Return a tuple of: (block quote content lines, attribution lines,

1315 attribution offset, remaining indented lines, remaining lines offset).

1316 """

1317 blank = None

1318 nonblank_seen = False

1319 for i in range(len(indented)):

1320 line = indented[i].rstrip()

1321 if line:

1322 if nonblank_seen and blank == i - 1: # last line blank

1323 match = self.attribution_pattern.match(line)

1324 if match:

1325 attribution_end, indent = self.check_attribution(

1326 indented, i)

1327 if attribution_end:

1328 a_lines = indented[i:attribution_end]

1329 a_lines.trim_left(match.end(), end=1)

1330 a_lines.trim_left(indent, start=1)

1331 return (indented[:i], a_lines,

1332 i, indented[attribution_end:],

1333 line_offset + attribution_end)

1334 nonblank_seen = True

1335 else:

1336 blank = i

1337 else:

1338 return indented, None, None, None, None

1339

1340 def check_attribution(self, indented, attribution_start):

1341 """

1342 Check attribution shape.

1343 Return the index past the end of the attribution, and the indent.

1344 """

1345 indent = None

1346 i = attribution_start + 1

1347 for i in range(attribution_start + 1, len(indented)):

1348 line = indented[i].rstrip()

1349 if not line:

1350 break

1351 if indent is None:

1352 indent = len(line) - len(line.lstrip())

1353 elif len(line) - len(line.lstrip()) != indent:

1354 return None, None # bad shape; not an attribution

1355 else:

1356 # return index of line after last attribution line:

1357 i += 1

1358 return i, (indent or 0)

1359

1360 def parse_attribution(self, indented, line_offset):

1361 text = '\n'.join(indented).rstrip()

1362 lineno = 1 + line_offset # line_offset is zero-based

1363 textnodes, messages = self.inline_text(text, lineno)

1364 node = nodes.attribution(text, '', *textnodes)

1365 node.source, node.line = self.state_machine.get_source_and_line(lineno)

1366 return node, messages

1367

1368 def bullet(self, match, context, next_state):

1369 """Bullet list item."""

1370 ul = nodes.bullet_list()

1371 ul.source, ul.line = self.state_machine.get_source_and_line()

1372 self.parent += ul

1373 ul['bullet'] = match.string[0]

1374 i, blank_finish = self.list_item(match.end())

1375 ul += i

1376 offset = self.state_machine.line_offset + 1 # next line

1377 new_line_offset, blank_finish = self.nested_list_parse(

1378 self.state_machine.input_lines[offset:],

1379 input_offset=self.state_machine.abs_line_offset() + 1,

1380 node=ul, initial_state='BulletList',

1381 blank_finish=blank_finish)

1382 self.goto_line(new_line_offset)

1383 if not blank_finish:

1384 self.parent += self.unindent_warning('Bullet list')

1385 return [], next_state, []

1386

1387 def list_item(self, indent):

1388 src, srcline = self.state_machine.get_source_and_line()

1389 if self.state_machine.line[indent:]:

1390 indented, line_offset, blank_finish = (

1391 self.state_machine.get_known_indented(indent))

1392 else:

1393 indented, indent, line_offset, blank_finish = (

1394 self.state_machine.get_first_known_indented(indent))

1395 listitem = nodes.list_item('\n'.join(indented))

1396 listitem.source, listitem.line = src, srcline

1397 if indented:

1398 self.nested_parse(indented, input_offset=line_offset,

1399 node=listitem)

1400 return listitem, blank_finish

1401

1402 def enumerator(self, match, context, next_state):

1403 """Enumerated List Item"""

1404 format, sequence, text, ordinal = self.parse_enumerator(match)

1405 if not self.is_enumerated_list_item(ordinal, sequence, format):

1406 raise statemachine.TransitionCorrection('text')

1407 enumlist = nodes.enumerated_list()

1408 (enumlist.source,

1409 enumlist.line) = self.state_machine.get_source_and_line()

1410 self.parent += enumlist

1411 if sequence == '#':

1412 enumlist['enumtype'] = 'arabic'

1413 else:

1414 enumlist['enumtype'] = sequence

1415 enumlist['prefix'] = self.enum.formatinfo[format].prefix

1416 enumlist['suffix'] = self.enum.formatinfo[format].suffix

1417 if ordinal != 1:

1418 enumlist['start'] = ordinal

1419 msg = self.reporter.info(

1420 'Enumerated list start value not ordinal-1: "%s" (ordinal %s)'

1421 % (text, ordinal), base_node=enumlist)

1422 self.parent += msg

1423 listitem, blank_finish = self.list_item(match.end())

1424 enumlist += listitem

1425 offset = self.state_machine.line_offset + 1 # next line

1426 newline_offset, blank_finish = self.nested_list_parse(

1427 self.state_machine.input_lines[offset:],

1428 input_offset=self.state_machine.abs_line_offset() + 1,

1429 node=enumlist, initial_state='EnumeratedList',

1430 blank_finish=blank_finish,

1431 extra_settings={'lastordinal': ordinal,

1432 'format': format,

1433 'auto': sequence == '#'})

1434 self.goto_line(newline_offset)

1435 if not blank_finish:

1436 self.parent += self.unindent_warning('Enumerated list')

1437 return [], next_state, []

1438

1439 def parse_enumerator(self, match, expected_sequence=None):

1440 """

1441 Analyze an enumerator and return the results.

1442

1443 :Return:

1444 - the enumerator format ('period', 'parens', or 'rparen'),

1445 - the sequence used ('arabic', 'loweralpha', 'upperroman', etc.),

1446 - the text of the enumerator, stripped of formatting, and

1447 - the ordinal value of the enumerator ('a' -> 1, 'ii' -> 2, etc.;

1448 ``None`` is returned for invalid enumerator text).

1449

1450 The enumerator format has already been determined by the regular

1451 expression match. If `expected_sequence` is given, that sequence is

1452 tried first. If not, we check for Roman numeral 1. This way,

1453 single-character Roman numerals (which are also alphabetical) can be

1454 matched. If no sequence has been matched, all sequences are checked in

1455 order.

1456 """

1457 groupdict = match.groupdict()

1458 sequence = ''

1459 for format in self.enum.formats:

1460 if groupdict[format]: # was this the format matched?

1461 break # yes; keep `format`

1462 else: # shouldn't happen

1463 raise ParserError('enumerator format not matched')

1464 text = groupdict[format][self.enum.formatinfo[format].start # noqa: E203,E501

1465 : self.enum.formatinfo[format].end]

1466 if text == '#':

1467 sequence = '#'

1468 elif expected_sequence:

1469 try:

1470 if self.enum.sequenceregexps[expected_sequence].match(text):

1471 sequence = expected_sequence

1472 except KeyError: # shouldn't happen

1473 raise ParserError('unknown enumerator sequence: %s'

1474 % sequence)

1475 elif text == 'i':

1476 sequence = 'lowerroman'

1477 elif text == 'I':

1478 sequence = 'upperroman'

1479 if not sequence:

1480 for sequence in self.enum.sequences:

1481 if self.enum.sequenceregexps[sequence].match(text):

1482 break

1483 else: # shouldn't happen

1484 raise ParserError('enumerator sequence not matched')

1485 if sequence == '#':

1486 ordinal = 1

1487 else:

1488 try:

1489 ordinal = int(self.enum.converters[sequence](text))

1490 except InvalidRomanNumeralError:

1491 ordinal = None

1492 return format, sequence, text, ordinal

1493

1494 def is_enumerated_list_item(self, ordinal, sequence, format):

1495 """

1496 Check validity based on the ordinal value and the second line.

1497

1498 Return true if the ordinal is valid and the second line is blank,

1499 indented, or starts with the next enumerator or an auto-enumerator.

1500 """

1501 if ordinal is None:

1502 return None

1503 try:

1504 next_line = self.state_machine.next_line()

1505 except EOFError: # end of input lines

1506 self.state_machine.previous_line()

1507 return 1

1508 else:

1509 self.state_machine.previous_line()

1510 if not next_line[:1].strip(): # blank or indented

1511 return 1

1512 result = self.make_enumerator(ordinal + 1, sequence, format)

1513 if result:

1514 next_enumerator, auto_enumerator = result

1515 try:

1516 if next_line.startswith((next_enumerator, auto_enumerator)):

1517 return 1

1518 except TypeError:

1519 pass

1520 return None

1521

1522 def make_enumerator(self, ordinal, sequence, format):

1523 """

1524 Construct and return the next enumerated list item marker, and an

1525 auto-enumerator ("#" instead of the regular enumerator).

1526

1527 Return ``None`` for invalid (out of range) ordinals.

1528 """

1529 if sequence == '#':

1530 enumerator = '#'

1531 elif sequence == 'arabic':

1532 enumerator = str(ordinal)

1533 else:

1534 if sequence.endswith('alpha'):

1535 if ordinal > 26:

1536 return None

1537 enumerator = chr(ordinal + ord('a') - 1)

1538 elif sequence.endswith('roman'):

1539 try:

1540 enumerator = RomanNumeral(ordinal).to_uppercase()

1541 except TypeError:

1542 return None

1543 else: # shouldn't happen

1544 raise ParserError('unknown enumerator sequence: "%s"'

1545 % sequence)

1546 if sequence.startswith('lower'):

1547 enumerator = enumerator.lower()

1548 elif sequence.startswith('upper'):

1549 enumerator = enumerator.upper()

1550 else: # shouldn't happen

1551 raise ParserError('unknown enumerator sequence: "%s"'

1552 % sequence)

1553 formatinfo = self.enum.formatinfo[format]

1554 next_enumerator = (formatinfo.prefix + enumerator + formatinfo.suffix

1555 + ' ')

1556 auto_enumerator = formatinfo.prefix + '#' + formatinfo.suffix + ' '

1557 return next_enumerator, auto_enumerator

1558

1559 def field_marker(self, match, context, next_state):

1560 """Field list item."""

1561 field_list = nodes.field_list()

1562 self.parent += field_list

1563 field, blank_finish = self.field(match)

1564 field_list += field

1565 offset = self.state_machine.line_offset + 1 # next line

1566 newline_offset, blank_finish = self.nested_list_parse(

1567 self.state_machine.input_lines[offset:],

1568 input_offset=self.state_machine.abs_line_offset() + 1,

1569 node=field_list, initial_state='FieldList',

1570 blank_finish=blank_finish)

1571 self.goto_line(newline_offset)

1572 if not blank_finish:

1573 self.parent += self.unindent_warning('Field list')

1574 return [], next_state, []

1575

1576 def field(self, match):

1577 name = self.parse_field_marker(match)

1578 src, srcline = self.state_machine.get_source_and_line()

1579 lineno = self.state_machine.abs_line_number()

1580 (indented, indent, line_offset, blank_finish

1581 ) = self.state_machine.get_first_known_indented(match.end())

1582 field_node = nodes.field()

1583 field_node.source = src

1584 field_node.line = srcline

1585 name_nodes, name_messages = self.inline_text(name, lineno)

1586 field_node += nodes.field_name(name, '', *name_nodes)

1587 field_body = nodes.field_body('\n'.join(indented), *name_messages)

1588 field_node += field_body

1589 if indented:

1590 self.parse_field_body(indented, line_offset, field_body)

1591 return field_node, blank_finish

1592

1593 def parse_field_marker(self, match):

1594 """Extract & return field name from a field marker match."""

1595 field = match.group()[1:] # strip off leading ':'

1596 field = field[:field.rfind(':')] # strip off trailing ':' etc.

1597 return field

1598

1599 def parse_field_body(self, indented, offset, node) -> None:

1600 self.nested_parse(indented, input_offset=offset, node=node)

1601

1602 def option_marker(self, match, context, next_state):

1603 """Option list item."""

1604 optionlist = nodes.option_list()

1605 (optionlist.source, optionlist.line

1606 ) = self.state_machine.get_source_and_line()

1607 try:

1608 listitem, blank_finish = self.option_list_item(match)

1609 except MarkupError as error:

1610 # This shouldn't happen; pattern won't match.

1611 msg = self.reporter.error('Invalid option list marker: %s'

1612 % error)

1613 self.parent += msg

1614 (indented, indent, line_offset, blank_finish

1615 ) = self.state_machine.get_first_known_indented(match.end())

1616 elements = self.block_quote(indented, line_offset)

1617 self.parent += elements

1618 if not blank_finish:

1619 self.parent += self.unindent_warning('Option list')

1620 return [], next_state, []

1621 self.parent += optionlist

1622 optionlist += listitem

1623 offset = self.state_machine.line_offset + 1 # next line

1624 newline_offset, blank_finish = self.nested_list_parse(

1625 self.state_machine.input_lines[offset:],

1626 input_offset=self.state_machine.abs_line_offset() + 1,

1627 node=optionlist, initial_state='OptionList',

1628 blank_finish=blank_finish)

1629 self.goto_line(newline_offset)

1630 if not blank_finish:

1631 self.parent += self.unindent_warning('Option list')

1632 return [], next_state, []

1633

1634 def option_list_item(self, match):

1635 offset = self.state_machine.abs_line_offset()

1636 options = self.parse_option_marker(match)

1637 (indented, indent, line_offset, blank_finish

1638 ) = self.state_machine.get_first_known_indented(match.end())

1639 if not indented: # not an option list item

1640 self.goto_line(offset)

1641 raise statemachine.TransitionCorrection('text')

1642 option_group = nodes.option_group('', *options)

1643 description = nodes.description('\n'.join(indented))

1644 option_list_item = nodes.option_list_item('', option_group,

1645 description)

1646 if indented:

1647 self.nested_parse(indented, input_offset=line_offset,

1648 node=description)

1649 return option_list_item, blank_finish

1650

1651 def parse_option_marker(self, match):

1652 """

1653 Return a list of `node.option` and `node.option_argument` objects,

1654 parsed from an option marker match.

1655

1656 :Exception: `MarkupError` for invalid option markers.

1657 """

1658 optlist = []

1659 # split at ", ", except inside < > (complex arguments)

1660 optionstrings = re.split(r', (?![^<]*>)', match.group().rstrip())

1661 for optionstring in optionstrings:

1662 tokens = optionstring.split()

1663 delimiter = ' '

1664 firstopt = tokens[0].split('=', 1)

1665 if len(firstopt) > 1:

1666 # "--opt=value" form

1667 tokens[:1] = firstopt

1668 delimiter = '='

1669 elif (len(tokens[0]) > 2

1670 and ((tokens[0].startswith('-')

1671 and not tokens[0].startswith('--'))

1672 or tokens[0].startswith('+'))):

1673 # "-ovalue" form

1674 tokens[:1] = [tokens[0][:2], tokens[0][2:]]

1675 delimiter = ''

1676 if len(tokens) > 1 and (tokens[1].startswith('<')

1677 and tokens[-1].endswith('>')):

1678 # "-o <value1 value2>" form; join all values into one token

1679 tokens[1:] = [' '.join(tokens[1:])]

1680 if 0 < len(tokens) <= 2:

1681 option = nodes.option(optionstring)

1682 option += nodes.option_string(tokens[0], tokens[0])

1683 if len(tokens) > 1:

1684 option += nodes.option_argument(tokens[1], tokens[1],

1685 delimiter=delimiter)

1686 optlist.append(option)

1687 else:

1688 raise MarkupError(

1689 'wrong number of option tokens (=%s), should be 1 or 2: '

1690 '"%s"' % (len(tokens), optionstring))

1691 return optlist

1692

1693 def doctest(self, match, context, next_state):

1694 line = self.document.current_line

1695 data = '\n'.join(self.state_machine.get_text_block())

1696 # TODO: Parse with `directives.body.CodeBlock` with

1697 # argument 'pycon' (Python Console) in Docutils 1.0.

1698 n = nodes.doctest_block(data, data)

1699 n.line = line

1700 self.parent += n

1701 return [], next_state, []

1702

1703 def line_block(self, match, context, next_state):

1704 """First line of a line block."""

1705 block = nodes.line_block()

1706 self.parent += block

1707 lineno = self.state_machine.abs_line_number()

1708 (block.source,

1709 block.line) = self.state_machine.get_source_and_line(lineno)

1710 line, messages, blank_finish = self.line_block_line(match, lineno)

1711 block += line

1712 self.parent += messages

1713 if not blank_finish:

1714 offset = self.state_machine.line_offset + 1 # next line

1715 new_line_offset, blank_finish = self.nested_list_parse(

1716 self.state_machine.input_lines[offset:],

1717 input_offset=self.state_machine.abs_line_offset() + 1,

1718 node=block, initial_state='LineBlock',

1719 blank_finish=False)

1720 self.goto_line(new_line_offset)

1721 if not blank_finish:

1722 self.parent += self.reporter.warning(

1723 'Line block ends without a blank line.',

1724 line=lineno+1)

1725 if len(block):

1726 if block[0].indent is None:

1727 block[0].indent = 0

1728 self.nest_line_block_lines(block)

1729 return [], next_state, []

1730

1731 def line_block_line(self, match, lineno):

1732 """Return one line element of a line_block."""

1733 (indented, indent, line_offset, blank_finish

1734 ) = self.state_machine.get_first_known_indented(match.end(),

1735 until_blank=True)

1736 text = '\n'.join(indented)

1737 text_nodes, messages = self.inline_text(text, lineno)

1738 line = nodes.line(text, '', *text_nodes)

1739 (line.source,

1740 line.line) = self.state_machine.get_source_and_line(lineno)

1741 if match.string.rstrip() != '|': # not empty

1742 line.indent = len(match.group(1)) - 1

1743 return line, messages, blank_finish

1744

1745 def nest_line_block_lines(self, block) -> None:

1746 for index in range(1, len(block)):

1747 if block[index].indent is None:

1748 block[index].indent = block[index - 1].indent

1749 self.nest_line_block_segment(block)

1750

1751 def nest_line_block_segment(self, block) -> None:

1752 indents = [item.indent for item in block]

1753 least = min(indents)

1754 new_items = []

1755 new_block = nodes.line_block()

1756 for item in block:

1757 if item.indent > least:

1758 new_block.append(item)

1759 else:

1760 if len(new_block):

1761 self.nest_line_block_segment(new_block)

1762 new_items.append(new_block)

1763 new_block = nodes.line_block()

1764 new_items.append(item)

1765 if len(new_block):

1766 self.nest_line_block_segment(new_block)

1767 new_items.append(new_block)

1768 block[:] = new_items

1769

1770 def grid_table_top(self, match, context, next_state):

1771 """Top border of a full table."""

1772 return self.table_top(match, context, next_state,

1773 self.isolate_grid_table,

1774 tableparser.GridTableParser)

1775

1776 def simple_table_top(self, match, context, next_state):

1777 """Top border of a simple table."""

1778 return self.table_top(match, context, next_state,

1779 self.isolate_simple_table,

1780 tableparser.SimpleTableParser)

1781

1782 def table_top(self, match, context, next_state,

1783 isolate_function, parser_class):

1784 """Top border of a generic table."""

1785 nodelist, blank_finish = self.table(isolate_function, parser_class)

1786 self.parent += nodelist

1787 if not blank_finish:

1788 msg = self.reporter.warning(

1789 'Blank line required after table.',

1790 line=self.state_machine.abs_line_number()+1)

1791 self.parent += msg

1792 return [], next_state, []

1793

1794 def table(self, isolate_function, parser_class):

1795 """Parse a table."""

1796 block, messages, blank_finish = isolate_function()

1797 if block:

1798 try:

1799 parser = parser_class()

1800 tabledata = parser.parse(block)

1801 tableline = (self.state_machine.abs_line_number() - len(block)

1802 + 1)

1803 table = self.build_table(tabledata, tableline)

1804 nodelist = [table] + messages

1805 except tableparser.TableMarkupError as err:

1806 nodelist = self.malformed_table(block, ' '.join(err.args),

1807 offset=err.offset) + messages

1808 else:

1809 nodelist = messages

1810 return nodelist, blank_finish

1811

1812 def isolate_grid_table(self):

1813 messages = []

1814 blank_finish = True

1815 try:

1816 block = self.state_machine.get_text_block(flush_left=True)

1817 except statemachine.UnexpectedIndentationError as err:

1818 block, src, srcline = err.args

1819 messages.append(self.reporter.error('Unexpected indentation.',

1820 source=src, line=srcline))

1821 blank_finish = False

1822 block.disconnect()

1823 # for East Asian chars:

1824 block.pad_double_width(self.double_width_pad_char)

1825 width = len(block[0].strip())

1826 for i in range(len(block)):

1827 block[i] = block[i].strip()

1828 if block[i][0] not in '+|': # check left edge

1829 blank_finish = False

1830 self.state_machine.previous_line(len(block) - i)

1831 del block[i:]

1832 break

1833 if not self.grid_table_top_pat.match(block[-1]): # find bottom

1834 # from second-last to third line of table:

1835 for i in range(len(block) - 2, 1, -1):

1836 if self.grid_table_top_pat.match(block[i]):

1837 self.state_machine.previous_line(len(block) - i + 1)

1838 del block[i+1:]

1839 blank_finish = False

1840 break

1841 else:

1842 detail = 'Bottom border missing or corrupt.'

1843 messages.extend(self.malformed_table(block, detail, i))

1844 return [], messages, blank_finish

1845 for i in range(len(block)): # check right edge

1846 if len(strip_combining_chars(block[i])

1847 ) != width or block[i][-1] not in '+|':

1848 detail = 'Right border not aligned or missing.'

1849 messages.extend(self.malformed_table(block, detail, i))

1850 return [], messages, blank_finish

1851 return block, messages, blank_finish

1852

1853 def isolate_simple_table(self):

1854 start = self.state_machine.line_offset

1855 lines = self.state_machine.input_lines

1856 limit = len(lines) - 1

1857 toplen = len(lines[start].strip())

1858 pattern_match = self.simple_table_border_pat.match

1859 found = 0

1860 found_at = None

1861 i = start + 1

1862 while i <= limit:

1863 line = lines[i]

1864 match = pattern_match(line)

1865 if match:

1866 if len(line.strip()) != toplen:

1867 self.state_machine.next_line(i - start)

1868 messages = self.malformed_table(

1869 lines[start:i+1], 'Bottom border or header rule does '

1870 'not match top border.', i-start)

1871 return [], messages, i == limit or not lines[i+1].strip()

1872 found += 1

1873 found_at = i

1874 if found == 2 or i == limit or not lines[i+1].strip():

1875 end = i

1876 break

1877 i += 1

1878 else: # reached end of input_lines

1879 details = 'No bottom table border found'

1880 if found:

1881 details += ' or no blank line after table bottom'

1882 self.state_machine.next_line(found_at - start)

1883 block = lines[start:found_at+1]

1884 else:

1885 self.state_machine.next_line(i - start - 1)

1886 block = lines[start:]

1887 messages = self.malformed_table(block, details + '.')

1888 return [], messages, not found

1889 self.state_machine.next_line(end - start)

1890 block = lines[start:end+1]

1891 # for East Asian chars:

1892 block.pad_double_width(self.double_width_pad_char)

1893 return block, [], end == limit or not lines[end+1].strip()

1894

1895 def malformed_table(self, block, detail='', offset=0):

1896 block.replace(self.double_width_pad_char, '')

1897 data = '\n'.join(block)

1898 message = 'Malformed table.'

1899 startline = self.state_machine.abs_line_number() - len(block) + 1

1900 if detail:

1901 message += '\n' + detail

1902 error = self.reporter.error(message, nodes.literal_block(data, data),

1903 line=startline+offset)

1904 return [error]

1905

1906 def build_table(self, tabledata, tableline, stub_columns=0, widths=None):

1907 colwidths, headrows, bodyrows = tabledata

1908 table = nodes.table()

1909 (table.source,

1910 table.line) = self.state_machine.get_source_and_line(tableline)

1911 if widths == 'auto':

1912 table['classes'] += ['colwidths-auto']

1913 elif widths: # "grid" or list of integers

1914 table['classes'] += ['colwidths-given']

1915 tgroup = nodes.tgroup(cols=len(colwidths))

1916 table += tgroup

1917 for colwidth in colwidths:

1918 colspec = nodes.colspec(colwidth=colwidth)

1919 if stub_columns:

1920 colspec.attributes['stub'] = True

1921 stub_columns -= 1

1922 tgroup += colspec

1923 if headrows:

1924 thead = nodes.thead()

1925 tgroup += thead

1926 for row in headrows:

1927 thead += self.build_table_row(row, tableline)

1928 tbody = nodes.tbody()

1929 tgroup += tbody

1930 for row in bodyrows:

1931 tbody += self.build_table_row(row, tableline)

1932 return table

1933

1934 def build_table_row(self, rowdata, tableline):

1935 row = nodes.row()

1936 for cell in rowdata:

1937 if cell is None:

1938 continue

1939 morerows, morecols, offset, cellblock = cell

1940 attributes = {}

1941 if morerows:

1942 attributes['morerows'] = morerows

1943 if morecols:

1944 attributes['morecols'] = morecols

1945 entry = nodes.entry(**attributes)

1946 row += entry

1947 if ''.join(cellblock):

1948 self.nested_parse(cellblock, input_offset=tableline+offset-1,

1949 node=entry)

1950 return row

1951

1952 explicit = Struct()

1953 """Patterns and constants used for explicit markup recognition."""

1954

1955 explicit.patterns = Struct(

1956 target=re.compile(r"""

1957 (

1958 _ # anonymous target

1959 | # *OR*

1960 (?!_) # no underscore at the beginning

1961 (?P<quote>`?) # optional open quote

1962 (?![ `]) # first char. not space or

1963 # backquote

1964 (?P<name> # reference name

1965 .+?

1966 )

1967 %(non_whitespace_escape_before)s

1968 (?P=quote) # close quote if open quote used

1969 )

1970 (?<!(?<!\x00):) # no unescaped colon at end

1971 %(non_whitespace_escape_before)s

1972 [ ]? # optional space

1973 : # end of reference name

1974 ([ ]+|$) # followed by whitespace

1975 """ % vars(Inliner), re.VERBOSE),

1976 reference=re.compile(r"""

1977 (

1978 (?P<simple>%(simplename)s)_

1979 | # *OR*

1980 ` # open backquote

1981 (?![ ]) # not space

1982 (?P<phrase>.+?) # hyperlink phrase

1983 %(non_whitespace_escape_before)s

1984 `_ # close backquote,

1985 # reference mark

1986 )

1987 $ # end of string

1988 """ % vars(Inliner), re.VERBOSE),

1989 substitution=re.compile(r"""

1990 (

1991 (?![ ]) # first char. not space

1992 (?P<name>.+?) # substitution text

1993 %(non_whitespace_escape_before)s

1994 \| # close delimiter

1995 )

1996 ([ ]+|$) # followed by whitespace

1997 """ % vars(Inliner),

1998 re.VERBOSE),)

1999

2000 def footnote(self, match):

2001 src, srcline = self.state_machine.get_source_and_line()

2002 (indented, indent, offset, blank_finish

2003 ) = self.state_machine.get_first_known_indented(match.end())

2004 label = match.group(1)

2005 name = normalize_name(label)

2006 footnote = nodes.footnote('\n'.join(indented))

2007 footnote.source = src

2008 footnote.line = srcline

2009 if name[0] == '#': # auto-numbered

2010 name = name[1:] # autonumber label

2011 footnote['auto'] = 1

2012 if name:

2013 footnote['names'].append(name)

2014 self.document.note_autofootnote(footnote)

2015 elif name == '*': # auto-symbol

2016 name = ''

2017 footnote['auto'] = '*'

2018 self.document.note_symbol_footnote(footnote)

2019 else: # manually numbered

2020 footnote += nodes.label('', label)

2021 footnote['names'].append(name)

2022 self.document.note_footnote(footnote)

2023 if name:

2024 self.document.note_explicit_target(footnote, footnote)

2025 else:

2026 self.document.set_id(footnote, footnote)

2027 if indented:

2028 self.nested_parse(indented, input_offset=offset, node=footnote)

2029 else:

2030 footnote += self.reporter.warning('Footnote content expected.')

2031 return [footnote], blank_finish

2032

2033 def citation(self, match):

2034 src, srcline = self.state_machine.get_source_and_line()

2035 (indented, indent, offset, blank_finish

2036 ) = self.state_machine.get_first_known_indented(match.end())

2037 label = match.group(1)

2038 name = normalize_name(label)

2039 citation = nodes.citation('\n'.join(indented))

2040 citation.source = src

2041 citation.line = srcline

2042 citation += nodes.label('', label)

2043 citation['names'].append(name)

2044 self.document.note_citation(citation)

2045 self.document.note_explicit_target(citation, citation)

2046 if indented:

2047 self.nested_parse(indented, input_offset=offset, node=citation)

2048 else:

2049 citation += self.reporter.warning('Citation content expected.')

2050 return [citation], blank_finish

2051

2052 def hyperlink_target(self, match):

2053 pattern = self.explicit.patterns.target

2054 lineno = self.state_machine.abs_line_number()

2055 (block, indent, offset, blank_finish

2056 ) = self.state_machine.get_first_known_indented(

2057 match.end(), until_blank=True, strip_indent=False)

2058 blocktext = match.string[:match.end()] + '\n'.join(block)

2059 block = [escape2null(line) for line in block]

2060 escaped = block[0]

2061 blockindex = 0

2062 while True:

2063 targetmatch = pattern.match(escaped)

2064 if targetmatch:

2065 break

2066 blockindex += 1

2067 try:

2068 escaped += block[blockindex]

2069 except IndexError:

2070 raise MarkupError('malformed hyperlink target.')

2071 del block[:blockindex]

2072 block[0] = (block[0] + ' ')[targetmatch.end()-len(escaped)-1:].strip()

2073 target = self.make_target(block, blocktext, lineno,

2074 targetmatch.group('name'))

2075 return [target], blank_finish

2076

2077 def make_target(self, block, block_text, lineno, target_name):

2078 target_type, data = self.parse_target(block, block_text, lineno)

2079 if target_type == 'refname':

2080 target = nodes.target(block_text, '', refname=normalize_name(data))

2081 self.add_target(target_name, '', target, lineno)

2082 self.document.note_indirect_target(target)

2083 return target

2084 elif target_type == 'refuri':

2085 target = nodes.target(block_text, '')

2086 self.add_target(target_name, data, target, lineno)

2087 return target

2088 else:

2089 return data

2090

2091 def parse_target(self, block, block_text, lineno):

2092 """

2093 Determine the type of reference of a target.

2094

2095 :Return: A 2-tuple, one of:

2096

2097 - 'refname' and the indirect reference name

2098 - 'refuri' and the URI

2099 - 'malformed' and a system_message node

2100 """

2101 if block and block[-1].strip()[-1:] == '_': # possible indirect target

2102 reference = ' '.join(line.strip() for line in block)

2103 refname = self.is_reference(reference)

2104 if refname:

2105 return 'refname', refname

2106 ref_parts = split_escaped_whitespace(' '.join(block))

2107 reference = ' '.join(''.join(unescape(part).split())

2108 for part in ref_parts)

2109 return 'refuri', reference

2110

2111 def is_reference(self, reference):

2112 match = self.explicit.patterns.reference.match(

2113 whitespace_normalize_name(reference))

2114 if not match:

2115 return None

2116 return unescape(match.group('simple') or match.group('phrase'))

2117

2118 def add_target(self, targetname, refuri, target, lineno):

2119 target.line = lineno

2120 if targetname:

2121 name = normalize_name(unescape(targetname))

2122 target['names'].append(name)

2123 if refuri:

2124 uri = self.inliner.adjust_uri(refuri)

2125 if uri:

2126 target['refuri'] = uri

2127 else:

2128 raise ApplicationError('problem with URI: %r' % refuri)

2129 self.document.note_explicit_target(target, self.parent)

2130 else: # anonymous target

2131 if refuri:

2132 target['refuri'] = refuri

2133 target['anonymous'] = True

2134 self.document.note_anonymous_target(target)

2135

2136 def substitution_def(self, match):

2137 pattern = self.explicit.patterns.substitution

2138 src, srcline = self.state_machine.get_source_and_line()

2139 (block, indent, offset, blank_finish

2140 ) = self.state_machine.get_first_known_indented(match.end(),

2141 strip_indent=False)

2142 blocktext = (match.string[:match.end()] + '\n'.join(block))

2143 block.disconnect()

2144 escaped = escape2null(block[0].rstrip())

2145 blockindex = 0

2146 while True:

2147 subdefmatch = pattern.match(escaped)

2148 if subdefmatch:

2149 break

2150 blockindex += 1

2151 try:

2152 escaped = escaped + ' ' + escape2null(

2153 block[blockindex].strip())

2154 except IndexError:

2155 raise MarkupError('malformed substitution definition.')

2156 del block[:blockindex] # strip out the substitution marker

2157 start = subdefmatch.end()-len(escaped)-1

2158 block[0] = (block[0].strip() + ' ')[start:-1]

2159 if not block[0]:

2160 del block[0]

2161 offset += 1

2162 while block and not block[-1].strip():

2163 block.pop()

2164 subname = subdefmatch.group('name')

2165 substitution_node = nodes.substitution_definition(blocktext)

2166 substitution_node.source = src

2167 substitution_node.line = srcline

2168 if not block:

2169 msg = self.reporter.warning(

2170 'Substitution definition "%s" missing contents.' % subname,

2171 nodes.literal_block(blocktext, blocktext),

2172 source=src, line=srcline)

2173 return [msg], blank_finish

2174 block[0] = block[0].strip()

2175 substitution_node['names'].append(

2176 nodes.whitespace_normalize_name(subname))

2177 new_abs_offset, blank_finish = self.nested_list_parse(

2178 block, input_offset=offset, node=substitution_node,

2179 initial_state='SubstitutionDef', blank_finish=blank_finish)

2180 i = 0

2181 for node in substitution_node[:]:

2182 if not (isinstance(node, nodes.Inline)

2183 or isinstance(node, nodes.Text)):

2184 self.parent += substitution_node[i]

2185 del substitution_node[i]

2186 else:

2187 i += 1

2188 for node in substitution_node.findall(nodes.Element,

2189 include_self=False):

2190 if isinstance(node, nodes.problematic):

2191 msg = self.reporter.error(

2192 'Problematic content in substitution definition',

2193 nodes.literal_block('', blocktext),

2194 source=src, line=srcline)

2195 msg.append(nodes.block_quote(

2196 '', nodes.paragraph('', '', *substitution_node.children)))

2197 return [msg], blank_finish

2198 illegal = self.disallowed_inside_substitution_definitions(node)

2199 if illegal:

2200 msg = self.reporter.error(f'{illegal} are not supported in '

2201 'a substitution definition.',

2202 nodes.literal_block('', blocktext),

2203 source=src, line=srcline)

2204 return [msg], blank_finish

2205 if len(substitution_node) == 0:

2206 msg = self.reporter.warning(

2207 'Substitution definition "%s" empty or invalid.' % subname,

2208 nodes.literal_block(blocktext, blocktext),

2209 source=src, line=srcline)

2210 return [msg], blank_finish

2211 self.document.note_substitution_def(

2212 substitution_node, subname, self.parent)

2213 return [substitution_node], blank_finish

2214

2215 def disallowed_inside_substitution_definitions(self, node) -> str:

2216 if isinstance(node, nodes.reference) and node.get('anonymous'):

2217 return 'Anonymous references'

2218 if isinstance(node, nodes.footnote_reference) and node.get('auto'):

2219 return 'References to auto-numbered and auto-symbol footnotes'

2220 if node['names'] or node['ids']:

2221 return 'Targets (names and identifiers)'

2222 else:

2223 return ''

2224

2225 def directive(self, match, **option_presets):

2226 """Returns a 2-tuple: list of nodes, and a "blank finish" boolean."""

2227 type_name = match.group(1)

2228 directive_class, messages = directives.directive(

2229 type_name, self.memo.language, self.document)

2230 self.parent += messages

2231 if directive_class:

2232 return self.run_directive(

2233 directive_class, match, type_name, option_presets)

2234 else:

2235 return self.unknown_directive(type_name)

2236

2237 def run_directive(self, directive, match, type_name, option_presets):

2238 """

2239 Parse a directive then run its directive function.

2240

2241 Parameters:

2242

2243 - `directive`: The class implementing the directive. Must be

2244 a subclass of `rst.Directive`.

2245

2246 - `match`: A regular expression match object which matched the first

2247 line of the directive.

2248

2249 - `type_name`: The directive name, as used in the source text.

2250

2251 - `option_presets`: A dictionary of preset options, defaults for the

2252 directive options. Currently, only an "alt" option is passed by

2253 substitution definitions (value: the substitution name), which may

2254 be used by an embedded image directive.

2255

2256 Returns a 2-tuple: list of nodes, and a "blank finish" boolean.

2257 """

2258 if isinstance(directive, (FunctionType, MethodType)):

2259 from docutils.parsers.rst import convert_directive_function

2260 directive = convert_directive_function(directive)

2261 lineno = self.state_machine.abs_line_number()

2262 initial_line_offset = self.state_machine.line_offset

2263 (indented, indent, line_offset, blank_finish

2264 ) = self.state_machine.get_first_known_indented(match.end(),

2265 strip_top=0)

2266 block_text = '\n'.join(self.state_machine.input_lines[

2267 initial_line_offset : self.state_machine.line_offset + 1]) # noqa: E203,E501

2268 try:

2269 arguments, options, content, content_offset = (

2270 self.parse_directive_block(indented, line_offset,

2271 directive, option_presets))

2272 except MarkupError as detail:

2273 error = self.reporter.error(

2274 'Error in "%s" directive:\n%s.' % (type_name,

2275 ' '.join(detail.args)),

2276 nodes.literal_block(block_text, block_text), line=lineno)

2277 return [error], blank_finish

2278 directive_instance = directive(

2279 type_name, arguments, options, content, lineno,

2280 content_offset, block_text, self, self.state_machine)

2281 try:

2282 result = directive_instance.run()

2283 except docutils.parsers.rst.DirectiveError as error:

2284 msg_node = self.reporter.system_message(error.level, error.msg,

2285 line=lineno)

2286 msg_node += nodes.literal_block(block_text, block_text)

2287 result = [msg_node]

2288 assert isinstance(result, list), \

2289 'Directive "%s" must return a list of nodes.' % type_name

2290 for i in range(len(result)):

2291 assert isinstance(result[i], nodes.Node), \

2292 ('Directive "%s" returned non-Node object (index %s): %r'

2293 % (type_name, i, result[i]))

2294 return (result,

2295 blank_finish or self.state_machine.is_next_line_blank())

2296

2297 def parse_directive_block(self, indented, line_offset, directive,

2298 option_presets):

2299 option_spec = directive.option_spec

2300 has_content = directive.has_content

2301 if indented and not indented[0].strip():

2302 indented.trim_start()

2303 line_offset += 1

2304 while indented and not indented[-1].strip():

2305 indented.trim_end()

2306 if indented and (directive.required_arguments

2307 or directive.optional_arguments

2308 or option_spec):

2309 for i, line in enumerate(indented):

2310 if not line.strip():

2311 break

2312 else:

2313 i += 1

2314 arg_block = indented[:i]

2315 content = indented[i+1:]

2316 content_offset = line_offset + i + 1

2317 else:

2318 content = indented

2319 content_offset = line_offset

2320 arg_block = []

2321 if option_spec:

2322 options, arg_block = self.parse_directive_options(

2323 option_presets, option_spec, arg_block)

2324 else:

2325 options = {}

2326 if arg_block and not (directive.required_arguments

2327 or directive.optional_arguments):

2328 if options and content:

2329 self.reporter.info(

2330 'Directive content before and after options.',

2331 nodes.literal_block('', '\n'.join(indented)),

2332 line=content_offset)

2333 content = arg_block + indented[i:]

2334 content_offset = line_offset

2335 arg_block = []

2336 while content and not content[0].strip():

2337 content.trim_start()

2338 content_offset += 1

2339 if directive.required_arguments or directive.optional_arguments:

2340 arguments = self.parse_directive_arguments(

2341 directive, arg_block)

2342 else:

2343 arguments = []

2344 if content and not has_content:

2345 raise MarkupError('no content permitted')

2346 return arguments, options, content, content_offset

2347

2348 def parse_directive_options(self, option_presets, option_spec, arg_block):

2349 options = option_presets.copy()

2350 for i, line in enumerate(arg_block):

2351 if re.match(Body.patterns['field_marker'], line):

2352 opt_block = arg_block[i:]

2353 arg_block = arg_block[:i]

2354 break

2355 else:

2356 opt_block = []

2357 if opt_block:

2358 success, data = self.parse_extension_options(option_spec,

2359 opt_block)

2360 if success: # data is a dict of options

2361 options.update(data)

2362 else: # data is an error string

2363 raise MarkupError(data)

2364 return options, arg_block

2365

2366 def parse_directive_arguments(self, directive, arg_block):

2367 required = directive.required_arguments

2368 optional = directive.optional_arguments

2369 arg_text = '\n'.join(arg_block)

2370 arguments = arg_text.split()

2371 if len(arguments) < required:

2372 raise MarkupError('%s argument(s) required, %s supplied'

2373 % (required, len(arguments)))

2374 elif len(arguments) > required + optional:

2375 if directive.final_argument_whitespace:

2376 arguments = arg_text.split(None, required + optional - 1)

2377 else:

2378 raise MarkupError(

2379 'maximum %s argument(s) allowed, %s supplied'

2380 % (required + optional, len(arguments)))

2381 return arguments

2382

2383 def parse_extension_options(self, option_spec, datalines):

2384 """

2385 Parse `datalines` for a field list containing extension options

2386 matching `option_spec`.

2387

2388 :Parameters:

2389 - `option_spec`: a mapping of option name to conversion

2390 function, which should raise an exception on bad input.

2391 - `datalines`: a list of input strings.

2392

2393 :Return:

2394 - Success value, 1 or 0.

2395 - An option dictionary on success, an error string on failure.

2396 """

2397 node = nodes.field_list()

2398 newline_offset, blank_finish = self.nested_list_parse(

2399 datalines, 0, node, initial_state='ExtensionOptions',

2400 blank_finish=True)

2401 if newline_offset != len(datalines): # incomplete parse of block

2402 return 0, 'invalid option block'

2403 try:

2404 options = utils.extract_extension_options(node, option_spec)

2405 except KeyError as detail:

2406 return 0, 'unknown option: "%s"' % detail.args[0]

2407 except (ValueError, TypeError) as detail:

2408 return 0, 'invalid option value: %s' % ' '.join(detail.args)

2409 except utils.ExtensionOptionError as detail:

2410 return 0, 'invalid option data: %s' % ' '.join(detail.args)

2411 if blank_finish:

2412 return 1, options

2413 else:

2414 return 0, 'option data incompletely parsed'

2415

2416 def unknown_directive(self, type_name):

2417 lineno = self.state_machine.abs_line_number()

2418 (indented, indent, offset, blank_finish

2419 ) = self.state_machine.get_first_known_indented(0, strip_indent=False)

2420 text = '\n'.join(indented)

2421 error = self.reporter.error('Unknown directive type "%s".' % type_name,

2422 nodes.literal_block(text, text),

2423 line=lineno)

2424 return [error], blank_finish

2425

2426 def comment(self, match):

2427 if self.state_machine.is_next_line_blank():

2428 first_comment_line = match.string[match.end():]

2429 if not first_comment_line.strip(): # empty comment

2430 return [nodes.comment()], True # "A tiny but practical wart."

2431 if first_comment_line.startswith('end of inclusion from "'):

2432 # cf. parsers.rst.directives.misc.Include

2433 self.document.include_log.pop()

2434 return [], True

2435 (indented, indent, offset, blank_finish

2436 ) = self.state_machine.get_first_known_indented(match.end())

2437 while indented and not indented[-1].strip():

2438 indented.trim_end()

2439 text = '\n'.join(indented)

2440 return [nodes.comment(text, text)], blank_finish

2441

2442 explicit.constructs = [

2443 (footnote,

2444 re.compile(r"""

2445 \.\.[ ]+ # explicit markup start

2446 \[

2447 ( # footnote label:

2448 [0-9]+ # manually numbered footnote

2449 | # *OR*

2450 \# # anonymous auto-numbered footnote

2451 | # *OR*

2452 \#%s # auto-number ed?) footnote label

2453 | # *OR*

2454 \* # auto-symbol footnote

2455 )

2456 \]

2457 ([ ]+|$) # whitespace or end of line

2458 """ % Inliner.simplename, re.VERBOSE)),

2459 (citation,

2460 re.compile(r"""

2461 \.\.[ ]+ # explicit markup start

2462 \[(%s)\] # citation label

2463 ([ ]+|$) # whitespace or end of line

2464 """ % Inliner.simplename, re.VERBOSE)),

2465 (hyperlink_target,

2466 re.compile(r"""

2467 \.\.[ ]+ # explicit markup start

2468 _ # target indicator

2469 (?![ ]|$) # first char. not space or EOL

2470 """, re.VERBOSE)),

2471 (substitution_def,

2472 re.compile(r"""

2473 \.\.[ ]+ # explicit markup start

2474 \| # substitution indicator

2475 (?![ ]|$) # first char. not space or EOL

2476 """, re.VERBOSE)),

2477 (directive,

2478 re.compile(r"""

2479 \.\.[ ]+ # explicit markup start

2480 (%s) # directive name

2481 [ ]? # optional space

2482 :: # directive delimiter

2483 ([ ]+|$) # whitespace or end of line

2484 """ % Inliner.simplename, re.VERBOSE))]

2485

2486 def explicit_markup(self, match, context, next_state):

2487 """Footnotes, hyperlink targets, directives, comments."""

2488 nodelist, blank_finish = self.explicit_construct(match)

2489 self.parent += nodelist

2490 self.explicit_list(blank_finish)

2491 return [], next_state, []

2492

2493 def explicit_construct(self, match):

2494 """Determine which explicit construct this is, parse & return it."""

2495 errors = []

2496 for method, pattern in self.explicit.constructs:

2497 expmatch = pattern.match(match.string)

2498 if expmatch:

2499 try:

2500 return method(self, expmatch)

2501 except MarkupError as error:

2502 lineno = self.state_machine.abs_line_number()

2503 message = ' '.join(error.args)

2504 errors.append(self.reporter.warning(message, line=lineno))

2505 break

2506 nodelist, blank_finish = self.comment(match)

2507 return nodelist + errors, blank_finish

2508

2509 def explicit_list(self, blank_finish) -> None:

2510 """

2511 Create a nested state machine for a series of explicit markup

2512 constructs (including anonymous hyperlink targets).

2513 """

2514 offset = self.state_machine.line_offset + 1 # next line

2515 newline_offset, blank_finish = self.nested_list_parse(

2516 self.state_machine.input_lines[offset:],

2517 input_offset=self.state_machine.abs_line_offset() + 1,

2518 node=self.parent, initial_state='Explicit',

2519 blank_finish=blank_finish)

2520 self.goto_line(newline_offset)

2521 if not blank_finish:

2522 self.parent += self.unindent_warning('Explicit markup')

2523

2524 def anonymous(self, match, context, next_state):

2525 """Anonymous hyperlink targets."""

2526 nodelist, blank_finish = self.anonymous_target(match)

2527 self.parent += nodelist

2528 self.explicit_list(blank_finish)

2529 return [], next_state, []

2530

2531 def anonymous_target(self, match):

2532 lineno = self.state_machine.abs_line_number()

2533 (block, indent, offset, blank_finish

2534 ) = self.state_machine.get_first_known_indented(match.end(),

2535 until_blank=True)

2536 blocktext = match.string[:match.end()] + '\n'.join(block)

2537 block = [escape2null(line) for line in block]

2538 target = self.make_target(block, blocktext, lineno, '')

2539 return [target], blank_finish

2540

2541 def line(self, match, context, next_state):

2542 """Section title overline or transition marker."""

2543 if self.state_machine.match_titles:

2544 return [match.string], 'Line', []

2545 elif match.string.strip() == '::':

2546 raise statemachine.TransitionCorrection('text')

2547 elif len(match.string.strip()) < 4:

2548 msg = self.reporter.info(

2549 'Unexpected possible title overline or transition.\n'

2550 "Treating it as ordinary text because it's so short.",

2551 line=self.state_machine.abs_line_number())

2552 self.parent += msg

2553 raise statemachine.TransitionCorrection('text')

2554 else:

2555 blocktext = self.state_machine.line

2556 msg = self.reporter.error(

2557 'Unexpected section title or transition.',

2558 nodes.literal_block(blocktext, blocktext),

2559 line=self.state_machine.abs_line_number())

2560 self.parent += msg

2561 return [], next_state, []

2562

2563 def text(self, match, context, next_state):

2564 """Titles, definition lists, paragraphs."""

2565 return [match.string], 'Text', []

2566

2567

2568class RFC2822Body(Body):

2569

2570 """

2571 RFC2822 headers are only valid as the first constructs in documents. As

2572 soon as anything else appears, the `Body` state should take over.

2573 """

2574

2575 patterns = Body.patterns.copy() # can't modify the original

2576 patterns['rfc2822'] = r'[!-9;-~]+:( +|$)'

2577 initial_transitions = [(name, 'Body')

2578 for name in Body.initial_transitions]

2579 initial_transitions.insert(-1, ('rfc2822', 'Body')) # just before 'text'

2580

2581 def rfc2822(self, match, context, next_state):

2582 """RFC2822-style field list item."""

2583 fieldlist = nodes.field_list(classes=['rfc2822'])

2584 self.parent += fieldlist

2585 field, blank_finish = self.rfc2822_field(match)

2586 fieldlist += field

2587 offset = self.state_machine.line_offset + 1 # next line

2588 newline_offset, blank_finish = self.nested_list_parse(

2589 self.state_machine.input_lines[offset:],

2590 input_offset=self.state_machine.abs_line_offset() + 1,

2591 node=fieldlist, initial_state='RFC2822List',

2592 blank_finish=blank_finish)

2593 self.goto_line(newline_offset)

2594 if not blank_finish:

2595 self.parent += self.unindent_warning(

2596 'RFC2822-style field list')

2597 return [], next_state, []

2598

2599 def rfc2822_field(self, match):

2600 name = match.string[:match.string.find(':')]

2601 (indented, indent, line_offset, blank_finish

2602 ) = self.state_machine.get_first_known_indented(match.end(),

2603 until_blank=True)

2604 fieldnode = nodes.field()

2605 fieldnode += nodes.field_name(name, name)

2606 fieldbody = nodes.field_body('\n'.join(indented))

2607 fieldnode += fieldbody

2608 if indented:

2609 self.nested_parse(indented, input_offset=line_offset,

2610 node=fieldbody)

2611 return fieldnode, blank_finish

2612

2613

2614class SpecializedBody(Body):

2615

2616 """

2617 Superclass for second and subsequent compound element members. Compound

2618 elements are lists and list-like constructs.

2619

2620 All transition methods are disabled (redefined as `invalid_input`).

2621 Override individual methods in subclasses to re-enable.

2622

2623 For example, once an initial bullet list item, say, is recognized, the

2624 `BulletList` subclass takes over, with a "bullet_list" node as its

2625 container. Upon encountering the initial bullet list item, `Body.bullet`

2626 calls its ``self.nested_list_parse`` (`RSTState.nested_list_parse`), which

2627 starts up a nested parsing session with `BulletList` as the initial state.

2628 Only the ``bullet`` transition method is enabled in `BulletList`; as long

2629 as only bullet list items are encountered, they are parsed and inserted

2630 into the container. The first construct which is *not* a bullet list item

2631 triggers the `invalid_input` method, which ends the nested parse and

2632 closes the container. `BulletList` needs to recognize input that is

2633 invalid in the context of a bullet list, which means everything *other

2634 than* bullet list items, so it inherits the transition list created in

2635 `Body`.

2636 """

2637

2638 def invalid_input(self, match=None, context=None, next_state=None):

2639 """Not a compound element member. Abort this state machine."""

2640 self.state_machine.previous_line() # back up so parent SM can reassess

2641 raise EOFError

2642

2643 indent = invalid_input

2644 bullet = invalid_input

2645 enumerator = invalid_input

2646 field_marker = invalid_input

2647 option_marker = invalid_input

2648 doctest = invalid_input

2649 line_block = invalid_input

2650 grid_table_top = invalid_input

2651 simple_table_top = invalid_input

2652 explicit_markup = invalid_input

2653 anonymous = invalid_input

2654 line = invalid_input

2655 text = invalid_input

2656

2657

2658class BulletList(SpecializedBody):

2659

2660 """Second and subsequent bullet_list list_items."""

2661

2662 def bullet(self, match, context, next_state):

2663 """Bullet list item."""

2664 if match.string[0] != self.parent['bullet']:

2665 # different bullet: new list

2666 self.invalid_input()

2667 listitem, blank_finish = self.list_item(match.end())

2668 self.parent += listitem

2669 self.blank_finish = blank_finish

2670 return [], next_state, []

2671

2672

2673class DefinitionList(SpecializedBody):

2674

2675 """Second and subsequent definition_list_items."""

2676

2677 def text(self, match, context, next_state):

2678 """Definition lists."""

2679 return [match.string], 'Definition', []

2680

2681

2682class EnumeratedList(SpecializedBody):

2683

2684 """Second and subsequent enumerated_list list_items."""

2685

2686 def enumerator(self, match, context, next_state):

2687 """Enumerated list item."""

2688 format, sequence, text, ordinal = self.parse_enumerator(

2689 match, self.parent['enumtype'])

2690 if (format != self.format

2691 or (sequence != '#' and (sequence != self.parent['enumtype']

2692 or self.auto

2693 or ordinal != (self.lastordinal + 1)))

2694 or not self.is_enumerated_list_item(ordinal, sequence, format)):

2695 # different enumeration: new list

2696 self.invalid_input()

2697 if sequence == '#':

2698 self.auto = 1

2699 listitem, blank_finish = self.list_item(match.end())

2700 self.parent += listitem

2701 self.blank_finish = blank_finish

2702 self.lastordinal = ordinal

2703 return [], next_state, []

2704

2705

2706class FieldList(SpecializedBody):

2707

2708 """Second and subsequent field_list fields."""

2709

2710 def field_marker(self, match, context, next_state):

2711 """Field list field."""

2712 field, blank_finish = self.field(match)

2713 self.parent += field

2714 self.blank_finish = blank_finish

2715 return [], next_state, []

2716

2717

2718class OptionList(SpecializedBody):

2719

2720 """Second and subsequent option_list option_list_items."""

2721

2722 def option_marker(self, match, context, next_state):

2723 """Option list item."""

2724 try:

2725 option_list_item, blank_finish = self.option_list_item(match)

2726 except MarkupError:

2727 self.invalid_input()

2728 self.parent += option_list_item

2729 self.blank_finish = blank_finish

2730 return [], next_state, []

2731

2732

2733class RFC2822List(SpecializedBody, RFC2822Body):

2734

2735 """Second and subsequent RFC2822-style field_list fields."""

2736

2737 patterns = RFC2822Body.patterns

2738 initial_transitions = RFC2822Body.initial_transitions

2739

2740 def rfc2822(self, match, context, next_state):

2741 """RFC2822-style field list item."""

2742 field, blank_finish = self.rfc2822_field(match)

2743 self.parent += field

2744 self.blank_finish = blank_finish

2745 return [], 'RFC2822List', []

2746

2747 blank = SpecializedBody.invalid_input

2748

2749

2750class ExtensionOptions(FieldList):

2751

2752 """

2753 Parse field_list fields for extension options.

2754

2755 No nested parsing is done (including inline markup parsing).

2756 """

2757

2758 def parse_field_body(self, indented, offset, node) -> None:

2759 """Override `Body.parse_field_body` for simpler parsing."""

2760 lines = []

2761 for line in list(indented) + ['']:

2762 if line.strip():

2763 lines.append(line)

2764 elif lines:

2765 text = '\n'.join(lines)

2766 node += nodes.paragraph(text, text)

2767 lines = []

2768

2769

2770class LineBlock(SpecializedBody):

2771

2772 """Second and subsequent lines of a line_block."""

2773

2774 blank = SpecializedBody.invalid_input

2775

2776 def line_block(self, match, context, next_state):

2777 """New line of line block."""

2778 lineno = self.state_machine.abs_line_number()

2779 line, messages, blank_finish = self.line_block_line(match, lineno)

2780 self.parent += line

2781 self.parent.parent += messages

2782 self.blank_finish = blank_finish

2783 return [], next_state, []

2784

2785

2786class Explicit(SpecializedBody):

2787

2788 """Second and subsequent explicit markup construct."""

2789

2790 def explicit_markup(self, match, context, next_state):

2791 """Footnotes, hyperlink targets, directives, comments."""

2792 nodelist, blank_finish = self.explicit_construct(match)

2793 self.parent += nodelist

2794 self.blank_finish = blank_finish

2795 return [], next_state, []

2796

2797 def anonymous(self, match, context, next_state):

2798 """Anonymous hyperlink targets."""

2799 nodelist, blank_finish = self.anonymous_target(match)

2800 self.parent += nodelist

2801 self.blank_finish = blank_finish

2802 return [], next_state, []

2803

2804 blank = SpecializedBody.invalid_input

2805

2806

2807class SubstitutionDef(Body):

2808

2809 """

2810 Parser for the contents of a substitution_definition element.

2811 """

2812

2813 patterns = {

2814 'embedded_directive': re.compile(r'(%s)::( +|$)'

2815 % Inliner.simplename),

2816 'text': r''}

2817 initial_transitions = ['embedded_directive', 'text']

2818

2819 def embedded_directive(self, match, context, next_state):

2820 nodelist, blank_finish = self.directive(match,

2821 alt=self.parent['names'][0])

2822 self.parent += nodelist

2823 if not self.state_machine.at_eof():

2824 self.blank_finish = blank_finish

2825 raise EOFError

2826

2827 def text(self, match, context, next_state):

2828 if not self.state_machine.at_eof():

2829 self.blank_finish = self.state_machine.is_next_line_blank()

2830 raise EOFError

2831

2832

2833class Text(RSTState):

2834

2835 """

2836 Classifier of second line of a text block.

2837

2838 Could be a paragraph, a definition list item, or a title.

2839 """

2840

2841 patterns = {'underline': Body.patterns['line'],

2842 'text': r''}

2843 initial_transitions = [('underline', 'Body'), ('text', 'Body')]

2844

2845 def blank(self, match, context, next_state):

2846 """End of paragraph."""

2847 # NOTE: self.paragraph returns [node, system_message(s)], literalnext

2848 paragraph, literalnext = self.paragraph(

2849 context, self.state_machine.abs_line_number() - 1)

2850 self.parent += paragraph

2851 if literalnext:

2852 self.parent += self.literal_block()

2853 return [], 'Body', []

2854

2855 def eof(self, context):

2856 if context:

2857 self.blank(None, context, None)

2858 return []

2859

2860 def indent(self, match, context, next_state):

2861 """Definition list item."""

2862 dl = nodes.definition_list()

2863 # the definition list starts on the line before the indent:

2864 lineno = self.state_machine.abs_line_number() - 1

2865 dl.source, dl.line = self.state_machine.get_source_and_line(lineno)

2866 dl_item, blank_finish = self.definition_list_item(context)

2867 dl += dl_item

2868 self.parent += dl

2869 offset = self.state_machine.line_offset + 1 # next line

2870 newline_offset, blank_finish = self.nested_list_parse(

2871 self.state_machine.input_lines[offset:],

2872 input_offset=self.state_machine.abs_line_offset() + 1,

2873 node=dl, initial_state='DefinitionList',

2874 blank_finish=blank_finish, blank_finish_state='Definition')

2875 self.goto_line(newline_offset)

2876 if not blank_finish:

2877 self.parent += self.unindent_warning('Definition list')

2878 return [], 'Body', []

2879

2880 def underline(self, match, context, next_state):

2881 """Section title."""

2882 lineno = self.state_machine.abs_line_number()

2883 title = context[0].rstrip()

2884 underline = match.string.rstrip()

2885 source = title + '\n' + underline

2886 messages = []

2887 if column_width(title) > len(underline):

2888 if len(underline) < 4:

2889 if self.state_machine.match_titles:

2890 msg = self.reporter.info(

2891 'Possible title underline, too short for the title.\n'

2892 "Treating it as ordinary text because it's so short.",

2893 line=lineno)

2894 self.parent += msg

2895 raise statemachine.TransitionCorrection('text')

2896 else:

2897 blocktext = context[0] + '\n' + self.state_machine.line

2898 msg = self.reporter.warning(

2899 'Title underline too short.',

2900 nodes.literal_block(blocktext, blocktext),

2901 line=lineno)

2902 messages.append(msg)

2903 if not self.state_machine.match_titles:

2904 blocktext = context[0] + '\n' + self.state_machine.line

2905 # We need get_source_and_line() here to report correctly

2906 src, srcline = self.state_machine.get_source_and_line()

2907 # TODO: why is abs_line_number() == srcline+1

2908 # if the error is in a table (try with test_tables.py)?

2909 # print("get_source_and_line", srcline)

2910 # print("abs_line_number", self.state_machine.abs_line_number())

2911 msg = self.reporter.error(

2912 'Unexpected section title.',

2913 nodes.literal_block(blocktext, blocktext),

2914 source=src, line=srcline)

2915 self.parent += messages

2916 self.parent += msg

2917 return [], next_state, []

2918 style = underline[0]

2919 context[:] = []

2920 self.section(title, source, style, lineno - 1, messages)

2921 return [], next_state, []

2922

2923 def text(self, match, context, next_state):

2924 """Paragraph."""

2925 startline = self.state_machine.abs_line_number() - 1

2926 msg = None

2927 try:

2928 block = self.state_machine.get_text_block(flush_left=True)

2929 except statemachine.UnexpectedIndentationError as err:

2930 block, src, srcline = err.args

2931 msg = self.reporter.error('Unexpected indentation.',

2932 source=src, line=srcline)

2933 lines = context + list(block)

2934 paragraph, literalnext = self.paragraph(lines, startline)

2935 self.parent += paragraph

2936 self.parent += msg

2937 if literalnext:

2938 try:

2939 self.state_machine.next_line()

2940 except EOFError:

2941 pass

2942 self.parent += self.literal_block()

2943 return [], next_state, []

2944

2945 def literal_block(self):

2946 """Return a list of nodes."""

2947 (indented, indent, offset, blank_finish

2948 ) = self.state_machine.get_indented()

2949 while indented and not indented[-1].strip():

2950 indented.trim_end()

2951 if not indented:

2952 return self.quoted_literal_block()

2953 data = '\n'.join(indented)

2954 literal_block = nodes.literal_block(data, data)

2955 (literal_block.source,

2956 literal_block.line) = self.state_machine.get_source_and_line(offset+1)

2957 nodelist = [literal_block]

2958 if not blank_finish:

2959 nodelist.append(self.unindent_warning('Literal block'))

2960 return nodelist

2961

2962 def quoted_literal_block(self):

2963 abs_line_offset = self.state_machine.abs_line_offset()

2964 offset = self.state_machine.line_offset

2965 parent_node = nodes.Element()

2966 new_abs_offset = self.nested_parse(

2967 self.state_machine.input_lines[offset:],

2968 input_offset=abs_line_offset, node=parent_node, match_titles=False,

2969 state_machine_kwargs={'state_classes': (QuotedLiteralBlock,),

2970 'initial_state': 'QuotedLiteralBlock'})

2971 self.goto_line(new_abs_offset)

2972 return parent_node.children

2973

2974 def definition_list_item(self, termline):

2975 # the parser is already on the second (indented) line:

2976 dd_lineno = self.state_machine.abs_line_number()

2977 dt_lineno = dd_lineno - 1

2978 (indented, indent, line_offset, blank_finish

2979 ) = self.state_machine.get_indented()

2980 dl_item = nodes.definition_list_item(

2981 '\n'.join(termline + list(indented)))

2982 (dl_item.source,

2983 dl_item.line) = self.state_machine.get_source_and_line(dt_lineno)

2984 dt_nodes, messages = self.term(termline, dt_lineno)

2985 dl_item += dt_nodes

2986 dd = nodes.definition('', *messages)

2987 dd.source, dd.line = self.state_machine.get_source_and_line(dd_lineno)

2988 dl_item += dd

2989 if termline[0][-2:] == '::':

2990 dd += self.reporter.info(

2991 'Blank line missing before literal block (after the "::")? '

2992 'Interpreted as a definition list item.',

2993 line=dd_lineno)

2994 # TODO: drop a definition if it is an empty comment to allow

2995 # definition list items with several terms?

2996 # https://sourceforge.net/p/docutils/feature-requests/60/

2997 self.nested_parse(indented, input_offset=line_offset, node=dd)

2998 return dl_item, blank_finish

2999

3000 classifier_delimiter = re.compile(' +: +')

3001

3002 def term(self, lines, lineno):

3003 """Return a definition_list's term and optional classifiers."""

3004 assert len(lines) == 1

3005 text_nodes, messages = self.inline_text(lines[0], lineno)

3006 dt = nodes.term(lines[0])

3007 dt.source, dt.line = self.state_machine.get_source_and_line(lineno)

3008 node_list = [dt]

3009 for i in range(len(text_nodes)):

3010 node = text_nodes[i]

3011 if isinstance(node, nodes.Text):

3012 parts = self.classifier_delimiter.split(node)

3013 if len(parts) == 1:

3014 node_list[-1] += node

3015 else:

3016 text = parts[0].rstrip()

3017 textnode = nodes.Text(text)

3018 node_list[-1] += textnode

3019 node_list += [nodes.classifier(unescape(part, True), part)

3020 for part in parts[1:]]

3021 else:

3022 node_list[-1] += node

3023 return node_list, messages

3024

3025

3026class SpecializedText(Text):

3027

3028 """

3029 Superclass for second and subsequent lines of Text-variants.

3030

3031 All transition methods are disabled. Override individual methods in

3032 subclasses to re-enable.

3033 """

3034

3035 def eof(self, context):

3036 """Incomplete construct."""

3037 return []

3038

3039 def invalid_input(self, match=None, context=None, next_state=None):

3040 """Not a compound element member. Abort this state machine."""

3041 raise EOFError

3042

3043 blank = invalid_input

3044 indent = invalid_input

3045 underline = invalid_input

3046 text = invalid_input

3047

3048

3049class Definition(SpecializedText):

3050

3051 """Second line of potential definition_list_item."""

3052

3053 def eof(self, context):

3054 """Not a definition."""

3055 self.state_machine.previous_line(2) # so parent SM can reassess

3056 return []

3057

3058 def indent(self, match, context, next_state):

3059 """Definition list item."""

3060 dl_item, blank_finish = self.definition_list_item(context)

3061 self.parent += dl_item

3062 self.blank_finish = blank_finish

3063 return [], 'DefinitionList', []

3064

3065

3066class Line(SpecializedText):

3067

3068 """

3069 Second line of over- & underlined section title or transition marker.

3070 """

3071

3072 eofcheck = 1 # ignored, will be removed in Docutils 2.0.

3073

3074 def eof(self, context):

3075 """Transition marker at end of section or document."""

3076 marker = context[0].strip()

3077 if len(marker) < 4:

3078 self.state_correction(context)

3079 src, srcline = self.state_machine.get_source_and_line()

3080 # lineno = self.state_machine.abs_line_number() - 1

3081 transition = nodes.transition(rawsource=context[0])

3082 transition.source = src

3083 transition.line = srcline - 1

3084 # transition.line = lineno

3085 self.parent += transition

3086 return []

3087

3088 def blank(self, match, context, next_state):

3089 """Transition marker."""

3090 src, srcline = self.state_machine.get_source_and_line()

3091 marker = context[0].strip()

3092 if len(marker) < 4:

3093 self.state_correction(context)

3094 transition = nodes.transition(rawsource=marker)

3095 transition.source = src

3096 transition.line = srcline - 1

3097 self.parent += transition

3098 return [], 'Body', []

3099

3100 def text(self, match, context, next_state):

3101 """Potential over- & underlined title."""

3102 lineno = self.state_machine.abs_line_number() - 1

3103 overline = context[0]

3104 title = match.string

3105 underline = ''

3106 try:

3107 underline = self.state_machine.next_line()

3108 except EOFError:

3109 blocktext = overline + '\n' + title

3110 if len(overline.rstrip()) < 4:

3111 self.short_overline(context, blocktext, lineno, 2)

3112 else:

3113 msg = self.reporter.error(

3114 'Incomplete section title.',

3115 nodes.literal_block(blocktext, blocktext),

3116 line=lineno)

3117 self.parent += msg

3118 return [], 'Body', []

3119 source = '%s\n%s\n%s' % (overline, title, underline)

3120 overline = overline.rstrip()

3121 underline = underline.rstrip()

3122 if not self.transitions['underline'][0].match(underline):

3123 blocktext = overline + '\n' + title + '\n' + underline

3124 if len(overline.rstrip()) < 4:

3125 self.short_overline(context, blocktext, lineno, 2)

3126 else:

3127 msg = self.reporter.error(

3128 'Missing matching underline for section title overline.',

3129 nodes.literal_block(source, source),

3130 line=lineno)

3131 self.parent += msg

3132 return [], 'Body', []

3133 elif overline != underline:

3134 blocktext = overline + '\n' + title + '\n' + underline

3135 if len(overline.rstrip()) < 4:

3136 self.short_overline(context, blocktext, lineno, 2)

3137 else:

3138 msg = self.reporter.error(

3139 'Title overline & underline mismatch.',

3140 nodes.literal_block(source, source),

3141 line=lineno)

3142 self.parent += msg

3143 return [], 'Body', []

3144 title = title.rstrip()

3145 messages = []

3146 if column_width(title) > len(overline):

3147 blocktext = overline + '\n' + title + '\n' + underline

3148 if len(overline.rstrip()) < 4:

3149 self.short_overline(context, blocktext, lineno, 2)

3150 else:

3151 msg = self.reporter.warning(

3152 'Title overline too short.',

3153 nodes.literal_block(source, source),

3154 line=lineno)

3155 messages.append(msg)

3156 style = (overline[0], underline[0])

3157 self.section(title.lstrip(), source, style, lineno + 1, messages)

3158 return [], 'Body', []

3159

3160 indent = text # indented title

3161

3162 def underline(self, match, context, next_state):

3163 overline = context[0]

3164 blocktext = overline + '\n' + self.state_machine.line

3165 lineno = self.state_machine.abs_line_number() - 1

3166 if len(overline.rstrip()) < 4:

3167 self.short_overline(context, blocktext, lineno, 1)

3168 msg = self.reporter.error(

3169 'Invalid section title or transition marker.',

3170 nodes.literal_block(blocktext, blocktext),

3171 line=lineno)

3172 self.parent += msg

3173 return [], 'Body', []

3174

3175 def short_overline(self, context, blocktext, lineno, lines=1) -> None:

3176 msg = self.reporter.info(

3177 'Possible incomplete section title.\nTreating the overline as '

3178 "ordinary text because it's so short.",

3179 line=lineno)

3180 self.parent += msg

3181 self.state_correction(context, lines)

3182

3183 def state_correction(self, context, lines=1):

3184 self.state_machine.previous_line(lines)

3185 context[:] = []

3186 raise statemachine.StateCorrection('Body', 'text')

3187

3188

3189class QuotedLiteralBlock(RSTState):

3190

3191 """

3192 Nested parse handler for quoted (unindented) literal blocks.

3193

3194 Special-purpose. Not for inclusion in `state_classes`.

3195 """

3196

3197 patterns = {'initial_quoted': r'(%(nonalphanum7bit)s)' % Body.pats,

3198 'text': r''}

3199 initial_transitions = ('initial_quoted', 'text')

3200

3201 def __init__(self, state_machine, debug=False) -> None:

3202 RSTState.__init__(self, state_machine, debug)

3203 self.messages = []

3204 self.initial_lineno = None

3205

3206 def blank(self, match, context, next_state):

3207 if context:

3208 raise EOFError

3209 else:

3210 return context, next_state, []

3211

3212 def eof(self, context):

3213 if context:

3214 src, srcline = self.state_machine.get_source_and_line(

3215 self.initial_lineno)

3216 text = '\n'.join(context)

3217 literal_block = nodes.literal_block(text, text)

3218 literal_block.source = src

3219 literal_block.line = srcline

3220 self.parent += literal_block

3221 else:

3222 self.parent += self.reporter.warning(

3223 'Literal block expected; none found.',

3224 line=self.state_machine.abs_line_number()

3225 ) # src not available, statemachine.input_lines is empty

3226 self.state_machine.previous_line()

3227 self.parent += self.messages

3228 return []

3229

3230 def indent(self, match, context, next_state):

3231 assert context, ('QuotedLiteralBlock.indent: context should not '

3232 'be empty!')

3233 self.messages.append(

3234 self.reporter.error('Unexpected indentation.',

3235 line=self.state_machine.abs_line_number()))

3236 self.state_machine.previous_line()

3237 raise EOFError

3238

3239 def initial_quoted(self, match, context, next_state):

3240 """Match arbitrary quote character on the first line only."""

3241 self.remove_transition('initial_quoted')

3242 quote = match.string[0]

3243 pattern = re.compile(re.escape(quote))

3244 # New transition matches consistent quotes only:

3245 self.add_transition('quoted',

3246 (pattern, self.quoted, self.__class__.__name__))

3247 self.initial_lineno = self.state_machine.abs_line_number()

3248 return [match.string], next_state, []

3249

3250 def quoted(self, match, context, next_state):

3251 """Match consistent quotes on subsequent lines."""

3252 context.append(match.string)

3253 return context, next_state, []

3254

3255 def text(self, match, context, next_state):

3256 if context:

3257 self.messages.append(

3258 self.reporter.error('Inconsistent literal block quoting.',

3259 line=self.state_machine.abs_line_number()))

3260 self.state_machine.previous_line()

3261 raise EOFError

3262

3263

3264state_classes = (Body, BulletList, DefinitionList, EnumeratedList, FieldList,

3265 OptionList, LineBlock, ExtensionOptions, Explicit, Text,

3266 Definition, Line, SubstitutionDef, RFC2822Body, RFC2822List)

3267"""Standard set of State classes used to start `RSTStateMachine`."""