Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1# Vendored from https://github.com/numpy/numpydoc/,
2# changeset 4ae1e00e72e522c126403c1814f0b99dc5978622
4# This file is licensed under the BSD License. See the LICENSE.txt file
5# in the root of the `numpydoc` repository for complete details.
7"""Extract reference documentation from the NumPy source tree.
9"""
10import inspect
11import textwrap
12import re
13import pydoc
14from warnings import warn
15from collections import namedtuple
16from collections.abc import Callable, Mapping
17import copy
18import sys
21def strip_blank_lines(l):
22 "Remove leading and trailing blank lines from a list of lines"
23 while l and not l[0].strip():
24 del l[0]
25 while l and not l[-1].strip():
26 del l[-1]
27 return l
30class Reader:
31 """A line-based string reader.
33 """
35 def __init__(self, data):
36 """
37 Parameters
38 ----------
39 data : str
40 String with lines separated by '\\n'.
42 """
43 if isinstance(data, list):
44 self._str = data
45 else:
46 self._str = data.split('\n') # store string as list of lines
48 self.reset()
50 def __getitem__(self, n):
51 return self._str[n]
53 def reset(self):
54 self._l = 0 # current line nr
56 def read(self):
57 if not self.eof():
58 out = self[self._l]
59 self._l += 1
60 return out
61 else:
62 return ''
64 def seek_next_non_empty_line(self):
65 for l in self[self._l:]:
66 if l.strip():
67 break
68 else:
69 self._l += 1
71 def eof(self):
72 return self._l >= len(self._str)
74 def read_to_condition(self, condition_func):
75 start = self._l
76 for line in self[start:]:
77 if condition_func(line):
78 return self[start:self._l]
79 self._l += 1
80 if self.eof():
81 return self[start:self._l+1]
82 return []
84 def read_to_next_empty_line(self):
85 self.seek_next_non_empty_line()
87 def is_empty(line):
88 return not line.strip()
90 return self.read_to_condition(is_empty)
92 def read_to_next_unindented_line(self):
93 def is_unindented(line):
94 return (line.strip() and (len(line.lstrip()) == len(line)))
95 return self.read_to_condition(is_unindented)
97 def peek(self, n=0):
98 if self._l + n < len(self._str):
99 return self[self._l + n]
100 else:
101 return ''
103 def is_empty(self):
104 return not ''.join(self._str).strip()
107class ParseError(Exception):
108 def __str__(self):
109 message = self.args[0]
110 if hasattr(self, 'docstring'):
111 message = "%s in %r" % (message, self.docstring)
112 return message
115Parameter = namedtuple('Parameter', ['name', 'type', 'desc'])
118class NumpyDocString(Mapping):
119 """Parses a numpydoc string to an abstract representation
121 Instances define a mapping from section title to structured data.
123 """
125 sections = {
126 'Signature': '',
127 'Summary': [''],
128 'Extended Summary': [],
129 'Parameters': [],
130 'Returns': [],
131 'Yields': [],
132 'Receives': [],
133 'Raises': [],
134 'Warns': [],
135 'Other Parameters': [],
136 'Attributes': [],
137 'Methods': [],
138 'See Also': [],
139 'Notes': [],
140 'Warnings': [],
141 'References': '',
142 'Examples': '',
143 'index': {}
144 }
146 def __init__(self, docstring, config=None):
147 orig_docstring = docstring
148 docstring = textwrap.dedent(docstring).split('\n')
150 self._doc = Reader(docstring)
151 self._parsed_data = copy.deepcopy(self.sections)
153 try:
154 self._parse()
155 except ParseError as e:
156 e.docstring = orig_docstring
157 raise
159 def __getitem__(self, key):
160 return self._parsed_data[key]
162 def __setitem__(self, key, val):
163 if key not in self._parsed_data:
164 self._error_location("Unknown section %s" % key, error=False)
165 else:
166 self._parsed_data[key] = val
168 def __iter__(self):
169 return iter(self._parsed_data)
171 def __len__(self):
172 return len(self._parsed_data)
174 def _is_at_section(self):
175 self._doc.seek_next_non_empty_line()
177 if self._doc.eof():
178 return False
180 l1 = self._doc.peek().strip() # e.g. Parameters
182 if l1.startswith('.. index::'):
183 return True
185 l2 = self._doc.peek(1).strip() # ---------- or ==========
186 if len(l2) >= 3 and (set(l2) in ({'-'}, {'='})) and len(l2) != len(l1):
187 snip = '\n'.join(self._doc._str[:2])+'...'
188 self._error_location("potentially wrong underline length... \n%s \n%s in \n%s"
189 % (l1, l2, snip), error=False)
190 return l2.startswith('-'*len(l1)) or l2.startswith('='*len(l1))
192 def _strip(self, doc):
193 i = 0
194 j = 0
195 for i, line in enumerate(doc):
196 if line.strip():
197 break
199 for j, line in enumerate(doc[::-1]):
200 if line.strip():
201 break
203 return doc[i:len(doc)-j]
205 def _read_to_next_section(self):
206 section = self._doc.read_to_next_empty_line()
208 while not self._is_at_section() and not self._doc.eof():
209 if not self._doc.peek(-1).strip(): # previous line was empty
210 section += ['']
212 section += self._doc.read_to_next_empty_line()
214 return section
216 def _read_sections(self):
217 while not self._doc.eof():
218 data = self._read_to_next_section()
219 name = data[0].strip()
221 if name.startswith('..'): # index section
222 yield name, data[1:]
223 elif len(data) < 2:
224 yield StopIteration
225 else:
226 yield name, self._strip(data[2:])
228 def _parse_param_list(self, content, single_element_is_type=False):
229 content = dedent_lines(content)
230 r = Reader(content)
231 params = []
232 while not r.eof():
233 header = r.read().strip()
234 if ' :' in header:
235 arg_name, arg_type = header.split(' :', maxsplit=1)
236 arg_name, arg_type = arg_name.strip(), arg_type.strip()
237 else:
238 if single_element_is_type:
239 arg_name, arg_type = '', header
240 else:
241 arg_name, arg_type = header, ''
243 desc = r.read_to_next_unindented_line()
244 desc = dedent_lines(desc)
245 desc = strip_blank_lines(desc)
247 params.append(Parameter(arg_name, arg_type, desc))
249 return params
251 # See also supports the following formats.
252 #
253 # <FUNCNAME>
254 # <FUNCNAME> SPACE* COLON SPACE+ <DESC> SPACE*
255 # <FUNCNAME> ( COMMA SPACE+ <FUNCNAME>)+ (COMMA | PERIOD)? SPACE*
256 # <FUNCNAME> ( COMMA SPACE+ <FUNCNAME>)* SPACE* COLON SPACE+ <DESC> SPACE*
258 # <FUNCNAME> is one of
259 # <PLAIN_FUNCNAME>
260 # COLON <ROLE> COLON BACKTICK <PLAIN_FUNCNAME> BACKTICK
261 # where
262 # <PLAIN_FUNCNAME> is a legal function name, and
263 # <ROLE> is any nonempty sequence of word characters.
264 # Examples: func_f1 :meth:`func_h1` :obj:`~baz.obj_r` :class:`class_j`
265 # <DESC> is a string describing the function.
267 _role = r":(?P<role>(py:)?\w+):"
268 _funcbacktick = r"`(?P<name>(?:~\w+\.)?[a-zA-Z0-9_\.-]+)`"
269 _funcplain = r"(?P<name2>[a-zA-Z0-9_\.-]+)"
270 _funcname = r"(" + _role + _funcbacktick + r"|" + _funcplain + r")"
271 _funcnamenext = _funcname.replace('role', 'rolenext')
272 _funcnamenext = _funcnamenext.replace('name', 'namenext')
273 _description = r"(?P<description>\s*:(\s+(?P<desc>\S+.*))?)?\s*$"
274 _func_rgx = re.compile(r"^\s*" + _funcname + r"\s*")
275 _line_rgx = re.compile(
276 r"^\s*" +
277 r"(?P<allfuncs>" + # group for all function names
278 _funcname +
279 r"(?P<morefuncs>([,]\s+" + _funcnamenext + r")*)" +
280 r")" + # end of "allfuncs"
281 # Some function lists have a trailing comma (or period) '\s*'
282 r"(?P<trailing>[,\.])?" +
283 _description)
285 # Empty <DESC> elements are replaced with '..'
286 empty_description = '..'
288 def _parse_see_also(self, content):
289 """
290 func_name : Descriptive text
291 continued text
292 another_func_name : Descriptive text
293 func_name1, func_name2, :meth:`func_name`, func_name3
295 """
297 content = dedent_lines(content)
299 items = []
301 def parse_item_name(text):
302 """Match ':role:`name`' or 'name'."""
303 m = self._func_rgx.match(text)
304 if not m:
305 self._error_location(f"Error parsing See Also entry {line!r}")
306 role = m.group('role')
307 name = m.group('name') if role else m.group('name2')
308 return name, role, m.end()
310 rest = []
311 for line in content:
312 if not line.strip():
313 continue
315 line_match = self._line_rgx.match(line)
316 description = None
317 if line_match:
318 description = line_match.group('desc')
319 if line_match.group('trailing') and description:
320 self._error_location(
321 'Unexpected comma or period after function list at index %d of '
322 'line "%s"' % (line_match.end('trailing'), line),
323 error=False)
324 if not description and line.startswith(' '):
325 rest.append(line.strip())
326 elif line_match:
327 funcs = []
328 text = line_match.group('allfuncs')
329 while True:
330 if not text.strip():
331 break
332 name, role, match_end = parse_item_name(text)
333 funcs.append((name, role))
334 text = text[match_end:].strip()
335 if text and text[0] == ',':
336 text = text[1:].strip()
337 rest = list(filter(None, [description]))
338 items.append((funcs, rest))
339 else:
340 self._error_location(f"Error parsing See Also entry {line!r}")
341 return items
343 def _parse_index(self, section, content):
344 """
345 .. index: default
346 :refguide: something, else, and more
348 """
349 def strip_each_in(lst):
350 return [s.strip() for s in lst]
352 out = {}
353 section = section.split('::')
354 if len(section) > 1:
355 out['default'] = strip_each_in(section[1].split(','))[0]
356 for line in content:
357 line = line.split(':')
358 if len(line) > 2:
359 out[line[1]] = strip_each_in(line[2].split(','))
360 return out
362 def _parse_summary(self):
363 """Grab signature (if given) and summary"""
364 if self._is_at_section():
365 return
367 # If several signatures present, take the last one
368 while True:
369 summary = self._doc.read_to_next_empty_line()
370 summary_str = " ".join([s.strip() for s in summary]).strip()
371 compiled = re.compile(r'^([\w., ]+=)?\s*[\w\.]+\(.*\)$')
372 if compiled.match(summary_str):
373 self['Signature'] = summary_str
374 if not self._is_at_section():
375 continue
376 break
378 if summary is not None:
379 self['Summary'] = summary
381 if not self._is_at_section():
382 self['Extended Summary'] = self._read_to_next_section()
384 def _parse(self):
385 self._doc.reset()
386 self._parse_summary()
388 sections = list(self._read_sections())
389 section_names = set([section for section, content in sections])
391 has_returns = 'Returns' in section_names
392 has_yields = 'Yields' in section_names
393 # We could do more tests, but we are not. Arbitrarily.
394 if has_returns and has_yields:
395 msg = 'Docstring contains both a Returns and Yields section.'
396 raise ValueError(msg)
397 if not has_yields and 'Receives' in section_names:
398 msg = 'Docstring contains a Receives section but not Yields.'
399 raise ValueError(msg)
401 for (section, content) in sections:
402 if not section.startswith('..'):
403 section = (s.capitalize() for s in section.split(' '))
404 section = ' '.join(section)
405 if self.get(section):
406 self._error_location("The section %s appears twice in %s"
407 % (section, '\n'.join(self._doc._str)))
409 if section in ('Parameters', 'Other Parameters', 'Attributes',
410 'Methods'):
411 self[section] = self._parse_param_list(content)
412 elif section in ('Returns', 'Yields', 'Raises', 'Warns', 'Receives'):
413 self[section] = self._parse_param_list(
414 content, single_element_is_type=True)
415 elif section.startswith('.. index::'):
416 self['index'] = self._parse_index(section, content)
417 elif section == 'See Also':
418 self['See Also'] = self._parse_see_also(content)
419 else:
420 self[section] = content
422 @property
423 def _obj(self):
424 if hasattr(self, '_cls'):
425 return self._cls
426 elif hasattr(self, '_f'):
427 return self._f
428 return None
430 def _error_location(self, msg, error=True):
431 if self._obj is not None:
432 # we know where the docs came from:
433 try:
434 filename = inspect.getsourcefile(self._obj)
435 except TypeError:
436 filename = None
437 msg += f" in the docstring of {self._obj.__name__}"
438 msg += f" in {filename}." if filename else ""
439 if error:
440 raise ValueError(msg)
441 else:
442 warn(msg)
444 # string conversion routines
446 def _str_header(self, name, symbol='-'):
447 return [name, len(name)*symbol]
449 def _str_indent(self, doc, indent=4):
450 return [' '*indent + line for line in doc]
452 def _str_signature(self):
453 if self['Signature']:
454 return [self['Signature'].replace('*', r'\*')] + ['']
455 return ['']
457 def _str_summary(self):
458 if self['Summary']:
459 return self['Summary'] + ['']
460 return []
462 def _str_extended_summary(self):
463 if self['Extended Summary']:
464 return self['Extended Summary'] + ['']
465 return []
467 def _str_param_list(self, name):
468 out = []
469 if self[name]:
470 out += self._str_header(name)
471 for param in self[name]:
472 parts = []
473 if param.name:
474 parts.append(param.name)
475 if param.type:
476 parts.append(param.type)
477 out += [' : '.join(parts)]
478 if param.desc and ''.join(param.desc).strip():
479 out += self._str_indent(param.desc)
480 out += ['']
481 return out
483 def _str_section(self, name):
484 out = []
485 if self[name]:
486 out += self._str_header(name)
487 out += self[name]
488 out += ['']
489 return out
491 def _str_see_also(self, func_role):
492 if not self['See Also']:
493 return []
494 out = []
495 out += self._str_header("See Also")
496 out += ['']
497 last_had_desc = True
498 for funcs, desc in self['See Also']:
499 assert isinstance(funcs, list)
500 links = []
501 for func, role in funcs:
502 if role:
503 link = ':%s:`%s`' % (role, func)
504 elif func_role:
505 link = ':%s:`%s`' % (func_role, func)
506 else:
507 link = "`%s`_" % func
508 links.append(link)
509 link = ', '.join(links)
510 out += [link]
511 if desc:
512 out += self._str_indent([' '.join(desc)])
513 last_had_desc = True
514 else:
515 last_had_desc = False
516 out += self._str_indent([self.empty_description])
518 if last_had_desc:
519 out += ['']
520 out += ['']
521 return out
523 def _str_index(self):
524 idx = self['index']
525 out = []
526 output_index = False
527 default_index = idx.get('default', '')
528 if default_index:
529 output_index = True
530 out += ['.. index:: %s' % default_index]
531 for section, references in idx.items():
532 if section == 'default':
533 continue
534 output_index = True
535 out += [' :%s: %s' % (section, ', '.join(references))]
536 if output_index:
537 return out
538 return ''
540 def __str__(self, func_role=''):
541 out = []
542 out += self._str_signature()
543 out += self._str_summary()
544 out += self._str_extended_summary()
545 for param_list in ('Parameters', 'Returns', 'Yields', 'Receives',
546 'Other Parameters', 'Raises', 'Warns'):
547 out += self._str_param_list(param_list)
548 out += self._str_section('Warnings')
549 out += self._str_see_also(func_role)
550 for s in ('Notes', 'References', 'Examples'):
551 out += self._str_section(s)
552 for param_list in ('Attributes', 'Methods'):
553 out += self._str_param_list(param_list)
554 out += self._str_index()
555 return '\n'.join(out)
558def dedent_lines(lines):
559 """Deindent a list of lines maximally"""
560 return textwrap.dedent("\n".join(lines)).split("\n")
563class FunctionDoc(NumpyDocString):
564 def __init__(self, func, role='func', doc=None, config=None):
565 self._f = func
566 self._role = role # e.g. "func" or "meth"
568 if doc is None:
569 if func is None:
570 raise ValueError("No function or docstring given")
571 doc = inspect.getdoc(func) or ''
572 if config is None:
573 config = {}
574 NumpyDocString.__init__(self, doc, config)
576 def get_func(self):
577 func_name = getattr(self._f, '__name__', self.__class__.__name__)
578 if inspect.isclass(self._f):
579 func = getattr(self._f, '__call__', self._f.__init__)
580 else:
581 func = self._f
582 return func, func_name
584 def __str__(self):
585 out = ''
587 func, func_name = self.get_func()
589 roles = {'func': 'function',
590 'meth': 'method'}
592 if self._role:
593 if self._role not in roles:
594 print("Warning: invalid role %s" % self._role)
595 out += '.. %s:: %s\n \n\n' % (roles.get(self._role, ''),
596 func_name)
598 out += super().__str__(func_role=self._role)
599 return out
602class ObjDoc(NumpyDocString):
603 def __init__(self, obj, doc=None, config=None):
604 self._f = obj
605 if config is None:
606 config = {}
607 NumpyDocString.__init__(self, doc, config=config)
610class ClassDoc(NumpyDocString):
612 extra_public_methods = ['__call__']
614 def __init__(self, cls, doc=None, modulename='', func_doc=FunctionDoc,
615 config=None):
616 if not inspect.isclass(cls) and cls is not None:
617 raise ValueError("Expected a class or None, but got %r" % cls)
618 self._cls = cls
620 if 'sphinx' in sys.modules:
621 from sphinx.ext.autodoc import ALL
622 else:
623 ALL = object()
625 if config is None:
626 config = {}
627 self.show_inherited_members = config.get(
628 'show_inherited_class_members', True)
630 if modulename and not modulename.endswith('.'):
631 modulename += '.'
632 self._mod = modulename
634 if doc is None:
635 if cls is None:
636 raise ValueError("No class or documentation string given")
637 doc = pydoc.getdoc(cls)
639 NumpyDocString.__init__(self, doc)
641 _members = config.get('members', [])
642 if _members is ALL:
643 _members = None
644 _exclude = config.get('exclude-members', [])
646 if config.get('show_class_members', True) and _exclude is not ALL:
647 def splitlines_x(s):
648 if not s:
649 return []
650 else:
651 return s.splitlines()
652 for field, items in [('Methods', self.methods),
653 ('Attributes', self.properties)]:
654 if not self[field]:
655 doc_list = []
656 for name in sorted(items):
657 if (name in _exclude or
658 (_members and name not in _members)):
659 continue
660 try:
661 doc_item = pydoc.getdoc(getattr(self._cls, name))
662 doc_list.append(
663 Parameter(name, '', splitlines_x(doc_item)))
664 except AttributeError:
665 pass # method doesn't exist
666 self[field] = doc_list
668 @property
669 def methods(self):
670 if self._cls is None:
671 return []
672 return [name for name, func in inspect.getmembers(self._cls)
673 if ((not name.startswith('_') or
674 name in self.extra_public_methods) and
675 isinstance(func, Callable) and
676 self._is_show_member(name))]
678 @property
679 def properties(self):
680 if self._cls is None:
681 return []
682 return [name for name, func in inspect.getmembers(self._cls)
683 if (not name.startswith('_') and
684 (func is None or isinstance(func, property) or
685 inspect.isdatadescriptor(func)) and
686 self._is_show_member(name))]
688 def _is_show_member(self, name):
689 if self.show_inherited_members:
690 return True # show all class members
691 if name not in self._cls.__dict__:
692 return False # class member is inherited, we do not show it
693 return True
696def get_doc_object(obj, what=None, doc=None, config=None):
697 if what is None:
698 if inspect.isclass(obj):
699 what = 'class'
700 elif inspect.ismodule(obj):
701 what = 'module'
702 elif isinstance(obj, Callable):
703 what = 'function'
704 else:
705 what = 'object'
706 if config is None:
707 config = {}
709 if what == 'class':
710 return ClassDoc(obj, func_doc=FunctionDoc, doc=doc, config=config)
711 elif what in ('function', 'method'):
712 return FunctionDoc(obj, doc=doc, config=config)
713 else:
714 if doc is None:
715 doc = pydoc.getdoc(obj)
716 return ObjDoc(obj, doc, config=config)