Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/nbconvert/exporters/templateexporter.py: 29%
316 statements
« prev ^ index » next coverage.py v7.2.7, created at 2023-07-01 06:54 +0000
1"""This module defines TemplateExporter, a highly configurable converter
2that uses Jinja2 to export notebook files into different formats.
3"""
5# Copyright (c) IPython Development Team.
6# Distributed under the terms of the Modified BSD License.
9import html
10import json
11import os
12import typing as t
13import uuid
14import warnings
15from pathlib import Path
17from jinja2 import (
18 BaseLoader,
19 ChoiceLoader,
20 DictLoader,
21 Environment,
22 FileSystemLoader,
23 TemplateNotFound,
24)
25from jupyter_core.paths import jupyter_path
26from nbformat import NotebookNode
27from traitlets import Bool, Dict, HasTraits, List, Unicode, default, observe, validate
28from traitlets.config import Config
29from traitlets.utils.importstring import import_item
31from nbconvert import filters
33from .exporter import Exporter
35# Jinja2 extensions to load.
36JINJA_EXTENSIONS = ["jinja2.ext.loopcontrols"]
38ROOT = os.path.dirname(__file__)
39DEV_MODE = os.path.exists(os.path.join(ROOT, "../../.git"))
42default_filters = {
43 "indent": filters.indent,
44 "markdown2html": filters.markdown2html,
45 "markdown2asciidoc": filters.markdown2asciidoc,
46 "ansi2html": filters.ansi2html,
47 "filter_data_type": filters.DataTypeFilter,
48 "get_lines": filters.get_lines,
49 "highlight2html": filters.Highlight2HTML,
50 "highlight2latex": filters.Highlight2Latex,
51 "ipython2python": filters.ipython2python,
52 "posix_path": filters.posix_path,
53 "markdown2latex": filters.markdown2latex,
54 "markdown2rst": filters.markdown2rst,
55 "comment_lines": filters.comment_lines,
56 "strip_ansi": filters.strip_ansi,
57 "strip_dollars": filters.strip_dollars,
58 "strip_files_prefix": filters.strip_files_prefix,
59 "html2text": filters.html2text,
60 "add_anchor": filters.add_anchor,
61 "ansi2latex": filters.ansi2latex,
62 "wrap_text": filters.wrap_text,
63 "escape_latex": filters.escape_latex,
64 "citation2latex": filters.citation2latex,
65 "path2url": filters.path2url,
66 "add_prompts": filters.add_prompts,
67 "ascii_only": filters.ascii_only,
68 "prevent_list_blocks": filters.prevent_list_blocks,
69 "get_metadata": filters.get_metadata,
70 "convert_pandoc": filters.convert_pandoc,
71 "json_dumps": json.dumps,
72 # For removing any HTML
73 "escape_html": lambda s: html.escape(str(s)),
74 "escape_html_keep_quotes": lambda s: html.escape(str(s), quote=False),
75 # For sanitizing HTML for any XSS
76 "clean_html": filters.clean_html,
77 "strip_trailing_newline": filters.strip_trailing_newline,
78 "text_base64": filters.text_base64,
79}
82# copy of https://github.com/jupyter/jupyter_server/blob/b62458a7f5ad6b5246d2f142258dedaa409de5d9/jupyter_server/config_manager.py#L19
83def recursive_update(target, new):
84 """Recursively update one dictionary using another.
85 None values will delete their keys.
86 """
87 for k, v in new.items():
88 if isinstance(v, dict):
89 if k not in target:
90 target[k] = {}
91 recursive_update(target[k], v)
92 if not target[k]:
93 # Prune empty subdicts
94 del target[k]
96 elif v is None:
97 target.pop(k, None)
99 else:
100 target[k] = v
101 return target # return for convenience
104# define function at the top level to avoid pickle errors
105def deprecated(msg):
106 """Emit a deprecation warning."""
107 warnings.warn(msg, DeprecationWarning, stacklevel=2)
110class ExtensionTolerantLoader(BaseLoader):
111 """A template loader which optionally adds a given extension when searching.
113 Constructor takes two arguments: *loader* is another Jinja loader instance
114 to wrap. *extension* is the extension, which will be added to the template
115 name if finding the template without it fails. This should include the dot,
116 e.g. '.tpl'.
117 """
119 def __init__(self, loader, extension):
120 """Initialize the loader."""
121 self.loader = loader
122 self.extension = extension
124 def get_source(self, environment, template):
125 """Get the source for a template."""
126 try:
127 return self.loader.get_source(environment, template)
128 except TemplateNotFound:
129 if template.endswith(self.extension):
130 raise TemplateNotFound(template) from None
131 return self.loader.get_source(environment, template + self.extension)
133 def list_templates(self):
134 """List available templates."""
135 return self.loader.list_templates()
138class TemplateExporter(Exporter):
139 """
140 Exports notebooks into other file formats. Uses Jinja 2 templating engine
141 to output new formats. Inherit from this class if you are creating a new
142 template type along with new filters/preprocessors. If the filters/
143 preprocessors provided by default suffice, there is no need to inherit from
144 this class. Instead, override the template_file and file_extension
145 traits via a config file.
147 Filters available by default for templates:
149 {filters}
150 """
152 # finish the docstring
153 __doc__ = __doc__.format(filters="- " + "\n - ".join(sorted(default_filters.keys()))) # noqa
155 _template_cached = None
157 def _invalidate_template_cache(self, change=None):
158 self._template_cached = None
160 @property
161 def template(self):
162 if self._template_cached is None:
163 self._template_cached = self._load_template()
164 return self._template_cached
166 _environment_cached = None
168 def _invalidate_environment_cache(self, change=None):
169 self._environment_cached = None
170 self._invalidate_template_cache()
172 @property
173 def environment(self):
174 if self._environment_cached is None:
175 self._environment_cached = self._create_environment()
176 return self._environment_cached
178 @property
179 def default_config(self):
180 c = Config(
181 {
182 "RegexRemovePreprocessor": {"enabled": True},
183 "TagRemovePreprocessor": {"enabled": True},
184 }
185 )
186 if super().default_config:
187 c2 = super().default_config.copy()
188 c2.merge(c)
189 c = c2
190 return c
192 template_name = Unicode(help="Name of the template to use").tag(
193 config=True, affects_template=True
194 )
196 template_file = Unicode(None, allow_none=True, help="Name of the template file to use").tag(
197 config=True, affects_template=True
198 )
200 raw_template = Unicode("", help="raw template string").tag(affects_environment=True)
202 enable_async = Bool(False, help="Enable Jinja async template execution").tag(
203 affects_environment=True
204 )
206 _last_template_file = ""
207 _raw_template_key = "<memory>"
209 @validate("template_name")
210 def _template_name_validate(self, change):
211 template_name = change["value"]
212 if template_name and template_name.endswith(".tpl"):
213 warnings.warn(
214 f"5.x style template name passed '{self.template_name}'. Use --template-name for the template directory with a index.<ext>.j2 file and/or --template-file to denote a different template.",
215 DeprecationWarning,
216 stacklevel=2,
217 )
218 directory, self.template_file = os.path.split(self.template_name)
219 if directory:
220 directory, template_name = os.path.split(directory)
221 if directory and os.path.isabs(directory):
222 self.extra_template_basedirs = [directory]
223 return template_name
225 @observe("template_file")
226 def _template_file_changed(self, change):
227 new = change["new"]
228 if new == "default":
229 self.template_file = self.default_template # type:ignore
230 return
231 # check if template_file is a file path
232 # rather than a name already on template_path
233 full_path = os.path.abspath(new)
234 if os.path.isfile(full_path):
235 directory, self.template_file = os.path.split(full_path)
236 self.extra_template_paths = [directory, *self.extra_template_paths]
237 # While not strictly an invalid template file name, the extension hints that there isn't a template directory involved
238 if self.template_file.endswith(".tpl"):
239 warnings.warn(
240 f"5.x style template file passed '{new}'. Use --template-name for the template directory with a index.<ext>.j2 file and/or --template-file to denote a different template.",
241 DeprecationWarning,
242 stacklevel=2,
243 )
245 @default("template_file")
246 def _template_file_default(self):
247 if self.template_extension:
248 return "index" + self.template_extension
250 @observe("raw_template")
251 def _raw_template_changed(self, change):
252 if not change["new"]:
253 self.template_file = self._last_template_file
254 self._invalidate_template_cache()
256 template_paths = List(["."]).tag(config=True, affects_environment=True)
257 extra_template_basedirs = List().tag(config=True, affects_environment=True)
258 extra_template_paths = List([]).tag(config=True, affects_environment=True)
260 @default("extra_template_basedirs")
261 def _default_extra_template_basedirs(self):
262 return [os.getcwd()]
264 # Extension that the template files use.
265 template_extension = Unicode().tag(config=True, affects_environment=True)
267 template_data_paths = List(
268 jupyter_path("nbconvert", "templates"), help="Path where templates can be installed too."
269 ).tag(affects_environment=True)
271 # Extension that the template files use.
272 template_extension = Unicode().tag(config=True, affects_environment=True)
274 @default("template_extension")
275 def _template_extension_default(self):
276 if self.file_extension:
277 return self.file_extension + ".j2"
278 else:
279 return self.file_extension
281 exclude_input = Bool(
282 False, help="This allows you to exclude code cell inputs from all templates if set to True."
283 ).tag(config=True)
285 exclude_input_prompt = Bool(
286 False, help="This allows you to exclude input prompts from all templates if set to True."
287 ).tag(config=True)
289 exclude_output = Bool(
290 False,
291 help="This allows you to exclude code cell outputs from all templates if set to True.",
292 ).tag(config=True)
294 exclude_output_prompt = Bool(
295 False, help="This allows you to exclude output prompts from all templates if set to True."
296 ).tag(config=True)
298 exclude_output_stdin = Bool(
299 True,
300 help="This allows you to exclude output of stdin stream from lab template if set to True.",
301 ).tag(config=True)
303 exclude_code_cell = Bool(
304 False, help="This allows you to exclude code cells from all templates if set to True."
305 ).tag(config=True)
307 exclude_markdown = Bool(
308 False, help="This allows you to exclude markdown cells from all templates if set to True."
309 ).tag(config=True)
311 exclude_raw = Bool(
312 False, help="This allows you to exclude raw cells from all templates if set to True."
313 ).tag(config=True)
315 exclude_unknown = Bool(
316 False, help="This allows you to exclude unknown cells from all templates if set to True."
317 ).tag(config=True)
319 extra_loaders = List(
320 help="Jinja loaders to find templates. Will be tried in order "
321 "before the default FileSystem ones.",
322 ).tag(affects_environment=True)
324 filters = Dict(
325 help="""Dictionary of filters, by name and namespace, to add to the Jinja
326 environment."""
327 ).tag(config=True, affects_environment=True)
329 raw_mimetypes = List(
330 help="""formats of raw cells to be included in this Exporter's output."""
331 ).tag(config=True)
333 @default("raw_mimetypes")
334 def _raw_mimetypes_default(self):
335 return [self.output_mimetype, ""]
337 # TODO: passing config is wrong, but changing this revealed more complicated issues
338 def __init__(self, config=None, **kw):
339 """
340 Public constructor
342 Parameters
343 ----------
344 config : config
345 User configuration instance.
346 extra_loaders : list[of Jinja Loaders]
347 ordered list of Jinja loader to find templates. Will be tried in order
348 before the default FileSystem ones.
349 template_file : str (optional, kw arg)
350 Template to use when exporting.
351 """
352 super().__init__(config=config, **kw)
354 self.observe(
355 self._invalidate_environment_cache, list(self.traits(affects_environment=True))
356 )
357 self.observe(self._invalidate_template_cache, list(self.traits(affects_template=True)))
359 def _load_template(self):
360 """Load the Jinja template object from the template file
362 This is triggered by various trait changes that would change the template.
363 """
365 # this gives precedence to a raw_template if present
366 with self.hold_trait_notifications():
367 if self.template_file != self._raw_template_key:
368 self._last_template_file = self.template_file
369 if self.raw_template:
370 self.template_file = self._raw_template_key
372 if not self.template_file:
373 msg = "No template_file specified!"
374 raise ValueError(msg)
376 # First try to load the
377 # template by name with extension added, then try loading the template
378 # as if the name is explicitly specified.
379 template_file = self.template_file
380 self.log.debug("Attempting to load template %s", template_file)
381 self.log.debug(" template_paths: %s", os.pathsep.join(self.template_paths))
382 return self.environment.get_template(template_file)
384 def from_filename( # type:ignore
385 self, filename: str, resources: t.Optional[dict] = None, **kw: t.Any
386 ) -> t.Tuple[str, dict]:
387 """Convert a notebook from a filename."""
388 return super().from_filename(filename, resources, **kw) # type:ignore
390 def from_file( # type:ignore
391 self, file_stream: t.Any, resources: t.Optional[dict] = None, **kw: t.Any
392 ) -> t.Tuple[str, dict]:
393 """Convert a notebook from a file."""
394 return super().from_file(file_stream, resources, **kw) # type:ignore
396 def from_notebook_node( # type:ignore
397 self, nb: NotebookNode, resources: t.Optional[dict] = None, **kw: t.Any
398 ) -> t.Tuple[str, dict]:
399 """
400 Convert a notebook from a notebook node instance.
402 Parameters
403 ----------
404 nb : :class:`~nbformat.NotebookNode`
405 Notebook node
406 resources : dict
407 Additional resources that can be accessed read/write by
408 preprocessors and filters.
409 """
410 nb_copy, resources = super().from_notebook_node(nb, resources, **kw)
411 resources.setdefault("raw_mimetypes", self.raw_mimetypes)
412 resources["global_content_filter"] = {
413 "include_code": not self.exclude_code_cell,
414 "include_markdown": not self.exclude_markdown,
415 "include_raw": not self.exclude_raw,
416 "include_unknown": not self.exclude_unknown,
417 "include_input": not self.exclude_input,
418 "include_output": not self.exclude_output,
419 "include_output_stdin": not self.exclude_output_stdin,
420 "include_input_prompt": not self.exclude_input_prompt,
421 "include_output_prompt": not self.exclude_output_prompt,
422 "no_prompt": self.exclude_input_prompt and self.exclude_output_prompt,
423 }
425 # Top level variables are passed to the template_exporter here.
426 output = self.template.render(nb=nb_copy, resources=resources)
427 output = output.lstrip("\r\n")
428 return output, resources
430 def _register_filter(self, environ, name, jinja_filter):
431 """
432 Register a filter.
433 A filter is a function that accepts and acts on one string.
434 The filters are accessible within the Jinja templating engine.
436 Parameters
437 ----------
438 name : str
439 name to give the filter in the Jinja engine
440 filter : filter
441 """
442 if jinja_filter is None:
443 msg = "filter"
444 raise TypeError(msg)
445 isclass = isinstance(jinja_filter, type)
446 constructed = not isclass
448 # Handle filter's registration based on it's type
449 if constructed and isinstance(jinja_filter, (str,)):
450 # filter is a string, import the namespace and recursively call
451 # this register_filter method
452 filter_cls = import_item(jinja_filter)
453 return self._register_filter(environ, name, filter_cls)
455 if constructed and hasattr(jinja_filter, "__call__"): # noqa
456 # filter is a function, no need to construct it.
457 environ.filters[name] = jinja_filter
458 return jinja_filter
460 elif isclass and issubclass(jinja_filter, HasTraits):
461 # filter is configurable. Make sure to pass in new default for
462 # the enabled flag if one was specified.
463 filter_instance = jinja_filter(parent=self)
464 self._register_filter(environ, name, filter_instance)
466 elif isclass:
467 # filter is not configurable, construct it
468 filter_instance = jinja_filter()
469 self._register_filter(environ, name, filter_instance)
471 else:
472 # filter is an instance of something without a __call__
473 # attribute.
474 msg = "filter"
475 raise TypeError(msg)
477 def register_filter(self, name, jinja_filter):
478 """
479 Register a filter.
480 A filter is a function that accepts and acts on one string.
481 The filters are accessible within the Jinja templating engine.
483 Parameters
484 ----------
485 name : str
486 name to give the filter in the Jinja engine
487 filter : filter
488 """
489 return self._register_filter(self.environment, name, jinja_filter)
491 def default_filters(self):
492 """Override in subclasses to provide extra filters.
494 This should return an iterable of 2-tuples: (name, class-or-function).
495 You should call the method on the parent class and include the filters
496 it provides.
498 If a name is repeated, the last filter provided wins. Filters from
499 user-supplied config win over filters provided by classes.
500 """
501 return default_filters.items()
503 def _create_environment(self):
504 """
505 Create the Jinja templating environment.
506 """
507 paths = self.template_paths
508 self.log.debug("Template paths:\n\t%s", "\n\t".join(paths))
510 loaders = [
511 *self.extra_loaders,
512 ExtensionTolerantLoader(FileSystemLoader(paths), self.template_extension),
513 DictLoader({self._raw_template_key: self.raw_template}),
514 ]
515 environment = Environment( # noqa
516 loader=ChoiceLoader(loaders),
517 extensions=JINJA_EXTENSIONS,
518 enable_async=self.enable_async,
519 )
521 environment.globals["uuid4"] = uuid.uuid4
523 # Add default filters to the Jinja2 environment
524 for key, value in self.default_filters():
525 self._register_filter(environment, key, value)
527 # Load user filters. Overwrite existing filters if need be.
528 if self.filters:
529 for key, user_filter in self.filters.items():
530 self._register_filter(environment, key, user_filter)
532 return environment
534 def _init_preprocessors(self):
535 super()._init_preprocessors()
536 conf = self._get_conf()
537 preprocessors = conf.get("preprocessors", {})
538 # preprocessors is a dict for three reasons
539 # * We rely on recursive_update, which can only merge dicts, lists will be overwritten
540 # * We can use the key with numerical prefixing to guarantee ordering (/etc/*.d/XY-file style)
541 # * We can disable preprocessors by overwriting the value with None
542 for _, preprocessor in sorted(preprocessors.items(), key=lambda x: x[0]):
543 if preprocessor is not None:
544 kwargs = preprocessor.copy()
545 preprocessor_cls = kwargs.pop("type")
546 preprocessor_cls = import_item(preprocessor_cls)
547 if preprocessor_cls.__name__ in self.config:
548 kwargs.update(self.config[preprocessor_cls.__name__])
549 preprocessor = preprocessor_cls(**kwargs) # noqa
550 self.register_preprocessor(preprocessor)
552 def _get_conf(self):
553 conf: dict = {} # the configuration once all conf files are merged
554 for path in map(Path, self.template_paths):
555 conf_path = path / "conf.json"
556 if conf_path.exists():
557 with conf_path.open() as f:
558 conf = recursive_update(conf, json.load(f))
559 return conf
561 @default("template_paths")
562 def _template_paths(self, prune=True, root_dirs=None):
563 paths = []
564 root_dirs = self.get_prefix_root_dirs()
565 template_names = self.get_template_names()
566 for template_name in template_names:
567 for base_dir in self.extra_template_basedirs:
568 path = os.path.join(base_dir, template_name)
569 if not prune or os.path.exists(path):
570 paths.append(path)
571 for root_dir in root_dirs:
572 base_dir = os.path.join(root_dir, "nbconvert", "templates")
573 path = os.path.join(base_dir, template_name)
574 if not prune or os.path.exists(path):
575 paths.append(path)
577 for root_dir in root_dirs:
578 # we include root_dir for when we want to be very explicit, e.g.
579 # {% extends 'nbconvert/templates/classic/base.html' %}
580 paths.append(root_dir)
581 # we include base_dir for when we want to be explicit, but less than root_dir, e.g.
582 # {% extends 'classic/base.html' %}
583 base_dir = os.path.join(root_dir, "nbconvert", "templates")
584 paths.append(base_dir)
586 compatibility_dir = os.path.join(root_dir, "nbconvert", "templates", "compatibility")
587 paths.append(compatibility_dir)
589 additional_paths = []
590 for path in self.template_data_paths:
591 if not prune or os.path.exists(path):
592 additional_paths.append(path)
594 return paths + self.extra_template_paths + additional_paths
596 @classmethod
597 def get_compatibility_base_template_conf(cls, name):
598 """Get the base template config."""
599 # Hard-coded base template confs to use for backwards compatibility for 5.x-only templates
600 if name == "display_priority":
601 return {"base_template": "base"}
602 if name == "full":
603 return {"base_template": "classic", "mimetypes": {"text/html": True}}
605 def get_template_names(self): # noqa
606 """Finds a list of template names where each successive template name is the base template"""
607 template_names = []
608 root_dirs = self.get_prefix_root_dirs()
609 base_template = self.template_name
610 merged_conf: dict = {} # the configuration once all conf files are merged
611 while base_template is not None:
612 template_names.append(base_template)
613 conf: dict = {}
614 found_at_least_one = False
615 for base_dir in self.extra_template_basedirs:
616 template_dir = os.path.join(base_dir, base_template)
617 if os.path.exists(template_dir):
618 found_at_least_one = True
619 conf_file = os.path.join(template_dir, "conf.json")
620 if os.path.exists(conf_file):
621 with open(conf_file) as f:
622 conf = recursive_update(json.load(f), conf)
623 for root_dir in root_dirs:
624 template_dir = os.path.join(root_dir, "nbconvert", "templates", base_template)
625 if os.path.exists(template_dir):
626 found_at_least_one = True
627 conf_file = os.path.join(template_dir, "conf.json")
628 if os.path.exists(conf_file):
629 with open(conf_file) as f:
630 conf = recursive_update(json.load(f), conf)
631 if not found_at_least_one:
632 # Check for backwards compatibility template names
633 for root_dir in root_dirs:
634 compatibility_file = base_template + ".tpl"
635 compatibility_path = os.path.join(
636 root_dir, "nbconvert", "templates", "compatibility", compatibility_file
637 )
638 if os.path.exists(compatibility_path):
639 found_at_least_one = True
640 warnings.warn(
641 f"5.x template name passed '{self.template_name}'. Use 'lab' or 'classic' for new template usage.",
642 DeprecationWarning,
643 stacklevel=2,
644 )
645 self.template_file = compatibility_file
646 conf = self.get_compatibility_base_template_conf(base_template)
647 self.template_name = conf.get("base_template")
648 break
649 if not found_at_least_one:
650 paths = "\n\t".join(root_dirs)
651 msg = f"No template sub-directory with name {base_template!r} found in the following paths:\n\t{paths}"
652 raise ValueError(msg)
653 merged_conf = recursive_update(dict(conf), merged_conf)
654 base_template = conf.get("base_template")
655 conf = merged_conf
656 mimetypes = [mimetype for mimetype, enabled in conf.get("mimetypes", {}).items() if enabled]
657 if self.output_mimetype and self.output_mimetype not in mimetypes and mimetypes:
658 supported_mimetypes = "\n\t".join(mimetypes)
659 msg = f"Unsupported mimetype {self.output_mimetype!r} for template {self.template_name!r}, mimetypes supported are: \n\t{supported_mimetypes}"
660 raise ValueError(msg)
661 return template_names
663 def get_prefix_root_dirs(self):
664 """Get the prefix root dirs."""
665 # We look at the usual jupyter locations, and for development purposes also
666 # relative to the package directory (first entry, meaning with highest precedence)
667 root_dirs = []
668 if DEV_MODE:
669 root_dirs.append(os.path.abspath(os.path.join(ROOT, "..", "..", "share", "jupyter")))
670 root_dirs.extend(jupyter_path())
671 return root_dirs
673 def _init_resources(self, resources):
674 resources = super()._init_resources(resources)
675 resources["deprecated"] = deprecated
676 return resources