Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Internal module for formatting output data in csv, html, xml,
3and latex files. This module also applies to display formatting.
4"""
6from __future__ import annotations
8from collections.abc import (
9 Callable,
10 Generator,
11 Hashable,
12 Mapping,
13 Sequence,
14)
15from contextlib import contextmanager
16from csv import QUOTE_NONE
17from decimal import Decimal
18from functools import partial
19from io import StringIO
20import math
21import re
22from shutil import get_terminal_size
23from typing import (
24 TYPE_CHECKING,
25 Any,
26 cast,
27)
28import warnings
30import numpy as np
32from pandas._config.config import (
33 _global_config as config,
34 set_option,
35)
37from pandas._libs import lib
38from pandas._libs.missing import NA
39from pandas._libs.tslibs import (
40 NaT,
41 Timedelta,
42 Timestamp,
43 iNaT,
44)
45from pandas._libs.tslibs.dtypes import (
46 NpyDatetimeUnit,
47 periods_per_second,
48)
49from pandas._libs.tslibs.nattype import NaTType
50from pandas.errors import Pandas4Warning
51from pandas.util._decorators import set_module
52from pandas.util._exceptions import find_stack_level
54from pandas.core.dtypes.common import (
55 is_complex_dtype,
56 is_float,
57 is_integer,
58 is_list_like,
59 is_numeric_dtype,
60 is_scalar,
61)
62from pandas.core.dtypes.dtypes import (
63 CategoricalDtype,
64 DatetimeTZDtype,
65 ExtensionDtype,
66)
67from pandas.core.dtypes.missing import (
68 isna,
69 notna,
70)
72from pandas.core.arrays import (
73 Categorical,
74 DatetimeArray,
75 ExtensionArray,
76 TimedeltaArray,
77)
78from pandas.core.base import PandasObject
79import pandas.core.common as com
80from pandas.core.indexes.api import (
81 Index,
82 MultiIndex,
83 PeriodIndex,
84 ensure_index,
85)
86from pandas.core.indexes.datetimes import DatetimeIndex
87from pandas.core.indexes.timedeltas import TimedeltaIndex
89from pandas.io.common import (
90 check_parent_directory,
91 stringify_path,
92)
93from pandas.io.formats import printing
95if TYPE_CHECKING:
96 from pandas._typing import (
97 ArrayLike,
98 Axes,
99 ColspaceArgType,
100 ColspaceType,
101 CompressionOptions,
102 FilePath,
103 FloatFormatType,
104 FormattersType,
105 IndexLabel,
106 SequenceNotStr,
107 StorageOptions,
108 WriteBuffer,
109 )
111 from pandas import (
112 DataFrame,
113 Series,
114 )
117VALID_JUSTIFY_PARAMETERS = (
118 "left",
119 "right",
120 "center",
121 "justify",
122 "justify-all",
123 "start",
124 "end",
125 "inherit",
126 "match-parent",
127 "initial",
128 "unset",
129)
132class SeriesFormatter:
133 """
134 Implement the main logic of Series.to_string, which underlies
135 Series.__repr__.
136 """
138 def __init__(
139 self,
140 series: Series,
141 *,
142 length: bool | str = True,
143 header: bool = True,
144 index: bool = True,
145 na_rep: str = "NaN",
146 name: bool = False,
147 float_format: str | None = None,
148 dtype: bool = True,
149 max_rows: int | None = None,
150 min_rows: int | None = None,
151 ) -> None:
152 self.series = series
153 self.buf = StringIO()
154 self.name = name
155 self.na_rep = na_rep
156 self.header = header
157 self.length = length
158 self.index = index
159 self.max_rows = max_rows
160 self.min_rows = min_rows
162 if float_format is None:
163 float_format = config["display"]["float_format"]
164 self.float_format = float_format
165 self.dtype = dtype
166 self.adj = printing.get_adjustment()
168 self._chk_truncate()
170 def _chk_truncate(self) -> None:
171 self.tr_row_num: int | None
173 min_rows = self.min_rows
174 max_rows = self.max_rows
175 # truncation determined by max_rows, actual truncated number of rows
176 # used below by min_rows
177 is_truncated_vertically = max_rows and (len(self.series) > max_rows)
178 series = self.series
179 if is_truncated_vertically:
180 max_rows = cast("int", max_rows)
181 if min_rows:
182 # if min_rows is set (not None or 0), set max_rows to minimum
183 # of both
184 max_rows = min(min_rows, max_rows)
185 if max_rows == 1:
186 row_num = max_rows
187 series = series.iloc[:max_rows]
188 else:
189 row_num = max_rows // 2
190 _len = len(series)
191 _slice = np.hstack(
192 [np.arange(row_num), np.arange(_len - row_num, _len)]
193 )
194 series = series.iloc[_slice]
195 self.tr_row_num = row_num
196 else:
197 self.tr_row_num = None
198 self.tr_series = series
199 self.is_truncated_vertically = is_truncated_vertically
201 def _get_footer(self) -> str:
202 name = self.series.name
203 footer = ""
205 index = self.series.index
206 if (
207 isinstance(index, (DatetimeIndex, PeriodIndex, TimedeltaIndex))
208 and index.freq is not None
209 ):
210 footer += f"Freq: {index.freqstr}"
212 if self.name is not False and name is not None:
213 if footer:
214 footer += ", "
216 footer += f"Name: {format_name(name)}"
218 if self.length is True or (
219 self.length == "truncate" and self.is_truncated_vertically
220 ):
221 if footer:
222 footer += ", "
223 footer += f"Length: {len(self.series)}"
225 if self.dtype is not False and self.dtype is not None:
226 dtype_name = getattr(self.tr_series.dtype, "name", None)
227 if dtype_name:
228 if footer:
229 footer += ", "
230 footer += f"dtype: {printing.pprint_thing(dtype_name)}"
232 # level infos are added to the end and in a new line, like it is done
233 # for Categoricals
234 if isinstance(self.tr_series.dtype, CategoricalDtype):
235 level_info = self.tr_series._values._get_repr_footer()
236 if footer:
237 footer += "\n"
238 footer += level_info
240 return str(footer)
242 def _get_formatted_values(self) -> list[str]:
243 return format_array(
244 self.tr_series._values,
245 None,
246 float_format=self.float_format,
247 na_rep=self.na_rep,
248 leading_space=self.index,
249 )
251 def to_string(self) -> str:
252 series = self.tr_series
253 footer = self._get_footer()
255 if len(series) == 0:
256 return f"{type(self.series).__name__}([], {footer})"
258 index = series.index
259 have_header = _has_names(index)
260 if isinstance(index, MultiIndex):
261 fmt_index = index._format_multi(include_names=True, sparsify=None)
262 adj = printing.get_adjustment()
263 fmt_index = adj.adjoin(2, *fmt_index).split("\n")
264 else:
265 fmt_index = index._format_flat(include_name=True)
266 fmt_values = self._get_formatted_values()
268 if self.is_truncated_vertically:
269 n_header_rows = 0
270 row_num = self.tr_row_num
271 row_num = cast("int", row_num)
272 width = self.adj.len(fmt_values[row_num - 1])
273 if width > 3:
274 dot_str = "..."
275 else:
276 dot_str = ".."
277 # Series uses mode=center because it has single value columns
278 # DataFrame uses mode=left
279 dot_str = self.adj.justify([dot_str], width, mode="center")[0]
280 fmt_values.insert(row_num + n_header_rows, dot_str)
281 fmt_index.insert(row_num + 1, "")
283 if self.index:
284 result = self.adj.adjoin(3, *[fmt_index[1:], fmt_values])
285 else:
286 result = self.adj.adjoin(3, fmt_values)
288 if self.header and have_header:
289 result = fmt_index[0] + "\n" + result
291 if footer:
292 result += "\n" + footer
294 return str("".join(result))
297def get_dataframe_repr_params() -> dict[str, Any]:
298 """Get the parameters used to repr(dataFrame) calls using DataFrame.to_string.
300 Supplying these parameters to DataFrame.to_string is equivalent to calling
301 ``repr(DataFrame)``. This is useful if you want to adjust the repr output.
303 Example
304 -------
305 >>> import pandas as pd
306 >>>
307 >>> df = pd.DataFrame([[1, 2], [3, 4]])
308 >>> repr_params = pd.io.formats.format.get_dataframe_repr_params()
309 >>> repr(df) == df.to_string(**repr_params)
310 True
311 """
312 from pandas.io.formats import console
314 if config["display"]["expand_frame_repr"]:
315 line_width, _ = console.get_console_size()
316 else:
317 line_width = None
318 return {
319 "max_rows": config["display"]["max_rows"],
320 "min_rows": config["display"]["min_rows"],
321 "max_cols": config["display"]["max_columns"],
322 "max_colwidth": config["display"]["max_colwidth"],
323 "show_dimensions": config["display"]["show_dimensions"],
324 "line_width": line_width,
325 }
328def get_series_repr_params() -> dict[str, Any]:
329 """Get the parameters used to repr(Series) calls using Series.to_string.
331 Supplying these parameters to Series.to_string is equivalent to calling
332 ``repr(series)``. This is useful if you want to adjust the series repr output.
334 Example
335 -------
336 >>> import pandas as pd
337 >>>
338 >>> ser = pd.Series([1, 2, 3, 4])
339 >>> repr_params = pd.io.formats.format.get_series_repr_params()
340 >>> repr(ser) == ser.to_string(**repr_params)
341 True
342 """
343 width, height = get_terminal_size()
344 max_rows_opt = config["display"]["max_rows"]
345 max_rows = height if max_rows_opt == 0 else max_rows_opt
346 min_rows = height if max_rows_opt == 0 else config["display"]["min_rows"]
348 return {
349 "name": True,
350 "dtype": True,
351 "min_rows": min_rows,
352 "max_rows": max_rows,
353 "length": config["display"]["show_dimensions"],
354 }
357class DataFrameFormatter:
358 """
359 Class for processing dataframe formatting options and data.
361 Used by DataFrame.to_string, which backs DataFrame.__repr__.
363 Parameters
364 ----------
365 buf : str, Path or StringIO-like, optional, default None
366 Buffer to write to. If None, the output is returned as a string.
367 columns : array-like, optional, default None
368 The subset of columns to write. Writes all columns by default.
369 col_space : int, list or dict of int, optional
370 The minimum width of each column. If a list of ints is given
371 every integers corresponds with one column. If a dict is given,
372 the key references the column, while the value defines the space
373 to use.
374 header : bool or list of str, optional
375 Write out the column names. If a list of columns is given,
376 it is assumed to be aliases for the column names.
377 index : bool, optional, default True
378 Whether to print index (row) labels.
379 na_rep : str, optional, default 'NaN'
380 String representation of ``NaN`` to use.
381 formatters : list, tuple or dict of one-param. functions, optional
382 Formatter functions to apply to columns' elements by position or
383 name.
384 The result of each function must be a unicode string.
385 List/tuple must be of length equal to the number of columns.
386 float_format : one-parameter function, optional, default None
387 Formatter function to apply to columns' elements if they are
388 floats. This function must return a unicode string and will be
389 applied only to the non-``NaN`` elements, with ``NaN`` being
390 handled by ``na_rep``.
391 sparsify : bool, optional, default True
392 Set to False for a DataFrame with a hierarchical index to print
393 every multiindex key at each row.
394 index_names : bool, optional, default True
395 Prints the names of the indexes.
396 justify : str, default None
397 How to justify the column labels. If None uses the option from
398 the print configuration (controlled by set_option), 'right' out
399 of the box. Valid values are
401 * left
402 * right
403 * center
404 * justify
405 * justify-all
406 * start
407 * end
408 * inherit
409 * match-parent
410 * initial
411 * unset.
412 max_rows : int, optional
413 Maximum number of rows to display in the console.
414 max_cols : int, optional
415 Maximum number of columns to display in the console.
416 show_dimensions : bool, default False
417 Display DataFrame dimensions (number of rows by number of columns).
418 decimal : str, default '.'
419 Character recognized as decimal separator, e.g. ',' in Europe.
421 Returns
422 -------
423 str or None
424 If buf is None, returns the result as a string. Otherwise returns
425 None.
426 """
428 def __init__(
429 self,
430 frame: DataFrame,
431 columns: Axes | None = None,
432 col_space: ColspaceArgType | None = None,
433 header: bool | SequenceNotStr[str] = True,
434 index: bool = True,
435 na_rep: str = "NaN",
436 formatters: FormattersType | None = None,
437 justify: str | None = None,
438 float_format: FloatFormatType | None = None,
439 sparsify: bool | None = None,
440 index_names: bool = True,
441 max_rows: int | None = None,
442 min_rows: int | None = None,
443 max_cols: int | None = None,
444 show_dimensions: bool | str = False,
445 decimal: str = ".",
446 bold_rows: bool = False,
447 escape: bool = True,
448 ) -> None:
449 self.frame = frame
450 self.columns = self._initialize_columns(columns)
451 self.col_space = self._initialize_colspace(col_space)
452 self.header = header
453 self.index = index
454 self.na_rep = na_rep
455 self.formatters = self._initialize_formatters(formatters)
456 self.justify = self._initialize_justify(justify)
457 self.float_format = self._validate_float_format(float_format)
458 self.sparsify = self._initialize_sparsify(sparsify)
459 self.show_index_names = index_names
460 self.decimal = decimal
461 self.bold_rows = bold_rows
462 self.escape = escape
463 self.max_rows = max_rows
464 self.min_rows = min_rows
465 self.max_cols = max_cols
466 self.show_dimensions = show_dimensions
468 self.max_cols_fitted = self._calc_max_cols_fitted()
469 self.max_rows_fitted = self._calc_max_rows_fitted()
471 self.tr_frame = self.frame
472 self.truncate()
473 self.adj = printing.get_adjustment()
475 def get_strcols(self) -> list[list[str]]:
476 """
477 Render a DataFrame to a list of columns (as lists of strings).
478 """
479 strcols = self._get_strcols_without_index()
481 if self.index:
482 str_index = self._get_formatted_index(self.tr_frame)
483 strcols.insert(0, str_index)
485 return strcols
487 @property
488 def should_show_dimensions(self) -> bool:
489 return self.show_dimensions is True or (
490 self.show_dimensions == "truncate" and self.is_truncated
491 )
493 @property
494 def is_truncated(self) -> bool:
495 return bool(self.is_truncated_horizontally or self.is_truncated_vertically)
497 @property
498 def is_truncated_horizontally(self) -> bool:
499 return bool(self.max_cols_fitted and (len(self.columns) > self.max_cols_fitted))
501 @property
502 def is_truncated_vertically(self) -> bool:
503 return bool(self.max_rows_fitted and (len(self.frame) > self.max_rows_fitted))
505 @property
506 def dimensions_info(self) -> str:
507 return f"\n\n[{len(self.frame)} rows x {len(self.frame.columns)} columns]"
509 @property
510 def has_index_names(self) -> bool:
511 return _has_names(self.frame.index)
513 @property
514 def has_column_names(self) -> bool:
515 return _has_names(self.frame.columns)
517 @property
518 def show_row_idx_names(self) -> bool:
519 return all((self.has_index_names, self.index, self.show_index_names))
521 @property
522 def show_col_idx_names(self) -> bool:
523 return all((self.has_column_names, self.show_index_names, self.header))
525 @property
526 def max_rows_displayed(self) -> int:
527 return min(self.max_rows or len(self.frame), len(self.frame))
529 def _initialize_sparsify(self, sparsify: bool | None) -> bool:
530 if sparsify is None:
531 return config["display"]["multi_sparse"]
532 return sparsify
534 def _initialize_formatters(
535 self, formatters: FormattersType | None
536 ) -> FormattersType:
537 if formatters is None:
538 return {}
539 elif len(self.frame.columns) == len(formatters) or isinstance(formatters, dict):
540 return formatters
541 else:
542 raise ValueError(
543 f"Formatters length({len(formatters)}) should match "
544 f"DataFrame number of columns({len(self.frame.columns)})"
545 )
547 def _initialize_justify(self, justify: str | None) -> str:
548 if justify is None:
549 return config["display"]["colheader_justify"]
550 else:
551 return justify
553 def _initialize_columns(self, columns: Axes | None) -> Index:
554 if columns is not None:
555 cols = ensure_index(columns)
556 self.frame = self.frame[cols]
557 return cols
558 else:
559 return self.frame.columns
561 def _initialize_colspace(self, col_space: ColspaceArgType | None) -> ColspaceType:
562 result: ColspaceType
564 if col_space is None:
565 result = {}
566 elif isinstance(col_space, (int, str)):
567 result = {"": col_space}
568 result.update(dict.fromkeys(self.frame.columns, col_space))
569 elif isinstance(col_space, Mapping):
570 for column in col_space.keys():
571 if column not in self.frame.columns and column != "":
572 raise ValueError(
573 f"Col_space is defined for an unknown column: {column}"
574 )
575 result = col_space
576 else:
577 if len(self.frame.columns) != len(col_space):
578 raise ValueError(
579 f"Col_space length({len(col_space)}) should match "
580 f"DataFrame number of columns({len(self.frame.columns)})"
581 )
582 result = dict(zip(self.frame.columns, col_space, strict=True))
583 return result
585 def _calc_max_cols_fitted(self) -> int | None:
586 """Number of columns fitting the screen."""
587 if not self._is_in_terminal():
588 return self.max_cols
590 width, _ = get_terminal_size()
591 if self._is_screen_narrow(width):
592 return width
593 else:
594 return self.max_cols
596 def _calc_max_rows_fitted(self) -> int | None:
597 """Number of rows with data fitting the screen."""
598 max_rows: int | None
600 if self._is_in_terminal():
601 _, height = get_terminal_size()
602 if self.max_rows == 0:
603 # rows available to fill with actual data
604 return height - self._get_number_of_auxiliary_rows()
606 if self._is_screen_short(height):
607 max_rows = height
608 else:
609 max_rows = self.max_rows
610 else:
611 max_rows = self.max_rows
613 return self._adjust_max_rows(max_rows)
615 def _adjust_max_rows(self, max_rows: int | None) -> int | None:
616 """Adjust max_rows using display logic.
618 See description here:
619 https://pandas.pydata.org/docs/dev/user_guide/options.html#frequently-used-options
621 GH #37359
622 """
623 if max_rows:
624 if (len(self.frame) > max_rows) and self.min_rows:
625 # if truncated, set max_rows showed to min_rows
626 max_rows = min(self.min_rows, max_rows)
627 return max_rows
629 def _is_in_terminal(self) -> bool:
630 """Check if the output is to be shown in terminal."""
631 return bool(self.max_cols == 0 or self.max_rows == 0)
633 def _is_screen_narrow(self, max_width) -> bool:
634 return bool(self.max_cols == 0 and len(self.frame.columns) > max_width)
636 def _is_screen_short(self, max_height) -> bool:
637 return bool(self.max_rows == 0 and len(self.frame) > max_height)
639 def _get_number_of_auxiliary_rows(self) -> int:
640 """Get number of rows occupied by prompt, dots and dimension info."""
641 dot_row = 1
642 prompt_row = 1
643 num_rows = dot_row + prompt_row
645 if self.show_dimensions:
646 num_rows += len(self.dimensions_info.splitlines())
648 if self.header:
649 num_rows += 1
651 return num_rows
653 def truncate(self) -> None:
654 """
655 Check whether the frame should be truncated. If so, slice the frame up.
656 """
657 if self.is_truncated_horizontally:
658 self._truncate_horizontally()
660 if self.is_truncated_vertically:
661 self._truncate_vertically()
663 def _truncate_horizontally(self) -> None:
664 """Remove columns, which are not to be displayed and adjust formatters.
666 Attributes affected:
667 - tr_frame
668 - formatters
669 - tr_col_num
670 """
671 assert self.max_cols_fitted is not None
672 col_num = self.max_cols_fitted // 2
673 if col_num >= 1:
674 _len = len(self.tr_frame.columns)
675 _slice = np.hstack([np.arange(col_num), np.arange(_len - col_num, _len)])
676 self.tr_frame = self.tr_frame.iloc[:, _slice]
678 # truncate formatter
679 if isinstance(self.formatters, (list, tuple)):
680 self.formatters = [
681 *self.formatters[:col_num],
682 *self.formatters[-col_num:],
683 ]
684 else:
685 col_num = cast("int", self.max_cols)
686 self.tr_frame = self.tr_frame.iloc[:, :col_num]
687 self.tr_col_num: int = col_num
689 def _truncate_vertically(self) -> None:
690 """Remove rows, which are not to be displayed.
692 Attributes affected:
693 - tr_frame
694 - tr_row_num
695 """
696 assert self.max_rows_fitted is not None
697 row_num = self.max_rows_fitted // 2
698 if row_num >= 1:
699 _len = len(self.tr_frame)
700 _slice = np.hstack([np.arange(row_num), np.arange(_len - row_num, _len)])
701 self.tr_frame = self.tr_frame.iloc[_slice]
702 else:
703 row_num = cast("int", self.max_rows)
704 self.tr_frame = self.tr_frame.iloc[:row_num, :]
705 self.tr_row_num = row_num
707 def _get_strcols_without_index(self) -> list[list[str]]:
708 strcols: list[list[str]] = []
710 if not is_list_like(self.header) and not self.header:
711 for i, c in enumerate(self.tr_frame):
712 fmt_values = self.format_col(i)
713 fmt_values, _ = _make_fixed_width(
714 strings=fmt_values,
715 justify=self.justify,
716 minimum=int(self.col_space.get(c, 0)),
717 adj=self.adj,
718 )
719 strcols.append(fmt_values)
720 return strcols
722 if is_list_like(self.header):
723 # cast here since can't be bool if is_list_like
724 self.header = cast("list[str]", self.header)
725 if len(self.header) != len(self.columns):
726 raise ValueError(
727 f"Writing {len(self.columns)} cols "
728 f"but got {len(self.header)} aliases"
729 )
730 str_columns = [[label] for label in self.header]
731 else:
732 str_columns = self._get_formatted_column_labels(self.tr_frame)
734 if self.show_row_idx_names:
735 for x in str_columns:
736 x.append("")
738 for i, c in enumerate(self.tr_frame):
739 cheader = str_columns[i]
740 header_colwidth = max(
741 int(self.col_space.get(c, 0)), *(self.adj.len(x) for x in cheader)
742 )
743 fmt_values = self.format_col(i)
744 fmt_values, max_len = _make_fixed_width(
745 fmt_values, self.justify, minimum=header_colwidth, adj=self.adj
746 )
747 max_len = max(max_len, header_colwidth)
748 cheader = self.adj.justify(cheader, max_len, mode=self.justify)
749 strcols.append(cheader + fmt_values)
751 return strcols
753 def format_col(self, i: int) -> list[str]:
754 frame = self.tr_frame
755 formatter = self._get_formatter(i)
756 return format_array(
757 frame._get_column_array(i),
758 formatter,
759 float_format=self.float_format,
760 na_rep=self.na_rep,
761 space=self.col_space.get(frame.columns[i]),
762 decimal=self.decimal,
763 leading_space=self.index,
764 )
766 def _get_formatter(self, i: str | int) -> Callable | None:
767 if isinstance(self.formatters, (list, tuple)):
768 if is_integer(i):
769 i = cast("int", i)
770 return self.formatters[i]
771 else:
772 return None
773 else:
774 if is_integer(i):
775 i = self.tr_frame.columns[i]
776 return self.formatters.get(i, None)
778 def _get_formatted_column_labels(self, frame: DataFrame) -> list[list[str]]:
779 from pandas.core.indexes.multi import sparsify_labels
781 columns = frame.columns
783 if isinstance(columns, MultiIndex):
784 fmt_columns = columns._format_multi(sparsify=False, include_names=False)
785 if self.sparsify and len(fmt_columns):
786 fmt_columns = sparsify_labels(fmt_columns)
788 str_columns = [list(x) for x in zip(*fmt_columns, strict=True)]
789 else:
790 fmt_columns = columns._format_flat(include_name=False)
791 str_columns = [
792 [
793 " " + x
794 if not self._get_formatter(i) and is_numeric_dtype(dtype)
795 else x
796 ]
797 for i, (x, dtype) in enumerate(
798 zip(fmt_columns, self.frame.dtypes, strict=False)
799 )
800 ]
801 return str_columns
803 def _get_formatted_index(self, frame: DataFrame) -> list[str]:
804 # Note: this is only used by to_string() and to_latex(), not by
805 # to_html(). so safe to cast col_space here.
806 col_space = {k: cast("int", v) for k, v in self.col_space.items()}
807 index = frame.index
808 columns = frame.columns
809 fmt = self._get_formatter("__index__")
811 if isinstance(index, MultiIndex):
812 fmt_index = index._format_multi(
813 sparsify=self.sparsify,
814 include_names=self.show_row_idx_names,
815 formatter=fmt,
816 )
817 else:
818 fmt_index = [
819 index._format_flat(include_name=self.show_row_idx_names, formatter=fmt)
820 ]
822 fmt_index = [
823 tuple(
824 _make_fixed_width(
825 list(x), justify="left", minimum=col_space.get("", 0), adj=self.adj
826 )[0]
827 )
828 for x in fmt_index
829 ]
831 adjoined = self.adj.adjoin(1, *fmt_index).split("\n")
833 # empty space for columns
834 if self.show_col_idx_names:
835 col_header = [format_name(x) for x in self._get_column_name_list()]
836 else:
837 col_header = [""] * columns.nlevels
839 if self.header:
840 return col_header + adjoined
841 else:
842 return adjoined
844 def _get_column_name_list(self) -> list[Hashable]:
845 names: list[Hashable] = []
846 columns = self.frame.columns
847 if isinstance(columns, MultiIndex):
848 names.extend(columns.names)
849 else:
850 names.append(columns.name)
851 return names
853 def _validate_float_format(
854 self, fmt: FloatFormatType | None
855 ) -> FloatFormatType | None:
856 """
857 Validates and processes the float_format argument.
858 Converts new-style format strings to callables.
859 """
860 if fmt is None or callable(fmt):
861 return fmt
863 if isinstance(fmt, str):
864 if "%" in fmt:
865 # Keeps old-style format strings as they are (C code handles them)
866 return fmt
867 else:
868 try:
869 _ = fmt.format(1.0) # Test with an arbitrary float
870 return fmt.format
871 except (ValueError, KeyError, IndexError) as e:
872 raise ValueError(f"Invalid new-style format string {fmt!r}") from e
874 raise ValueError("float_format must be a string or callable")
877class DataFrameRenderer:
878 """Class for creating dataframe output in multiple formats.
880 Called in pandas.core.generic.NDFrame:
881 - to_csv
882 - to_latex
884 Called in pandas.DataFrame:
885 - to_html
886 - to_string
888 Parameters
889 ----------
890 fmt : DataFrameFormatter
891 Formatter with the formatting options.
892 """
894 def __init__(self, fmt: DataFrameFormatter) -> None:
895 self.fmt = fmt
897 def to_html(
898 self,
899 buf: FilePath | WriteBuffer[str] | None = None,
900 encoding: str | None = None,
901 classes: str | list | tuple | None = None,
902 notebook: bool = False,
903 border: int | bool | None = None,
904 table_id: str | None = None,
905 render_links: bool = False,
906 ) -> str | None:
907 """
908 Render a DataFrame to an html table.
910 Parameters
911 ----------
912 buf : str, path object, file-like object, or None, default None
913 String, path object (implementing ``os.PathLike[str]``), or file-like
914 object implementing a string ``write()`` function. If None, the result is
915 returned as a string.
916 encoding : str, default “utf-8”
917 Set character encoding.
918 classes : str or list-like
919 classes to include in the `class` attribute of the opening
920 ``<table>`` tag, in addition to the default "dataframe".
921 notebook : {True, False}, optional, default False
922 Whether the generated HTML is for IPython Notebook.
923 border : int or bool
924 When an integer value is provided, it sets the border attribute in
925 the opening tag, specifying the thickness of the border.
926 If ``False`` or ``0`` is passed, the border attribute will not
927 be present in the ``<table>`` tag.
928 The default value for this parameter is governed by
929 ``pd.options.display.html.border``.
930 table_id : str, optional
931 A css id is included in the opening `<table>` tag if specified.
932 render_links : bool, default False
933 Convert URLs to HTML links.
934 """
935 from pandas.io.formats.html import (
936 HTMLFormatter,
937 NotebookFormatter,
938 )
940 Klass = NotebookFormatter if notebook else HTMLFormatter
942 html_formatter = Klass(
943 self.fmt,
944 classes=classes,
945 border=border,
946 table_id=table_id,
947 render_links=render_links,
948 )
949 string = html_formatter.to_string()
950 return save_to_buffer(string, buf=buf, encoding=encoding)
952 def to_string(
953 self,
954 buf: FilePath | WriteBuffer[str] | None = None,
955 encoding: str | None = None,
956 line_width: int | None = None,
957 ) -> str | None:
958 """
959 Render a DataFrame to a console-friendly tabular output.
961 Parameters
962 ----------
963 buf : str, path object, file-like object, or None, default None
964 String, path object (implementing ``os.PathLike[str]``), or file-like
965 object implementing a string ``write()`` function. If None, the result is
966 returned as a string.
967 encoding: str, default “utf-8”
968 Set character encoding.
969 line_width : int, optional
970 Width to wrap a line in characters.
971 """
972 from pandas.io.formats.string import StringFormatter
974 string_formatter = StringFormatter(self.fmt, line_width=line_width)
975 string = string_formatter.to_string()
976 return save_to_buffer(string, buf=buf, encoding=encoding)
978 def to_csv(
979 self,
980 path_or_buf: FilePath | WriteBuffer[bytes] | WriteBuffer[str] | None = None,
981 encoding: str | None = None,
982 sep: str = ",",
983 columns: Sequence[Hashable] | None = None,
984 index_label: IndexLabel | None = None,
985 mode: str = "w",
986 compression: CompressionOptions = "infer",
987 quoting: int | None = None,
988 quotechar: str = '"',
989 lineterminator: str | None = None,
990 chunksize: int | None = None,
991 date_format: str | None = None,
992 doublequote: bool = True,
993 escapechar: str | None = None,
994 errors: str = "strict",
995 storage_options: StorageOptions | None = None,
996 ) -> str | None:
997 """
998 Render dataframe as comma-separated file.
999 """
1000 from pandas.io.formats.csvs import CSVFormatter
1002 if path_or_buf is None:
1003 created_buffer = True
1004 path_or_buf = StringIO()
1005 else:
1006 created_buffer = False
1008 csv_formatter = CSVFormatter(
1009 path_or_buf=path_or_buf,
1010 lineterminator=lineterminator,
1011 sep=sep,
1012 encoding=encoding,
1013 errors=errors,
1014 compression=compression,
1015 quoting=quoting,
1016 cols=columns,
1017 index_label=index_label,
1018 mode=mode,
1019 chunksize=chunksize,
1020 quotechar=quotechar,
1021 date_format=date_format,
1022 doublequote=doublequote,
1023 escapechar=escapechar,
1024 storage_options=storage_options,
1025 formatter=self.fmt,
1026 )
1027 csv_formatter.save()
1029 if created_buffer:
1030 assert isinstance(path_or_buf, StringIO)
1031 content = path_or_buf.getvalue()
1032 path_or_buf.close()
1033 return content
1035 return None
1038def save_to_buffer(
1039 string: str,
1040 buf: FilePath | WriteBuffer[str] | None = None,
1041 encoding: str | None = None,
1042) -> str | None:
1043 """
1044 Perform serialization. Write to buf or return as string if buf is None.
1045 """
1046 with _get_buffer(buf, encoding=encoding) as fd:
1047 fd.write(string)
1048 if buf is None:
1049 # error: "WriteBuffer[str]" has no attribute "getvalue"
1050 return fd.getvalue() # type: ignore[attr-defined]
1051 return None
1054@contextmanager
1055def _get_buffer(
1056 buf: FilePath | WriteBuffer[str] | None, encoding: str | None = None
1057) -> Generator[WriteBuffer[str]] | Generator[StringIO]:
1058 """
1059 Context manager to open, yield and close buffer for filenames or Path-like
1060 objects, otherwise yield buf unchanged.
1061 """
1062 if buf is not None:
1063 buf = stringify_path(buf)
1064 else:
1065 buf = StringIO()
1067 if encoding is None:
1068 encoding = "utf-8"
1069 elif not isinstance(buf, str):
1070 raise ValueError("buf is not a file name and encoding is specified.")
1072 if hasattr(buf, "write"):
1073 # Incompatible types in "yield" (actual type "Union[str, WriteBuffer[str],
1074 # StringIO]", expected type "Union[WriteBuffer[str], StringIO]")
1075 yield buf # type: ignore[misc]
1076 elif isinstance(buf, str):
1077 check_parent_directory(str(buf))
1078 with open(buf, "w", encoding=encoding, newline="") as f:
1079 # GH#30034 open instead of codecs.open prevents a file leak
1080 # if we have an invalid encoding argument.
1081 # newline="" is needed to roundtrip correctly on
1082 # windows test_to_latex_filename
1083 yield f
1084 else:
1085 raise TypeError("buf is not a file name and it has no write method")
1088# ----------------------------------------------------------------------
1089# Array formatters
1092def format_name(name: Hashable) -> str:
1093 """
1094 Render an Index or Series name for display.
1096 Float names follow ``display.precision`` like float values do (GH#25917).
1097 """
1098 if name is None:
1099 return ""
1100 if lib.is_float(name) and not isna(name):
1101 return format_array(np.array([name]), None, leading_space=False)[0]
1102 return printing.pprint_thing(name, escape_chars=("\t", "\r", "\n"))
1105def format_array(
1106 values: ArrayLike,
1107 formatter: Callable | None,
1108 float_format: FloatFormatType | None = None,
1109 na_rep: str = "NaN",
1110 digits: int | None = None,
1111 space: str | int | None = None,
1112 justify: str = "right",
1113 decimal: str = ".",
1114 leading_space: bool | None = True,
1115 quoting: int | None = None,
1116 fallback_formatter: Callable | None = None,
1117) -> list[str]:
1118 """
1119 Format an array for printing.
1121 Parameters
1122 ----------
1123 values : np.ndarray or ExtensionArray
1124 formatter
1125 float_format
1126 na_rep
1127 digits
1128 space
1129 justify
1130 decimal
1131 leading_space : bool, optional, default True
1132 Whether the array should be formatted with a leading space.
1133 When an array as a column of a Series or DataFrame, we do want
1134 the leading space to pad between columns.
1136 When formatting an Index subclass
1137 (e.g. IntervalIndex._get_values_for_csv), we don't want the
1138 leading space since it should be left-aligned.
1139 fallback_formatter
1141 Returns
1142 -------
1143 List[str]
1144 """
1145 fmt_klass: type[_GenericArrayFormatter]
1146 if lib.is_np_dtype(values.dtype, "M") or isinstance(values.dtype, DatetimeTZDtype):
1147 fmt_klass = _Datetime64Formatter
1148 values = cast("DatetimeArray", values)
1149 if na_rep == "NaN":
1150 na_rep = "NaT"
1151 elif lib.is_np_dtype(values.dtype, "m"):
1152 fmt_klass = _Timedelta64Formatter
1153 values = cast("TimedeltaArray", values)
1154 if na_rep == "NaN":
1155 na_rep = "NaT"
1156 elif isinstance(values.dtype, ExtensionDtype):
1157 fmt_klass = _ExtensionArrayFormatter
1158 elif lib.is_np_dtype(values.dtype, "fc"):
1159 fmt_klass = FloatArrayFormatter
1160 elif lib.is_np_dtype(values.dtype, "iu"):
1161 fmt_klass = _IntArrayFormatter
1162 else:
1163 fmt_klass = _GenericArrayFormatter
1165 if space is None:
1166 space = 12
1168 if float_format is None:
1169 float_format = config["display"]["float_format"]
1171 if digits is None:
1172 digits = config["display"]["precision"]
1174 fmt_obj = fmt_klass(
1175 values,
1176 digits=digits,
1177 na_rep=na_rep,
1178 float_format=float_format,
1179 formatter=formatter,
1180 space=space,
1181 justify=justify,
1182 decimal=decimal,
1183 leading_space=leading_space,
1184 quoting=quoting,
1185 fallback_formatter=fallback_formatter,
1186 )
1188 return fmt_obj.get_result()
1191class _GenericArrayFormatter:
1192 def __init__(
1193 self,
1194 values: ArrayLike,
1195 digits: int = 7,
1196 formatter: Callable | None = None,
1197 na_rep: str = "NaN",
1198 space: str | int = 12,
1199 float_format: FloatFormatType | None = None,
1200 justify: str = "right",
1201 decimal: str = ".",
1202 quoting: int | None = None,
1203 fixed_width: bool = True,
1204 leading_space: bool | None = True,
1205 fallback_formatter: Callable | None = None,
1206 ) -> None:
1207 self.values = values
1208 self.digits = digits
1209 self.na_rep = na_rep
1210 self.space = space
1211 self.formatter = formatter
1212 self.float_format = float_format
1213 self.justify = justify
1214 self.decimal = decimal
1215 self.quoting = quoting
1216 self.fixed_width = fixed_width
1217 self.leading_space = leading_space
1218 self.fallback_formatter = fallback_formatter
1220 def get_result(self) -> list[str]:
1221 fmt_values = self._format_strings()
1222 result, _ = _make_fixed_width(fmt_values, self.justify)
1223 return result
1225 def _format_strings(self) -> list[str]:
1226 if self.float_format is None:
1227 float_format = config["display"]["float_format"]
1228 if float_format is None:
1229 precision = config["display"]["precision"]
1230 float_format = lambda x: _trim_zeros_single_float(
1231 f"{x: .{precision:d}f}"
1232 )
1233 else:
1234 float_format = self.float_format
1236 if self.formatter is not None:
1237 formatter = self.formatter
1238 elif self.fallback_formatter is not None:
1239 formatter = self.fallback_formatter
1240 else:
1241 quote_strings = self.quoting is not None and self.quoting != QUOTE_NONE
1242 formatter = partial(
1243 printing.pprint_thing,
1244 escape_chars=("\t", "\r", "\n"),
1245 quote_strings=quote_strings,
1246 )
1248 def _format(x):
1249 if self.na_rep is not None and is_scalar(x) and isna(x):
1250 if x is None:
1251 return "None"
1252 elif x is NA:
1253 return str(NA)
1254 elif x is NaT or isinstance(x, (np.datetime64, np.timedelta64)):
1255 return "NaT"
1256 return self.na_rep
1257 elif isinstance(x, PandasObject):
1258 return str(x)
1259 else:
1260 # object dtype
1261 return str(formatter(x))
1263 vals = self.values
1264 if not isinstance(vals, np.ndarray):
1265 raise TypeError(
1266 "ExtensionArray formatting should use _ExtensionArrayFormatter"
1267 )
1268 inferred = lib.map_infer(vals, is_float)
1269 is_float_type = (
1270 inferred # type: ignore[operator]
1271 # vals may have 2 or more dimensions
1272 & np.all(notna(vals), axis=tuple(range(1, len(vals.shape))))
1273 )
1274 leading_space = self.leading_space
1275 if leading_space is None:
1276 leading_space = is_float_type.any() # type: ignore[assignment]
1278 fmt_values = []
1279 for i, v in enumerate(vals):
1280 if self.formatter is not None:
1281 # match the int/float/datetime formatters, GH#26002
1282 fmt_values.append(_format(v))
1283 elif not is_float_type[i] and leading_space:
1284 fmt_values.append(f" {_format(v)}")
1285 elif is_float_type[i]:
1286 fmt_values.append(float_format(v))
1287 else:
1288 if leading_space is False:
1289 # False specifically, so that the default is
1290 # to include a space if we get here.
1291 tpl = "{v}"
1292 else:
1293 tpl = " {v}"
1294 fmt_values.append(tpl.format(v=_format(v)))
1296 return fmt_values
1299class FloatArrayFormatter(_GenericArrayFormatter):
1300 def __init__(self, *args, **kwargs) -> None:
1301 super().__init__(*args, **kwargs)
1303 # float_format is expected to be a string
1304 # formatter should be used to pass a function
1305 if self.float_format is not None and self.formatter is None:
1306 # GH21625, GH22270
1307 self.fixed_width = False
1308 if callable(self.float_format):
1309 self.formatter = self.float_format
1310 self.float_format = None
1312 def _value_formatter(
1313 self,
1314 float_format: FloatFormatType | None = None,
1315 threshold: float | None = None,
1316 ) -> Callable:
1317 """Returns a function to be applied on each value to format it"""
1318 # the float_format parameter supersedes self.float_format
1319 if float_format is None:
1320 float_format = self.float_format
1322 # we are going to compose different functions, to first convert to
1323 # a string, then replace the decimal symbol, and finally chop according
1324 # to the threshold
1326 # when there is no float_format, we use str instead of '%g'
1327 # because str(0.0) = '0.0' while '%g' % 0.0 = '0'
1328 if float_format:
1330 def base_formatter(v): # pyright: ignore[reportRedeclaration]
1331 assert float_format is not None # for mypy
1332 # error: "str" not callable
1333 # error: Unexpected keyword argument "value" for "__call__" of
1334 # "EngFormatter"
1335 return (
1336 float_format(value=v) # type: ignore[operator,call-arg]
1337 if notna(v)
1338 else self.na_rep
1339 )
1341 else:
1343 def base_formatter(v):
1344 return str(v) if notna(v) else self.na_rep
1346 if self.decimal != ".":
1348 def decimal_formatter(v):
1349 return base_formatter(v).replace(".", self.decimal, 1)
1351 else:
1352 decimal_formatter = base_formatter
1354 if threshold is None:
1355 return decimal_formatter
1357 def formatter(value):
1358 if notna(value):
1359 if abs(value) > threshold:
1360 return decimal_formatter(value)
1361 else:
1362 return decimal_formatter(0.0)
1363 else:
1364 return self.na_rep
1366 return formatter
1368 def get_result_as_array(self) -> np.ndarray:
1369 """
1370 Returns the float values converted into strings using
1371 the parameters given at initialisation, as a numpy array
1372 """
1374 def format_with_na_rep(
1375 values: ArrayLike, formatter: Callable, na_rep: str
1376 ) -> np.ndarray:
1377 mask = isna(values)
1378 formatted = np.array(
1379 [
1380 formatter(val) if not m else na_rep
1381 for val, m in zip(values.ravel(), mask.ravel(), strict=True)
1382 ]
1383 ).reshape(values.shape)
1384 return formatted
1386 def format_complex_with_na_rep(
1387 values: ArrayLike, formatter: Callable, na_rep: str
1388 ) -> np.ndarray:
1389 real_values = np.real(values).ravel() # type: ignore[arg-type]
1390 imag_values = np.imag(values).ravel() # type: ignore[arg-type]
1391 real_mask, imag_mask = isna(real_values), isna(imag_values)
1392 formatted_lst = []
1393 for val, real_val, imag_val, re_isna, im_isna in zip(
1394 values.ravel(),
1395 real_values,
1396 imag_values,
1397 real_mask,
1398 imag_mask,
1399 strict=True,
1400 ):
1401 if not re_isna and not im_isna:
1402 formatted_lst.append(formatter(val))
1403 elif not re_isna: # xxx+nanj
1404 formatted_lst.append(f"{formatter(real_val)}+{na_rep}j")
1405 elif not im_isna: # nan[+/-]xxxj
1406 # The imaginary part may either start with a "-" or a space
1407 imag_formatted = formatter(imag_val).strip()
1408 if imag_formatted.startswith("-"):
1409 formatted_lst.append(f"{na_rep}{imag_formatted}j")
1410 else:
1411 formatted_lst.append(f"{na_rep}+{imag_formatted}j")
1412 else: # nan+nanj
1413 formatted_lst.append(f"{na_rep}+{na_rep}j")
1414 return np.array(formatted_lst).reshape(values.shape)
1416 if self.formatter is not None:
1417 return format_with_na_rep(self.values, self.formatter, self.na_rep)
1419 if self.fixed_width:
1420 threshold = config["display"]["chop_threshold"]
1421 else:
1422 threshold = None
1424 # if we have a fixed_width, we'll need to try different float_format
1425 def format_values_with(float_format):
1426 formatter = self._value_formatter(float_format, threshold)
1428 # default formatter leaves a space to the left when formatting
1429 # floats, must be consistent for left-justifying NaNs (GH #25061)
1430 na_rep = " " + self.na_rep if self.justify == "left" else self.na_rep
1432 # different formatting strategies for complex and non-complex data
1433 # need to distinguish complex and float NaNs (GH #53762)
1434 values = self.values
1435 is_complex = is_complex_dtype(values)
1437 # separate the wheat from the chaff
1438 if is_complex:
1439 values = format_complex_with_na_rep(values, formatter, na_rep)
1440 else:
1441 values = format_with_na_rep(values, formatter, na_rep)
1443 if self.fixed_width:
1444 if is_complex:
1445 result = _trim_zeros_complex(values, self.decimal)
1446 else:
1447 result = _trim_zeros_float(values, self.decimal)
1448 return np.asarray(result, dtype="object")
1450 return values
1452 # There is a special default string when we are fixed-width
1453 # The default is otherwise to use str instead of a formatting string
1454 float_format: FloatFormatType | None
1455 if self.float_format is None:
1456 if self.fixed_width:
1457 if self.leading_space is True:
1458 fmt_str = "{value: .{digits:d}f}"
1459 else:
1460 fmt_str = "{value:.{digits:d}f}"
1461 float_format = partial(fmt_str.format, digits=self.digits)
1462 else:
1463 float_format = self.float_format
1464 else:
1465 float_format = lambda value: self.float_format % value
1467 formatted_values = format_values_with(float_format)
1469 if not self.fixed_width:
1470 return formatted_values
1472 # we need do convert to engineering format if some values are too small
1473 # and would appear as 0, or if some values are too big and take too
1474 # much space
1476 if len(formatted_values) > 0:
1477 maxlen = max(len(x) for x in formatted_values)
1478 too_long = maxlen > self.digits + 6
1479 else:
1480 too_long = False
1482 abs_vals = np.abs(self.values)
1483 # this is pretty arbitrary for now
1484 # large values: more that 8 characters including decimal symbol
1485 # and first digit, hence > 1e6
1486 has_large_values = (abs_vals > 1e6).any()
1487 has_small_values = ((abs_vals < 10 ** (-self.digits)) & (abs_vals > 0)).any()
1489 if has_small_values or (too_long and has_large_values):
1490 if self.leading_space is True:
1491 fmt_str = "{value: .{digits:d}e}"
1492 else:
1493 fmt_str = "{value:.{digits:d}e}"
1494 float_format = partial(fmt_str.format, digits=self.digits)
1495 formatted_values = format_values_with(float_format)
1497 return formatted_values
1499 def _format_strings(self) -> list[str]:
1500 return list(self.get_result_as_array())
1503class _IntArrayFormatter(_GenericArrayFormatter):
1504 def _format_strings(self) -> list[str]:
1505 if self.leading_space is False:
1506 formatter_str = lambda x: f"{x:d}"
1507 else:
1508 formatter_str = lambda x: f"{x: d}"
1509 formatter = self.formatter or formatter_str
1510 fmt_values = [formatter(x) for x in self.values]
1511 return fmt_values
1514class _Datetime64Formatter(_GenericArrayFormatter):
1515 values: DatetimeArray
1517 def __init__(
1518 self,
1519 values: DatetimeArray,
1520 na_rep: str = "NaT",
1521 date_format: None = None,
1522 **kwargs,
1523 ) -> None:
1524 super().__init__(values, na_rep=na_rep, **kwargs)
1525 self.date_format = date_format
1527 def _format_strings(self) -> list[str]:
1528 values = self.values
1530 if self.formatter is not None:
1531 return [self.formatter(x) for x in values]
1533 fmt_values = values._format_native_types(
1534 na_rep=self.na_rep, date_format=self.date_format
1535 )
1536 return fmt_values.tolist()
1539class _ExtensionArrayFormatter(_GenericArrayFormatter):
1540 values: ExtensionArray
1542 def _format_strings(self) -> list[str]:
1543 values = self.values
1545 formatter = self.formatter
1546 fallback_formatter = None
1547 if formatter is None:
1548 fallback_formatter = values._formatter(boxed=True)
1550 if isinstance(values, Categorical):
1551 # Categorical is special for now, so that we can preserve tzinfo
1552 array = values._internal_get_values()
1553 else:
1554 array = np.asarray(values, dtype=object)
1556 fmt_values = format_array(
1557 array,
1558 formatter,
1559 float_format=self.float_format,
1560 na_rep=self.na_rep,
1561 digits=self.digits,
1562 space=self.space,
1563 justify=self.justify,
1564 decimal=self.decimal,
1565 leading_space=self.leading_space,
1566 quoting=self.quoting,
1567 fallback_formatter=fallback_formatter,
1568 )
1569 return fmt_values
1572def format_percentiles(
1573 percentiles: np.ndarray | Sequence[float],
1574) -> list[str]:
1575 """
1576 Outputs rounded and formatted percentiles.
1578 Parameters
1579 ----------
1580 percentiles : list-like, containing floats from interval [0,1]
1582 Returns
1583 -------
1584 formatted : list of strings
1586 Notes
1587 -----
1588 Rounding precision is chosen so that: (1) if any two elements of
1589 ``percentiles`` differ, they remain different after rounding
1590 (2) no entry is *rounded* to 0% or 100%.
1591 Any non-integer is always rounded to at least 1 decimal place.
1593 Examples
1594 --------
1595 Keeps all entries different after rounding:
1597 >>> format_percentiles([0.01999, 0.02001, 0.5, 0.666666, 0.9999])
1598 ['1.999%', '2.001%', '50%', '66.667%', '99.99%']
1600 No element is rounded to 0% or 100% (unless already equal to it).
1601 Duplicates are allowed:
1603 >>> format_percentiles([0, 0.5, 0.02001, 0.5, 0.666666, 0.9999])
1604 ['0%', '50%', '2.0%', '50%', '66.67%', '99.99%']
1605 """
1606 if len(percentiles) == 0:
1607 return []
1609 percentiles = np.asarray(percentiles)
1611 # It checks for np.nan as well
1612 if (
1613 not is_numeric_dtype(percentiles)
1614 or not np.all(percentiles >= 0)
1615 or not np.all(percentiles <= 1)
1616 ):
1617 raise ValueError("percentiles should all be in the interval [0,1]")
1619 percentiles = 100 * percentiles
1620 prec = get_precision(percentiles)
1621 percentiles_round_type = percentiles.round(prec).astype(int)
1623 int_idx = np.isclose(percentiles_round_type, percentiles)
1625 if np.all(int_idx):
1626 out = percentiles_round_type.astype(str)
1627 return [i + "%" for i in out]
1629 unique_pcts = np.unique(percentiles)
1630 prec = get_precision(unique_pcts)
1631 out = np.empty_like(percentiles, dtype=object)
1632 out[int_idx] = percentiles[int_idx].round().astype(int).astype(str)
1634 out[~int_idx] = percentiles[~int_idx].round(prec).astype(str)
1635 return [i + "%" for i in out]
1638def get_precision(array: np.ndarray | Sequence[float]) -> int:
1639 to_begin = array[0] if array[0] > 0 else None
1640 to_end = 100 - array[-1] if array[-1] < 100 else None
1641 diff = np.ediff1d(array, to_begin=to_begin, to_end=to_end)
1642 diff = abs(diff)
1643 prec = -np.floor(np.log10(np.min(diff))).astype(int)
1644 prec = max(1, prec)
1645 return prec
1648def _format_datetime64(x: NaTType | Timestamp, na_rep: str = "NaT") -> str:
1649 if x is NaT:
1650 return na_rep
1652 # Timestamp.__str__ falls back to datetime.datetime.__str__ = isoformat(sep=' ')
1653 # so it already uses string formatting rather than strftime (faster).
1654 return str(x)
1657def _format_datetime64_dateonly(
1658 x: NaTType | Timestamp,
1659 na_rep: str = "NaT",
1660 date_format: str | None = None,
1661) -> str:
1662 if isinstance(x, NaTType):
1663 return na_rep
1665 if date_format:
1666 return x.strftime(date_format)
1667 else:
1668 # Timestamp._date_repr relies on string formatting (faster than strftime)
1669 return x._date_repr
1672def get_format_datetime64(
1673 is_dates_only: bool, na_rep: str = "NaT", date_format: str | None = None
1674) -> Callable:
1675 """Return a formatter callable taking a datetime64 as input and providing
1676 a string as output"""
1678 if is_dates_only:
1679 return lambda x: _format_datetime64_dateonly(
1680 x, na_rep=na_rep, date_format=date_format
1681 )
1682 else:
1683 return lambda x: _format_datetime64(x, na_rep=na_rep)
1686class _Timedelta64Formatter(_GenericArrayFormatter):
1687 values: TimedeltaArray
1689 def __init__(
1690 self,
1691 values: TimedeltaArray,
1692 na_rep: str = "NaT",
1693 **kwargs,
1694 ) -> None:
1695 super().__init__(values, na_rep=na_rep, **kwargs)
1697 def _format_strings(self) -> list[str]:
1698 formatter = self.formatter or get_format_timedelta64(
1699 self.values, na_rep=self.na_rep, box=False
1700 )
1701 return [formatter(x) for x in self.values]
1704def get_format_timedelta64(
1705 values: TimedeltaArray,
1706 na_rep: str | float = "NaT",
1707 box: bool = False,
1708) -> Callable:
1709 """
1710 Return a formatter function for a range of timedeltas.
1711 These will all have the same format argument
1713 If box, then show the return in quotes
1714 """
1715 if values._is_dates_only:
1716 format = None
1717 else:
1718 i8 = values.asi8
1719 vals = i8[i8 != iNaT]
1720 if vals.size == 0 or not (vals % periods_per_second(values._creso)).any():
1721 format = "long"
1722 elif NpyDatetimeUnit.NPY_FR_ns.value == values._creso and (vals % 1_000).any():
1723 format = "all" # nanosecond precision → 9 digits
1724 else:
1725 format = "us" # microsecond precision → 6 digits
1727 def _formatter(x):
1728 if x is None or (is_scalar(x) and isna(x)):
1729 return na_rep
1731 if not isinstance(x, Timedelta):
1732 x = Timedelta(x)
1734 # Timedelta._repr_base uses string formatting (faster than strftime)
1735 result = x._repr_base(format=format)
1736 if box:
1737 result = f"'{result}'"
1738 return result
1740 return _formatter
1743def _make_fixed_width(
1744 strings: list[str],
1745 justify: str = "right",
1746 minimum: int | None = None,
1747 adj: printing._TextAdjustment | None = None,
1748) -> tuple[list[str], int]:
1749 if adj is None:
1750 adjustment = printing.get_adjustment()
1751 else:
1752 adjustment = adj
1754 if len(strings) == 0 or justify == "all":
1755 max_len = max(adjustment.len(x) for x in strings) if strings else 0
1756 return strings, max_len
1758 max_len = max(adjustment.len(x) for x in strings)
1760 if minimum is not None:
1761 max_len = max(minimum, max_len)
1763 conf_max = config["display"]["max_colwidth"]
1764 if conf_max is not None and max_len > conf_max:
1765 max_len = conf_max
1767 if conf_max is not None:
1768 # When the column is too narrow to hold any text alongside the "..."
1769 # placeholder, show as much of the placeholder as fits (GH#16097).
1770 keep = max(max_len - 3, 0)
1771 placeholder = "..."[: max_len - keep]
1772 strings = [
1773 x[:keep] + placeholder if adjustment.len(x) > max_len else x
1774 for x in strings
1775 ]
1777 result = adjustment.justify(strings, max_len, mode=justify)
1778 return result, max_len
1781def _trim_zeros_complex(str_complexes: ArrayLike, decimal: str = ".") -> list[str]:
1782 """
1783 Separates the real and imaginary parts from the complex number, and
1784 executes the _trim_zeros_float method on each of those.
1785 """
1786 real_part, imag_part = [], []
1787 for x in str_complexes:
1788 # Complex numbers are represented as "(-)xxx(+/-)xxxj"
1789 # The split will give [{"", "-"}, "xxx", "+/-", "xxx", "j", ""]
1790 # Therefore, the imaginary part is the 4th and 3rd last elements,
1791 # and the real part is everything before the imaginary part
1792 trimmed = re.split(r"(?<!e)([j+-])", x)
1793 real_part.append("".join(trimmed[:-4]))
1794 imag_part.append("".join(trimmed[-4:-2]))
1796 # We want to align the lengths of the real and imaginary parts of each complex
1797 # number, as well as the lengths the real (resp. complex) parts of all numbers
1798 # in the array
1799 n = len(str_complexes)
1800 padded_parts = _trim_zeros_float(real_part + imag_part, decimal)
1801 if len(padded_parts) == 0:
1802 return []
1803 padded_length = max(len(part) for part in padded_parts) - 1
1804 padded = [
1805 real_pt # real part, possibly NaN
1806 + imag_pt[0] # +/-
1807 + f"{imag_pt[1:]:>{padded_length}}" # complex part (no sign), possibly nan
1808 + "j"
1809 for real_pt, imag_pt in zip(padded_parts[:n], padded_parts[n:], strict=True)
1810 ]
1811 return padded
1814def _trim_zeros_single_float(str_float: str) -> str:
1815 """
1816 Trims trailing zeros after a decimal point,
1817 leaving just one if necessary.
1818 """
1819 str_float = str_float.rstrip("0")
1820 if str_float.endswith("."):
1821 str_float += "0"
1823 return str_float
1826_NUMBER_WITH_DECIMAL_RE = re.compile(r"^\s*[+-]?[0-9]+\.[0-9]*$")
1829def _trim_zeros_float(
1830 str_floats: ArrayLike | list[str], decimal: str = "."
1831) -> list[str]:
1832 """
1833 Trims the maximum number of trailing zeros equally from
1834 all numbers containing decimals, leaving just one if
1835 necessary.
1836 """
1837 if decimal != ".":
1838 number_regex = re.compile(rf"^\s*[+-]?[0-9]+\{decimal}[0-9]*$")
1839 else:
1840 number_regex = _NUMBER_WITH_DECIMAL_RE
1842 # Identify which elements are decimal numbers once, up front
1843 is_number = [number_regex.match(x) is not None for x in str_floats]
1845 if not any(is_number):
1846 return list(str_floats)
1848 trimmed = list(str_floats)
1850 # Trim trailing zeros from all decimal numbers equally
1851 while True:
1852 # Check if all decimal numbers end in "0"
1853 if not all(
1854 x.endswith("0")
1855 for x, is_num in zip(trimmed, is_number, strict=True)
1856 if is_num
1857 ):
1858 break
1859 trimmed = [
1860 x[:-1] if is_num else x
1861 for x, is_num in zip(trimmed, is_number, strict=True)
1862 ]
1864 # Leave one 0 after the decimal point if needed
1865 result = [
1866 x + "0" if is_num and x.endswith(decimal) else x
1867 for x, is_num in zip(trimmed, is_number, strict=True)
1868 ]
1869 return result
1872def _has_names(index: Index) -> bool:
1873 if isinstance(index, MultiIndex):
1874 return com.any_not_none(*index.names)
1875 else:
1876 return index.name is not None
1879class EngFormatter:
1880 """
1881 Formats float values according to engineering format.
1883 Based on matplotlib.ticker.EngFormatter
1884 """
1886 # The SI engineering prefixes
1887 ENG_PREFIXES = {
1888 -24: "y",
1889 -21: "z",
1890 -18: "a",
1891 -15: "f",
1892 -12: "p",
1893 -9: "n",
1894 -6: "u",
1895 -3: "m",
1896 0: "",
1897 3: "k",
1898 6: "M",
1899 9: "G",
1900 12: "T",
1901 15: "P",
1902 18: "E",
1903 21: "Z",
1904 24: "Y",
1905 }
1907 def __init__(
1908 self, accuracy: int | None = None, use_eng_prefix: bool = False
1909 ) -> None:
1910 self.accuracy = accuracy
1911 self.use_eng_prefix = use_eng_prefix
1913 def __call__(self, num: float) -> str:
1914 """
1915 Formats a number in engineering notation, appending a letter
1916 representing the power of 1000 of the original number. Some examples:
1917 >>> format_eng = EngFormatter(accuracy=0, use_eng_prefix=True)
1918 >>> format_eng(0)
1919 ' 0'
1920 >>> format_eng = EngFormatter(accuracy=1, use_eng_prefix=True)
1921 >>> format_eng(1_000_000)
1922 ' 1.0M'
1923 >>> format_eng = EngFormatter(accuracy=2, use_eng_prefix=False)
1924 >>> format_eng("-1e-6")
1925 '-1.00E-06'
1927 @param num: the value to represent
1928 @type num: either a numeric value or a string that can be converted to
1929 a numeric value (as per decimal.Decimal constructor)
1931 @return: engineering formatted string
1932 """
1933 dnum = Decimal(str(num))
1935 if Decimal.is_nan(dnum):
1936 return "NaN"
1938 if Decimal.is_infinite(dnum):
1939 return "inf"
1941 sign = 1
1943 if dnum < 0: # pragma: no cover
1944 sign = -1
1945 dnum = -dnum
1947 if dnum != 0:
1948 pow10 = Decimal(int(math.floor(dnum.log10() / 3) * 3))
1949 else:
1950 pow10 = Decimal(0)
1952 pow10 = pow10.min(max(self.ENG_PREFIXES.keys()))
1953 pow10 = pow10.max(min(self.ENG_PREFIXES.keys()))
1954 int_pow10 = int(pow10)
1956 if self.use_eng_prefix:
1957 prefix = self.ENG_PREFIXES[int_pow10]
1958 elif int_pow10 < 0:
1959 prefix = f"E-{-int_pow10:02d}"
1960 else:
1961 prefix = f"E+{int_pow10:02d}"
1963 mant = sign * dnum / (10**pow10)
1965 if self.accuracy is None: # pragma: no cover
1966 format_str = "{mant: g}{prefix}"
1967 else:
1968 format_str = f"{{mant: .{self.accuracy:d}f}}{{prefix}}"
1970 formatted = format_str.format(mant=mant, prefix=prefix)
1972 return formatted
1975@set_module("pandas")
1976def set_eng_float_format(accuracy: int = 3, use_eng_prefix: bool = False) -> None:
1977 """
1978 Format float representation in DataFrame with SI notation.
1980 .. deprecated:: 3.1.0
1981 Use ``pd.set_option("display.precision", N)`` to control decimal
1982 precision, or pass a custom callable to
1983 ``pd.set_option("display.float_format", func)``.
1985 Sets the floating-point display format for ``DataFrame`` objects using engineering
1986 notation (SI units), allowing easier readability of values across wide ranges.
1988 Parameters
1989 ----------
1990 accuracy : int, default 3
1991 Number of decimal digits after the floating point.
1992 use_eng_prefix : bool, default False
1993 Whether to represent a value with SI prefixes.
1995 Returns
1996 -------
1997 None
1998 This method does not return a value. it updates the global display format
1999 for floats in DataFrames.
2001 See Also
2002 --------
2003 set_option : Set the value of the specified option or options.
2004 reset_option : Reset one or more options to their default value.
2006 Examples
2007 --------
2008 Use ``pd.set_option("display.precision", N)`` to control decimal
2009 precision instead:
2011 >>> with pd.option_context("display.precision", 3):
2012 ... print(pd.DataFrame([1e-9, 1e-3, 1, 1e3, 1e6]))
2013 0
2014 0 1.000e-09
2015 1 1.000e-03
2016 2 1.000e+00
2017 3 1.000e+03
2018 4 1.000e+06
2019 """
2020 warnings.warn(
2021 "set_eng_float_format is deprecated and will be removed in a future "
2022 "version. Use pd.set_option('display.precision', N) to control decimal "
2023 "precision, or pass a custom callable to "
2024 "pd.set_option('display.float_format', func).",
2025 Pandas4Warning,
2026 stacklevel=find_stack_level(),
2027 )
2028 set_option("display.float_format", EngFormatter(accuracy, use_eng_prefix))
2031def get_level_lengths(
2032 levels: Any, sentinel: bool | object | str = ""
2033) -> list[dict[int, int]]:
2034 """
2035 For each index in each level the function returns lengths of indexes.
2037 Parameters
2038 ----------
2039 levels : list of lists
2040 List of values on for level.
2041 sentinel : string, optional
2042 Value which states that no new index starts on there.
2044 Returns
2045 -------
2046 Returns list of maps. For each level returns map of indexes (key is index
2047 in row and value is length of index).
2048 """
2049 if len(levels) == 0:
2050 return []
2052 control = [True] * len(levels[0])
2054 result = []
2055 for level in levels:
2056 last_index = 0
2058 lengths = {}
2059 for i, key in enumerate(level):
2060 if control[i] and key == sentinel:
2061 pass
2062 else:
2063 control[i] = False
2064 lengths[last_index] = i - last_index
2065 last_index = i
2067 lengths[last_index] = len(level) - last_index
2069 result.append(lengths)
2071 return result
2074def buffer_put_lines(buf: WriteBuffer[str], lines: list[str]) -> None:
2075 """
2076 Appends lines to a buffer.
2078 Parameters
2079 ----------
2080 buf
2081 The buffer to write to
2082 lines
2083 The lines to append.
2084 """
2085 if any(isinstance(x, str) for x in lines):
2086 lines = [str(x) for x in lines]
2087 buf.write("\n".join(lines))