Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from abc import (
4 ABC,
5 abstractmethod,
6)
7from functools import cached_property
8import sys
9from typing import TYPE_CHECKING
11from pandas._config.config import _global_config as config
13from pandas.io.formats import format as fmt
14from pandas.io.formats.printing import (
15 get_adjustment,
16 pprint_thing,
17)
19if TYPE_CHECKING:
20 from collections.abc import (
21 Iterable,
22 Iterator,
23 Mapping,
24 Sequence,
25 )
27 from pandas._typing import (
28 Dtype,
29 WriteBuffer,
30 )
32 from pandas import (
33 DataFrame,
34 Index,
35 Series,
36 )
38 from pandas.io.formats import printing
41def _sizeof_fmt(num: float, size_qualifier: str) -> str:
42 """
43 Return size in human readable format.
45 Parameters
46 ----------
47 num : int
48 Size in bytes.
49 size_qualifier : str
50 Either empty, or '+' (if lower bound).
52 Returns
53 -------
54 str
55 Size in human readable format.
57 Examples
58 --------
59 >>> _sizeof_fmt(23028, "")
60 '22.5 KB'
62 >>> _sizeof_fmt(23028, "+")
63 '22.5+ KB'
64 """
65 for x in ["bytes", "KB", "MB", "GB", "TB"]:
66 if num < 1024.0:
67 return f"{num:3.1f}{size_qualifier} {x}"
68 num /= 1024.0
69 return f"{num:3.1f}{size_qualifier} PB"
72def _initialize_memory_usage(
73 memory_usage: bool | str | None = None,
74) -> bool | str:
75 """Get memory usage based on inputs and display options."""
76 if memory_usage is None:
77 memory_usage = config["display"]["memory_usage"]
78 return memory_usage
81class _BaseInfo(ABC):
82 """
83 Base class for DataFrameInfo and SeriesInfo.
85 Parameters
86 ----------
87 data : DataFrame or Series
88 Either dataframe or series.
89 memory_usage : bool or str, optional
90 If "deep", introspect the data deeply by interrogating object dtypes
91 for system-level memory consumption, and include it in the returned
92 values.
93 """
95 data: DataFrame | Series
96 memory_usage: bool | str
98 @property
99 @abstractmethod
100 def dtypes(self) -> Iterable[Dtype]:
101 """
102 Dtypes.
104 Returns
105 -------
106 dtypes : sequence
107 Dtype of each of the DataFrame's columns (or one series column).
108 """
110 @property
111 @abstractmethod
112 def dtype_counts(self) -> Mapping[str, int]:
113 """Mapping dtype - number of counts."""
115 @property
116 @abstractmethod
117 def non_null_counts(self) -> list[int] | Series:
118 """Sequence of non-null counts for all columns or column (if series)."""
120 @property
121 @abstractmethod
122 def memory_usage_bytes(self) -> int:
123 """
124 Memory usage in bytes.
126 Returns
127 -------
128 memory_usage_bytes : int
129 Object's total memory usage in bytes.
130 """
132 @property
133 def memory_usage_string(self) -> str:
134 """Memory usage in a form of human readable string."""
135 return f"{_sizeof_fmt(self.memory_usage_bytes, self.size_qualifier)}\n"
137 @property
138 def size_qualifier(self) -> str:
139 size_qualifier = ""
140 if self.memory_usage:
141 if self.memory_usage != "deep":
142 # size_qualifier is just a best effort; not guaranteed to catch
143 # all cases (e.g., it misses categorical data even with object
144 # categories)
145 if (
146 "object" in self.dtype_counts
147 or self.data.index._is_memory_usage_qualified
148 ):
149 size_qualifier = "+"
150 return size_qualifier
152 @abstractmethod
153 def render(
154 self,
155 *,
156 buf: WriteBuffer[str] | None,
157 max_cols: int | None,
158 verbose: bool | None,
159 show_counts: bool | None,
160 ) -> None:
161 pass
164class DataFrameInfo(_BaseInfo):
165 """
166 Class storing dataframe-specific info.
167 """
169 def __init__(
170 self,
171 data: DataFrame,
172 memory_usage: bool | str | None = None,
173 ) -> None:
174 self.data: DataFrame = data
175 self.memory_usage = _initialize_memory_usage(memory_usage)
177 @property
178 def dtype_counts(self) -> Mapping[str, int]:
179 return _get_dataframe_dtype_counts(self.data)
181 @property
182 def dtypes(self) -> Iterable[Dtype]:
183 """
184 Dtypes.
186 Returns
187 -------
188 dtypes
189 Dtype of each of the DataFrame's columns.
190 """
191 return self.data.dtypes
193 @property
194 def ids(self) -> Index:
195 """
196 Column names.
198 Returns
199 -------
200 ids : Index
201 DataFrame's column names.
202 """
203 return self.data.columns
205 @property
206 def col_count(self) -> int:
207 """Number of columns to be summarized."""
208 return len(self.ids)
210 @property
211 def non_null_counts(self) -> Series:
212 """Sequence of non-null counts for all columns or column (if series)."""
213 return self.data.count()
215 @property
216 def memory_usage_bytes(self) -> int:
217 deep = self.memory_usage == "deep"
218 return self.data.memory_usage(index=True, deep=deep).sum()
220 def render(
221 self,
222 *,
223 buf: WriteBuffer[str] | None,
224 max_cols: int | None,
225 verbose: bool | None,
226 show_counts: bool | None,
227 ) -> None:
228 printer = _DataFrameInfoPrinter(
229 info=self,
230 max_cols=max_cols,
231 verbose=verbose,
232 show_counts=show_counts,
233 )
234 printer.to_buffer(buf)
237class SeriesInfo(_BaseInfo):
238 """
239 Class storing series-specific info.
240 """
242 def __init__(
243 self,
244 data: Series,
245 memory_usage: bool | str | None = None,
246 ) -> None:
247 self.data: Series = data
248 self.memory_usage = _initialize_memory_usage(memory_usage)
250 def render(
251 self,
252 *,
253 buf: WriteBuffer[str] | None = None,
254 max_cols: int | None = None,
255 verbose: bool | None = None,
256 show_counts: bool | None = None,
257 ) -> None:
258 if max_cols is not None:
259 raise ValueError(
260 "Argument `max_cols` can only be passed "
261 "in DataFrame.info, not Series.info"
262 )
263 printer = _SeriesInfoPrinter(
264 info=self,
265 verbose=verbose,
266 show_counts=show_counts,
267 )
268 printer.to_buffer(buf)
270 @property
271 def non_null_counts(self) -> list[int]:
272 return [self.data.count()]
274 @property
275 def dtypes(self) -> Iterable[Dtype]:
276 return [self.data.dtypes]
278 @property
279 def dtype_counts(self) -> Mapping[str, int]:
280 from pandas.core.frame import DataFrame
282 return _get_dataframe_dtype_counts(DataFrame(self.data))
284 @property
285 def memory_usage_bytes(self) -> int:
286 """Memory usage in bytes.
288 Returns
289 -------
290 memory_usage_bytes : int
291 Object's total memory usage in bytes.
292 """
293 deep = self.memory_usage == "deep"
294 return self.data.memory_usage(index=True, deep=deep)
297class _InfoPrinterAbstract:
298 """
299 Class for printing dataframe or series info.
300 """
302 def to_buffer(self, buf: WriteBuffer[str] | None = None) -> None:
303 """Save dataframe info into buffer."""
304 table_builder = self._create_table_builder()
305 lines = table_builder.get_lines()
306 if buf is None: # pragma: no cover
307 buf = sys.stdout
308 fmt.buffer_put_lines(buf, lines)
310 @abstractmethod
311 def _create_table_builder(self) -> _TableBuilderAbstract:
312 """Create instance of table builder."""
315class _DataFrameInfoPrinter(_InfoPrinterAbstract):
316 """
317 Class for printing dataframe info.
319 Parameters
320 ----------
321 info : DataFrameInfo
322 Instance of DataFrameInfo.
323 max_cols : int, optional
324 When to switch from the verbose to the truncated output.
325 verbose : bool, optional
326 Whether to print the full summary.
327 show_counts : bool, optional
328 Whether to show the non-null counts.
329 """
331 def __init__(
332 self,
333 info: DataFrameInfo,
334 max_cols: int | None = None,
335 verbose: bool | None = None,
336 show_counts: bool | None = None,
337 ) -> None:
338 self.info = info
339 self.data = info.data
340 self.verbose = verbose
341 self.max_cols = self._initialize_max_cols(max_cols)
342 self.show_counts = self._initialize_show_counts(show_counts)
344 @property
345 def max_rows(self) -> int:
346 """Maximum info rows to be displayed."""
347 return config["display"]["max_info_rows"]
349 @property
350 def exceeds_info_cols(self) -> bool:
351 """Check if number of columns to be summarized does not exceed maximum."""
352 return bool(self.col_count > self.max_cols)
354 @property
355 def exceeds_info_rows(self) -> bool:
356 """Check if number of rows to be summarized does not exceed maximum."""
357 return bool(len(self.data) > self.max_rows)
359 @property
360 def col_count(self) -> int:
361 """Number of columns to be summarized."""
362 return self.info.col_count
364 def _initialize_max_cols(self, max_cols: int | None) -> int:
365 if max_cols is None:
366 return config["display"]["max_info_columns"]
367 return max_cols
369 def _initialize_show_counts(self, show_counts: bool | None) -> bool:
370 if show_counts is None:
371 return bool(not self.exceeds_info_cols and not self.exceeds_info_rows)
372 else:
373 return show_counts
375 def _create_table_builder(self) -> _DataFrameTableBuilder:
376 """
377 Create instance of table builder based on verbosity and display settings.
378 """
379 if self.verbose:
380 return _DataFrameTableBuilderVerbose(
381 info=self.info,
382 with_counts=self.show_counts,
383 )
384 elif self.verbose is False: # specifically set to False, not necessarily None
385 return _DataFrameTableBuilderNonVerbose(info=self.info)
386 elif self.exceeds_info_cols:
387 return _DataFrameTableBuilderNonVerbose(info=self.info)
388 else:
389 return _DataFrameTableBuilderVerbose(
390 info=self.info,
391 with_counts=self.show_counts,
392 )
395class _SeriesInfoPrinter(_InfoPrinterAbstract):
396 """Class for printing series info.
398 Parameters
399 ----------
400 info : SeriesInfo
401 Instance of SeriesInfo.
402 verbose : bool, optional
403 Whether to print the full summary.
404 show_counts : bool, optional
405 Whether to show the non-null counts.
406 """
408 def __init__(
409 self,
410 info: SeriesInfo,
411 verbose: bool | None = None,
412 show_counts: bool | None = None,
413 ) -> None:
414 self.info = info
415 self.data = info.data
416 self.verbose = verbose
417 self.show_counts = self._initialize_show_counts(show_counts)
419 def _create_table_builder(self) -> _SeriesTableBuilder:
420 """
421 Create instance of table builder based on verbosity.
422 """
423 if self.verbose or self.verbose is None:
424 return _SeriesTableBuilderVerbose(
425 info=self.info,
426 with_counts=self.show_counts,
427 )
428 else:
429 return _SeriesTableBuilderNonVerbose(info=self.info)
431 def _initialize_show_counts(self, show_counts: bool | None) -> bool:
432 if show_counts is None:
433 return True
434 else:
435 return show_counts
438class _TableBuilderAbstract(ABC):
439 """
440 Abstract builder for info table.
441 """
443 _lines: list[str]
444 info: _BaseInfo
446 @abstractmethod
447 def get_lines(self) -> list[str]:
448 """Product in a form of list of lines (strings)."""
450 @property
451 def data(self) -> DataFrame | Series:
452 return self.info.data
454 @property
455 def dtypes(self) -> Iterable[Dtype]:
456 """Dtypes of each of the DataFrame's columns."""
457 return self.info.dtypes
459 @property
460 def dtype_counts(self) -> Mapping[str, int]:
461 """Mapping dtype - number of counts."""
462 return self.info.dtype_counts
464 @property
465 def display_memory_usage(self) -> bool:
466 """Whether to display memory usage."""
467 return bool(self.info.memory_usage)
469 @property
470 def memory_usage_string(self) -> str:
471 """Memory usage string with proper size qualifier."""
472 return self.info.memory_usage_string
474 @property
475 def non_null_counts(self) -> list[int] | Series:
476 return self.info.non_null_counts
478 def add_object_type_line(self) -> None:
479 """Add line with string representation of dataframe to the table."""
480 self._lines.append(str(type(self.data)))
482 def add_index_range_line(self) -> None:
483 """Add line with range of indices to the table."""
484 self._lines.append(self.data.index._summary())
486 def add_dtypes_line(self) -> None:
487 """Add summary line with dtypes present in dataframe."""
488 collected_dtypes = [
489 f"{key}({val:d})" for key, val in sorted(self.dtype_counts.items())
490 ]
491 self._lines.append(f"dtypes: {', '.join(collected_dtypes)}")
494class _DataFrameTableBuilder(_TableBuilderAbstract):
495 """
496 Abstract builder for dataframe info table.
498 Parameters
499 ----------
500 info : DataFrameInfo.
501 Instance of DataFrameInfo.
502 """
504 def __init__(self, *, info: DataFrameInfo) -> None:
505 self.info: DataFrameInfo = info
507 def get_lines(self) -> list[str]:
508 self._lines = []
509 if self.col_count == 0:
510 self._fill_empty_info()
511 else:
512 self._fill_non_empty_info()
513 return self._lines
515 def _fill_empty_info(self) -> None:
516 """Add lines to the info table, pertaining to empty dataframe."""
517 self.add_object_type_line()
518 self.add_index_range_line()
519 self._lines.append(f"Empty {type(self.data).__name__}\n")
521 @abstractmethod
522 def _fill_non_empty_info(self) -> None:
523 """Add lines to the info table, pertaining to non-empty dataframe."""
525 @property
526 def data(self) -> DataFrame:
527 """DataFrame."""
528 return self.info.data
530 @property
531 def ids(self) -> Index:
532 """Dataframe columns."""
533 return self.info.ids
535 @property
536 def col_count(self) -> int:
537 """Number of dataframe columns to be summarized."""
538 return self.info.col_count
540 def add_memory_usage_line(self) -> None:
541 """Add line containing memory usage."""
542 self._lines.append(f"memory usage: {self.memory_usage_string}")
545class _DataFrameTableBuilderNonVerbose(_DataFrameTableBuilder):
546 """
547 Dataframe info table builder for non-verbose output.
548 """
550 def _fill_non_empty_info(self) -> None:
551 """Add lines to the info table, pertaining to non-empty dataframe."""
552 self.add_object_type_line()
553 self.add_index_range_line()
554 self.add_columns_summary_line()
555 self.add_dtypes_line()
556 if self.display_memory_usage:
557 self.add_memory_usage_line()
559 def add_columns_summary_line(self) -> None:
560 self._lines.append(self.ids._summary(name="Columns"))
563class _TableBuilderVerboseMixin(_TableBuilderAbstract):
564 """
565 Mixin for verbose info output.
566 """
568 SPACING: str = " " * 2
569 strrows: Sequence[Sequence[str]]
570 gross_column_widths: Sequence[int]
571 with_counts: bool
573 @cached_property
574 def _adj(self) -> printing._TextAdjustment:
575 # respects display.unicode.east_asian_width, like DataFrame.__repr__
576 return get_adjustment()
578 def _ljust(self, text: str, width: int) -> str:
579 return self._adj.justify([text], width, mode="left")[0]
581 @property
582 @abstractmethod
583 def headers(self) -> Sequence[str]:
584 """Headers names of the columns in verbose table."""
586 @property
587 def header_column_widths(self) -> Sequence[int]:
588 """Widths of header columns (only titles)."""
589 return [self._adj.len(col) for col in self.headers]
591 def _get_gross_column_widths(self) -> Sequence[int]:
592 """Get widths of columns containing both headers and actual content."""
593 body_column_widths = self._get_body_column_widths()
594 return [
595 max(*widths)
596 for widths in zip(
597 self.header_column_widths, body_column_widths, strict=False
598 )
599 ]
601 def _get_body_column_widths(self) -> Sequence[int]:
602 """Get widths of table content columns."""
603 strcols: Sequence[Sequence[str]] = list(zip(*self.strrows, strict=True))
604 return [max(self._adj.len(x) for x in col) for col in strcols]
606 def _gen_rows(self) -> Iterator[Sequence[str]]:
607 """
608 Generator function yielding rows content.
610 Each element represents a row comprising a sequence of strings.
611 """
612 if self.with_counts:
613 return self._gen_rows_with_counts()
614 else:
615 return self._gen_rows_without_counts()
617 @abstractmethod
618 def _gen_rows_with_counts(self) -> Iterator[Sequence[str]]:
619 """Iterator with string representation of body data with counts."""
621 @abstractmethod
622 def _gen_rows_without_counts(self) -> Iterator[Sequence[str]]:
623 """Iterator with string representation of body data without counts."""
625 def add_header_line(self) -> None:
626 header_line = self.SPACING.join(
627 [
628 self._ljust(header, col_width)
629 for header, col_width in zip(
630 self.headers, self.gross_column_widths, strict=True
631 )
632 ]
633 )
634 self._lines.append(header_line)
636 def add_separator_line(self) -> None:
637 separator_line = self.SPACING.join(
638 [
639 self._ljust("-" * header_colwidth, gross_colwidth)
640 for header_colwidth, gross_colwidth in zip(
641 self.header_column_widths, self.gross_column_widths, strict=True
642 )
643 ]
644 )
645 self._lines.append(separator_line)
647 def add_body_lines(self) -> None:
648 for row in self.strrows:
649 body_line = self.SPACING.join(
650 [
651 self._ljust(col, gross_colwidth)
652 for col, gross_colwidth in zip(
653 row, self.gross_column_widths, strict=True
654 )
655 ]
656 )
657 self._lines.append(body_line)
659 def _gen_non_null_counts(self) -> Iterator[str]:
660 """Iterator with string representation of non-null counts."""
661 for count in self.non_null_counts:
662 yield f"{count} non-null"
664 def _gen_dtypes(self) -> Iterator[str]:
665 """Iterator with string representation of column dtypes."""
666 for dtype in self.dtypes:
667 yield pprint_thing(dtype)
670class _DataFrameTableBuilderVerbose(_DataFrameTableBuilder, _TableBuilderVerboseMixin):
671 """
672 Dataframe info table builder for verbose output.
673 """
675 def __init__(
676 self,
677 *,
678 info: DataFrameInfo,
679 with_counts: bool,
680 ) -> None:
681 self.info = info
682 self.with_counts = with_counts
683 self.strrows: Sequence[Sequence[str]] = list(self._gen_rows())
684 self.gross_column_widths: Sequence[int] = self._get_gross_column_widths()
686 def _fill_non_empty_info(self) -> None:
687 """Add lines to the info table, pertaining to non-empty dataframe."""
688 self.add_object_type_line()
689 self.add_index_range_line()
690 self.add_columns_summary_line()
691 self.add_header_line()
692 self.add_separator_line()
693 self.add_body_lines()
694 self.add_dtypes_line()
695 if self.display_memory_usage:
696 self.add_memory_usage_line()
698 @property
699 def headers(self) -> Sequence[str]:
700 """Headers names of the columns in verbose table."""
701 if self.with_counts:
702 return [" # ", "Column", "Non-Null Count", "Dtype"]
703 return [" # ", "Column", "Dtype"]
705 def add_columns_summary_line(self) -> None:
706 self._lines.append(f"Data columns (total {self.col_count} columns):")
708 def _gen_rows_without_counts(self) -> Iterator[Sequence[str]]:
709 """Iterator with string representation of body data without counts."""
710 yield from zip(
711 self._gen_line_numbers(),
712 self._gen_columns(),
713 self._gen_dtypes(),
714 strict=True,
715 )
717 def _gen_rows_with_counts(self) -> Iterator[Sequence[str]]:
718 """Iterator with string representation of body data with counts."""
719 yield from zip(
720 self._gen_line_numbers(),
721 self._gen_columns(),
722 self._gen_non_null_counts(),
723 self._gen_dtypes(),
724 strict=True,
725 )
727 def _gen_line_numbers(self) -> Iterator[str]:
728 """Iterator with string representation of column numbers."""
729 for i, _ in enumerate(self.ids):
730 yield f" {i}"
732 def _gen_columns(self) -> Iterator[str]:
733 """Iterator with string representation of column names."""
734 for col in self.ids:
735 yield pprint_thing(col)
738class _SeriesTableBuilder(_TableBuilderAbstract):
739 """
740 Abstract builder for series info table.
742 Parameters
743 ----------
744 info : SeriesInfo.
745 Instance of SeriesInfo.
746 """
748 def __init__(self, *, info: SeriesInfo) -> None:
749 self.info: SeriesInfo = info
751 def get_lines(self) -> list[str]:
752 self._lines = []
753 self._fill_non_empty_info()
754 return self._lines
756 @property
757 def data(self) -> Series:
758 """Series."""
759 return self.info.data
761 def add_memory_usage_line(self) -> None:
762 """Add line containing memory usage."""
763 self._lines.append(f"memory usage: {self.memory_usage_string}")
765 @abstractmethod
766 def _fill_non_empty_info(self) -> None:
767 """Add lines to the info table, pertaining to non-empty series."""
770class _SeriesTableBuilderNonVerbose(_SeriesTableBuilder):
771 """
772 Series info table builder for non-verbose output.
773 """
775 def _fill_non_empty_info(self) -> None:
776 """Add lines to the info table, pertaining to non-empty series."""
777 self.add_object_type_line()
778 self.add_index_range_line()
779 self.add_dtypes_line()
780 if self.display_memory_usage:
781 self.add_memory_usage_line()
784class _SeriesTableBuilderVerbose(_SeriesTableBuilder, _TableBuilderVerboseMixin):
785 """
786 Series info table builder for verbose output.
787 """
789 def __init__(
790 self,
791 *,
792 info: SeriesInfo,
793 with_counts: bool,
794 ) -> None:
795 self.info = info
796 self.with_counts = with_counts
797 self.strrows: Sequence[Sequence[str]] = list(self._gen_rows())
798 self.gross_column_widths: Sequence[int] = self._get_gross_column_widths()
800 def _fill_non_empty_info(self) -> None:
801 """Add lines to the info table, pertaining to non-empty series."""
802 self.add_object_type_line()
803 self.add_index_range_line()
804 self.add_series_name_line()
805 self.add_header_line()
806 self.add_separator_line()
807 self.add_body_lines()
808 self.add_dtypes_line()
809 if self.display_memory_usage:
810 self.add_memory_usage_line()
812 def add_series_name_line(self) -> None:
813 self._lines.append(f"Series name: {self.data.name}")
815 @property
816 def headers(self) -> Sequence[str]:
817 """Headers names of the columns in verbose table."""
818 if self.with_counts:
819 return ["Non-Null Count", "Dtype"]
820 return ["Dtype"]
822 def _gen_rows_without_counts(self) -> Iterator[Sequence[str]]:
823 """Iterator with string representation of body data without counts."""
824 yield from ([dtype] for dtype in self._gen_dtypes())
826 def _gen_rows_with_counts(self) -> Iterator[Sequence[str]]:
827 """Iterator with string representation of body data with counts."""
828 yield from zip(self._gen_non_null_counts(), self._gen_dtypes(), strict=True)
831def _get_dataframe_dtype_counts(df: DataFrame) -> Mapping[str, int]:
832 """
833 Create mapping between datatypes and their number of occurrences.
834 """
835 # groupby dtype.name to collect e.g. Categorical columns
836 return df.dtypes.value_counts().groupby(lambda x: x.name).sum()