Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Module responsible for execution of NDFrame.describe() method.
4Method NDFrame.describe() delegates actual execution to function describe_ndframe().
5"""
7from __future__ import annotations
9from abc import (
10 ABC,
11 abstractmethod,
12)
13from typing import (
14 TYPE_CHECKING,
15 cast,
16)
17import warnings
19import numpy as np
21from pandas.errors import Pandas4Warning
22from pandas.util._exceptions import find_stack_level
23from pandas.util._validators import validate_percentile
25from pandas.core.dtypes.common import (
26 is_bool_dtype,
27 is_numeric_dtype,
28)
29from pandas.core.dtypes.dtypes import (
30 ArrowDtype,
31 DatetimeTZDtype,
32 ExtensionDtype,
33)
35from pandas.core.arrays.floating import Float64Dtype
36from pandas.core.reshape.concat import concat
38from pandas.io.formats.format import format_percentiles
40if TYPE_CHECKING:
41 from collections.abc import (
42 Callable,
43 Hashable,
44 Sequence,
45 )
47 from pandas._typing import (
48 DtypeObj,
49 NDFrameT,
50 npt,
51 )
53 from pandas import (
54 DataFrame,
55 Series,
56 )
59def describe_ndframe(
60 *,
61 obj: NDFrameT,
62 include: str | Sequence[str] | None,
63 exclude: str | Sequence[str] | None,
64 percentiles: Sequence[float] | np.ndarray | None,
65) -> NDFrameT:
66 """Describe series or dataframe.
68 Called from pandas.core.generic.NDFrame.describe()
70 Parameters
71 ----------
72 obj: DataFrame or Series
73 Either dataframe or series to be described.
74 include : 'all', list-like of dtypes or None (default), optional
75 A white list of data types to include in the result. Ignored for ``Series``.
76 exclude : list-like of dtypes or None (default), optional,
77 A black list of data types to omit from the result. Ignored for ``Series``.
78 percentiles : list-like of numbers, optional
79 The percentiles to include in the output. All should fall between 0 and 1.
80 The default is ``[.25, .5, .75]``, which returns the 25th, 50th, and
81 75th percentiles.
83 Returns
84 -------
85 Dataframe or series description.
86 """
87 percentiles = _refine_percentiles(percentiles)
89 describer: NDFrameDescriberAbstract
90 if obj.ndim == 1:
91 if include is not None or exclude is not None:
92 # GH#54193
93 warnings.warn(
94 "The 'include' and 'exclude' arguments are deprecated for "
95 "Series.describe and will be removed in a future version. "
96 "These arguments have no effect on Series and will be removed.",
97 Pandas4Warning,
98 stacklevel=find_stack_level(),
99 )
100 describer = SeriesDescriber(
101 obj=cast("Series", obj),
102 )
103 else:
104 describer = DataFrameDescriber(
105 obj=cast("DataFrame", obj),
106 include=include,
107 exclude=exclude,
108 )
110 result = describer.describe(percentiles=percentiles)
111 return cast("NDFrameT", result)
114class NDFrameDescriberAbstract(ABC):
115 """Abstract class for describing dataframe or series.
117 Parameters
118 ----------
119 obj : Series or DataFrame
120 Object to be described.
121 """
123 def __init__(self, obj: DataFrame | Series) -> None:
124 self.obj = obj
126 @abstractmethod
127 def describe(self, percentiles: Sequence[float] | np.ndarray) -> DataFrame | Series:
128 """Do describe either series or dataframe.
130 Parameters
131 ----------
132 percentiles : list-like of numbers
133 The percentiles to include in the output.
134 """
137class SeriesDescriber(NDFrameDescriberAbstract):
138 """Class responsible for creating series description."""
140 obj: Series
142 def describe(self, percentiles: Sequence[float] | np.ndarray) -> Series:
143 describe_func = select_describe_func(
144 self.obj,
145 )
146 return describe_func(self.obj, percentiles)
149class DataFrameDescriber(NDFrameDescriberAbstract):
150 """Class responsible for creating dataobj description.
152 Parameters
153 ----------
154 obj : DataFrame
155 DataFrame to be described.
156 include : 'all', list-like of dtypes or None
157 A white list of data types to include in the result.
158 exclude : list-like of dtypes or None
159 A black list of data types to omit from the result.
160 """
162 obj: DataFrame
164 def __init__(
165 self,
166 obj: DataFrame,
167 *,
168 include: str | Sequence[str] | None,
169 exclude: str | Sequence[str] | None,
170 ) -> None:
171 self.include = include
172 self.exclude = exclude
174 if obj.ndim == 2 and obj.columns.size == 0:
175 raise ValueError("Cannot describe a DataFrame without columns")
177 super().__init__(obj)
179 def describe(self, percentiles: Sequence[float] | np.ndarray) -> DataFrame:
180 data = self._select_data()
182 ldesc: list[Series] = []
183 for _, series in data.items():
184 describe_func = select_describe_func(series)
185 ldesc.append(describe_func(series, percentiles))
187 col_names = reorder_columns(ldesc)
188 d = concat(ldesc, axis=1, ignore_index=True, sort=False).reindex(col_names)
189 d.columns = data.columns.copy()
190 return d
192 def _select_data(self) -> DataFrame:
193 """Select columns to be described."""
194 if (self.include is None) and (self.exclude is None):
195 # when some numerics are found, keep only numerics
196 default_include: list[npt.DTypeLike] = [np.number, "datetime"]
197 data = self.obj.select_dtypes(include=default_include)
198 if len(data.columns) == 0:
199 data = self.obj
200 elif isinstance(self.include, str) and self.include == "all":
201 # isinstance guard so an array-like spec is not compared elementwise
202 if self.exclude is not None:
203 msg = "exclude must be None when include is 'all'"
204 raise ValueError(msg)
205 data = self.obj
206 else:
207 data = self.obj.select_dtypes(
208 include=self.include,
209 exclude=self.exclude,
210 )
211 if len(data.columns) == 0:
212 msg = "No columns match the specified include or exclude data types"
213 raise ValueError(msg)
214 return data
217def reorder_columns(ldesc: Sequence[Series]) -> list[Hashable]:
218 """Set a convenient order for rows for display."""
219 names: list[Hashable] = []
220 seen_names: set[Hashable] = set()
221 ldesc_indexes = sorted((x.index for x in ldesc), key=len)
222 for idxnames in ldesc_indexes:
223 for name in idxnames:
224 if name not in seen_names:
225 seen_names.add(name)
226 names.append(name)
227 return names
230def describe_numeric_1d(series: Series, percentiles: Sequence[float]) -> Series:
231 """Describe series containing numerical data.
233 Parameters
234 ----------
235 series : Series
236 Series to be described.
237 percentiles : list-like of numbers
238 The percentiles to include in the output.
239 """
240 from pandas import Series
242 formatted_percentiles = format_percentiles(percentiles)
244 if len(percentiles) == 0:
245 quantiles = []
246 else:
247 quantiles = series.quantile(percentiles).tolist()
249 stat_index = ["count", "mean", "std", "min", *formatted_percentiles, "max"]
250 d = [
251 series.count(),
252 series.mean(),
253 series.std(),
254 series.min(),
255 *quantiles,
256 series.max(),
257 ]
258 # GH#48340 - always return float on non-complex numeric data
259 dtype: DtypeObj | None
260 if isinstance(series.dtype, ExtensionDtype):
261 if isinstance(series.dtype, ArrowDtype):
262 if series.dtype.kind == "m":
263 # GH53001: describe timedeltas with object dtype
264 dtype = None
265 else:
266 import pyarrow as pa
268 dtype = ArrowDtype(pa.float64())
269 else:
270 dtype = Float64Dtype()
271 elif series.dtype.kind in "iufb":
272 # i.e. numeric but exclude complex dtype
273 dtype = np.dtype("float")
274 else:
275 dtype = None
276 return Series(d, index=stat_index, name=series.name, dtype=dtype)
279def describe_categorical_1d(
280 data: Series,
281 percentiles_ignored: Sequence[float],
282) -> Series:
283 """Describe series containing categorical data.
285 Parameters
286 ----------
287 data : Series
288 Series to be described.
289 percentiles_ignored : list-like of numbers
290 Ignored, but in place to unify interface.
291 """
292 names = ["count", "unique", "top", "freq"]
293 objcounts = data.value_counts()
294 count_unique = len(objcounts[objcounts != 0])
295 if count_unique > 0:
296 top, freq = objcounts.index[0], objcounts.iloc[0]
297 dtype = None
298 else:
299 # If the DataFrame is empty, set 'top' and 'freq' to None
300 # to maintain output shape consistency
301 top, freq = np.nan, np.nan
302 dtype = "object"
304 result = [data.count(), count_unique, top, freq]
306 from pandas import Series
308 return Series(result, index=names, name=data.name, dtype=dtype)
311def describe_timestamp_1d(data: Series, percentiles: Sequence[float]) -> Series:
312 """Describe series containing datetime64 dtype.
314 Parameters
315 ----------
316 data : Series
317 Series to be described.
318 percentiles : list-like of numbers
319 The percentiles to include in the output.
320 """
321 # GH-30164
322 from pandas import Series
324 formatted_percentiles = format_percentiles(percentiles)
326 stat_index = ["count", "mean", "min", *formatted_percentiles, "max"]
327 d = [
328 data.count(),
329 data.mean(),
330 data.min(),
331 *data.quantile(percentiles).tolist(),
332 data.max(),
333 ]
334 return Series(d, index=stat_index, name=data.name)
337def select_describe_func(
338 data: Series,
339) -> Callable[..., Series]:
340 """Select proper function for describing series based on data type.
342 Parameters
343 ----------
344 data : Series
345 Series to be described.
346 """
347 if is_bool_dtype(data.dtype):
348 return describe_categorical_1d
349 elif is_numeric_dtype(data):
350 return describe_numeric_1d
351 elif data.dtype.kind == "M" or isinstance(data.dtype, DatetimeTZDtype):
352 return describe_timestamp_1d
353 elif data.dtype.kind == "m":
354 return describe_numeric_1d
355 else:
356 return describe_categorical_1d
359def _refine_percentiles(
360 percentiles: Sequence[float] | np.ndarray | None,
361) -> npt.NDArray[np.float64]:
362 """
363 Ensure that percentiles are unique and sorted.
365 Parameters
366 ----------
367 percentiles : list-like of numbers, optional
368 The percentiles to include in the output.
369 """
370 if percentiles is None:
371 return np.array([0.25, 0.5, 0.75])
373 percentiles = np.asarray(percentiles)
375 # get them all to be in [0, 1]
376 validate_percentile(percentiles)
378 # sort and check for duplicates
379 unique_pcts = np.unique(percentiles)
380 assert percentiles is not None
381 if len(unique_pcts) < len(percentiles):
382 raise ValueError("percentiles cannot contain duplicates")
384 return unique_pcts