Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Module responsible for execution of NDFrame.describe() method.
4Method NDFrame.describe() delegates actual execution to function describe_ndframe().
5"""
7from __future__ import annotations
9from abc import (
10 ABC,
11 abstractmethod,
12)
13from typing import (
14 TYPE_CHECKING,
15 cast,
16)
18import numpy as np
20from pandas._typing import (
21 DtypeObj,
22 NDFrameT,
23 npt,
24)
25from pandas.util._validators import validate_percentile
27from pandas.core.dtypes.common import (
28 is_bool_dtype,
29 is_numeric_dtype,
30)
31from pandas.core.dtypes.dtypes import (
32 ArrowDtype,
33 DatetimeTZDtype,
34 ExtensionDtype,
35)
37from pandas.core.arrays.floating import Float64Dtype
38from pandas.core.reshape.concat import concat
40from pandas.io.formats.format import format_percentiles
42if TYPE_CHECKING:
43 from collections.abc import (
44 Callable,
45 Hashable,
46 Sequence,
47 )
49 from pandas import (
50 DataFrame,
51 Series,
52 )
55def describe_ndframe(
56 *,
57 obj: NDFrameT,
58 include: str | Sequence[str] | None,
59 exclude: str | Sequence[str] | None,
60 percentiles: Sequence[float] | np.ndarray | None,
61) -> NDFrameT:
62 """Describe series or dataframe.
64 Called from pandas.core.generic.NDFrame.describe()
66 Parameters
67 ----------
68 obj: DataFrame or Series
69 Either dataframe or series to be described.
70 include : 'all', list-like of dtypes or None (default), optional
71 A white list of data types to include in the result. Ignored for ``Series``.
72 exclude : list-like of dtypes or None (default), optional,
73 A black list of data types to omit from the result. Ignored for ``Series``.
74 percentiles : list-like of numbers, optional
75 The percentiles to include in the output. All should fall between 0 and 1.
76 The default is ``[.25, .5, .75]``, which returns the 25th, 50th, and
77 75th percentiles.
79 Returns
80 -------
81 Dataframe or series description.
82 """
83 percentiles = _refine_percentiles(percentiles)
85 describer: NDFrameDescriberAbstract
86 if obj.ndim == 1:
87 describer = SeriesDescriber(
88 obj=cast("Series", obj),
89 )
90 else:
91 describer = DataFrameDescriber(
92 obj=cast("DataFrame", obj),
93 include=include,
94 exclude=exclude,
95 )
97 result = describer.describe(percentiles=percentiles)
98 return cast(NDFrameT, result)
101class NDFrameDescriberAbstract(ABC):
102 """Abstract class for describing dataframe or series.
104 Parameters
105 ----------
106 obj : Series or DataFrame
107 Object to be described.
108 """
110 def __init__(self, obj: DataFrame | Series) -> None:
111 self.obj = obj
113 @abstractmethod
114 def describe(self, percentiles: Sequence[float] | np.ndarray) -> DataFrame | Series:
115 """Do describe either series or dataframe.
117 Parameters
118 ----------
119 percentiles : list-like of numbers
120 The percentiles to include in the output.
121 """
124class SeriesDescriber(NDFrameDescriberAbstract):
125 """Class responsible for creating series description."""
127 obj: Series
129 def describe(self, percentiles: Sequence[float] | np.ndarray) -> Series:
130 describe_func = select_describe_func(
131 self.obj,
132 )
133 return describe_func(self.obj, percentiles)
136class DataFrameDescriber(NDFrameDescriberAbstract):
137 """Class responsible for creating dataobj description.
139 Parameters
140 ----------
141 obj : DataFrame
142 DataFrame to be described.
143 include : 'all', list-like of dtypes or None
144 A white list of data types to include in the result.
145 exclude : list-like of dtypes or None
146 A black list of data types to omit from the result.
147 """
149 obj: DataFrame
151 def __init__(
152 self,
153 obj: DataFrame,
154 *,
155 include: str | Sequence[str] | None,
156 exclude: str | Sequence[str] | None,
157 ) -> None:
158 self.include = include
159 self.exclude = exclude
161 if obj.ndim == 2 and obj.columns.size == 0:
162 raise ValueError("Cannot describe a DataFrame without columns")
164 super().__init__(obj)
166 def describe(self, percentiles: Sequence[float] | np.ndarray) -> DataFrame:
167 data = self._select_data()
169 ldesc: list[Series] = []
170 for _, series in data.items():
171 describe_func = select_describe_func(series)
172 ldesc.append(describe_func(series, percentiles))
174 col_names = reorder_columns(ldesc)
175 d = concat(
176 [x.reindex(col_names) for x in ldesc],
177 axis=1,
178 ignore_index=True,
179 sort=False,
180 )
181 d.columns = data.columns.copy()
182 return d
184 def _select_data(self) -> DataFrame:
185 """Select columns to be described."""
186 if (self.include is None) and (self.exclude is None):
187 # when some numerics are found, keep only numerics
188 default_include: list[npt.DTypeLike] = [np.number, "datetime"]
189 data = self.obj.select_dtypes(include=default_include)
190 if len(data.columns) == 0:
191 data = self.obj
192 elif self.include == "all":
193 if self.exclude is not None:
194 msg = "exclude must be None when include is 'all'"
195 raise ValueError(msg)
196 data = self.obj
197 else:
198 data = self.obj.select_dtypes(
199 include=self.include,
200 exclude=self.exclude,
201 )
202 if len(data.columns) == 0:
203 msg = "No columns match the specified include or exclude data types"
204 raise ValueError(msg)
205 return data
208def reorder_columns(ldesc: Sequence[Series]) -> list[Hashable]:
209 """Set a convenient order for rows for display."""
210 names: list[Hashable] = []
211 seen_names: set[Hashable] = set()
212 ldesc_indexes = sorted((x.index for x in ldesc), key=len)
213 for idxnames in ldesc_indexes:
214 for name in idxnames:
215 if name not in seen_names:
216 seen_names.add(name)
217 names.append(name)
218 return names
221def describe_numeric_1d(series: Series, percentiles: Sequence[float]) -> Series:
222 """Describe series containing numerical data.
224 Parameters
225 ----------
226 series : Series
227 Series to be described.
228 percentiles : list-like of numbers
229 The percentiles to include in the output.
230 """
231 from pandas import Series
233 formatted_percentiles = format_percentiles(percentiles)
235 if len(percentiles) == 0:
236 quantiles = []
237 else:
238 quantiles = series.quantile(percentiles).tolist()
240 stat_index = ["count", "mean", "std", "min", *formatted_percentiles, "max"]
241 d = [
242 series.count(),
243 series.mean(),
244 series.std(),
245 series.min(),
246 *quantiles,
247 series.max(),
248 ]
249 # GH#48340 - always return float on non-complex numeric data
250 dtype: DtypeObj | None
251 if isinstance(series.dtype, ExtensionDtype):
252 if isinstance(series.dtype, ArrowDtype):
253 if series.dtype.kind == "m":
254 # GH53001: describe timedeltas with object dtype
255 dtype = None
256 else:
257 import pyarrow as pa
259 dtype = ArrowDtype(pa.float64())
260 else:
261 dtype = Float64Dtype()
262 elif series.dtype.kind in "iufb":
263 # i.e. numeric but exclude complex dtype
264 dtype = np.dtype("float")
265 else:
266 dtype = None
267 return Series(d, index=stat_index, name=series.name, dtype=dtype)
270def describe_categorical_1d(
271 data: Series,
272 percentiles_ignored: Sequence[float],
273) -> Series:
274 """Describe series containing categorical data.
276 Parameters
277 ----------
278 data : Series
279 Series to be described.
280 percentiles_ignored : list-like of numbers
281 Ignored, but in place to unify interface.
282 """
283 names = ["count", "unique", "top", "freq"]
284 objcounts = data.value_counts()
285 count_unique = len(objcounts[objcounts != 0])
286 if count_unique > 0:
287 top, freq = objcounts.index[0], objcounts.iloc[0]
288 dtype = None
289 else:
290 # If the DataFrame is empty, set 'top' and 'freq' to None
291 # to maintain output shape consistency
292 top, freq = np.nan, np.nan
293 dtype = "object"
295 result = [data.count(), count_unique, top, freq]
297 from pandas import Series
299 return Series(result, index=names, name=data.name, dtype=dtype)
302def describe_timestamp_1d(data: Series, percentiles: Sequence[float]) -> Series:
303 """Describe series containing datetime64 dtype.
305 Parameters
306 ----------
307 data : Series
308 Series to be described.
309 percentiles : list-like of numbers
310 The percentiles to include in the output.
311 """
312 # GH-30164
313 from pandas import Series
315 formatted_percentiles = format_percentiles(percentiles)
317 stat_index = ["count", "mean", "min", *formatted_percentiles, "max"]
318 d = [
319 data.count(),
320 data.mean(),
321 data.min(),
322 *data.quantile(percentiles).tolist(),
323 data.max(),
324 ]
325 return Series(d, index=stat_index, name=data.name)
328def select_describe_func(
329 data: Series,
330) -> Callable:
331 """Select proper function for describing series based on data type.
333 Parameters
334 ----------
335 data : Series
336 Series to be described.
337 """
338 if is_bool_dtype(data.dtype):
339 return describe_categorical_1d
340 elif is_numeric_dtype(data):
341 return describe_numeric_1d
342 elif data.dtype.kind == "M" or isinstance(data.dtype, DatetimeTZDtype):
343 return describe_timestamp_1d
344 elif data.dtype.kind == "m":
345 return describe_numeric_1d
346 else:
347 return describe_categorical_1d
350def _refine_percentiles(
351 percentiles: Sequence[float] | np.ndarray | None,
352) -> npt.NDArray[np.float64]:
353 """
354 Ensure that percentiles are unique and sorted.
356 Parameters
357 ----------
358 percentiles : list-like of numbers, optional
359 The percentiles to include in the output.
360 """
361 if percentiles is None:
362 return np.array([0.25, 0.5, 0.75])
364 percentiles = np.asarray(percentiles)
366 # get them all to be in [0, 1]
367 validate_percentile(percentiles)
369 # sort and check for duplicates
370 unique_pcts = np.unique(percentiles)
371 assert percentiles is not None
372 if len(unique_pcts) < len(percentiles):
373 raise ValueError("percentiles cannot contain duplicates")
375 return unique_pcts