Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Table Schema builders
4https://specs.frictionlessdata.io/table-schema/
5"""
7from __future__ import annotations
9from datetime import timezone
10from typing import (
11 TYPE_CHECKING,
12 Any,
13 cast,
14)
15import warnings
17from pandas._config import option_context
19from pandas._libs import lib
20from pandas._libs._ujson import ujson_loads
21from pandas._libs.tslibs import timezones
22from pandas.util._exceptions import find_stack_level
24from pandas.core.dtypes.base import _registry as registry
25from pandas.core.dtypes.common import (
26 is_bool_dtype,
27 is_integer_dtype,
28 is_numeric_dtype,
29 is_string_dtype,
30)
31from pandas.core.dtypes.dtypes import (
32 CategoricalDtype,
33 DatetimeTZDtype,
34 ExtensionDtype,
35 PeriodDtype,
36)
38from pandas import DataFrame
39import pandas.core.common as com
41from pandas.tseries.frequencies import to_offset
43if TYPE_CHECKING:
44 from pandas._typing import (
45 DtypeObj,
46 JSONSerializable,
47 )
49 from pandas import Series
50 from pandas.core.indexes.multi import MultiIndex
53TABLE_SCHEMA_VERSION = "1.4.0"
56def as_json_table_type(x: DtypeObj) -> str:
57 """
58 Convert a NumPy / pandas type to its corresponding json_table.
60 Parameters
61 ----------
62 x : np.dtype or ExtensionDtype
64 Returns
65 -------
66 str
67 the Table Schema data types
69 Notes
70 -----
71 This table shows the relationship between NumPy / pandas dtypes,
72 and Table Schema dtypes.
74 ============== =================
75 Pandas type Table Schema type
76 ============== =================
77 int64 integer
78 float64 number
79 bool boolean
80 datetime64[ns] datetime
81 timedelta64[ns] duration
82 object str
83 categorical any
84 =============== =================
85 """
86 if is_integer_dtype(x):
87 return "integer"
88 elif is_bool_dtype(x):
89 return "boolean"
90 elif is_numeric_dtype(x):
91 return "number"
92 elif lib.is_np_dtype(x, "M") or isinstance(x, (DatetimeTZDtype, PeriodDtype)):
93 return "datetime"
94 elif lib.is_np_dtype(x, "m"):
95 return "duration"
96 elif is_string_dtype(x):
97 return "string"
98 else:
99 return "any"
102def set_default_names(data):
103 """Sets index names to 'index' for regular, or 'level_x' for Multi"""
104 if com.all_not_none(*data.index.names):
105 nms = data.index.names
106 if len(nms) == 1 and data.index.name == "index":
107 warnings.warn(
108 "Index name of 'index' is not round-trippable.",
109 stacklevel=find_stack_level(),
110 )
111 elif len(nms) > 1 and any(x.startswith("level_") for x in nms):
112 warnings.warn(
113 "Index names beginning with 'level_' are not round-trippable.",
114 stacklevel=find_stack_level(),
115 )
116 return data
118 data = data.copy(deep=False)
119 if data.index.nlevels > 1:
120 data.index.names = com.fill_missing_names(data.index.names)
121 else:
122 data.index.name = data.index.name or "index"
123 return data
126def convert_pandas_type_to_json_field(arr) -> dict[str, JSONSerializable]:
127 dtype = arr.dtype
128 name: JSONSerializable
129 if arr.name is None:
130 name = "values"
131 else:
132 name = arr.name
133 field: dict[str, JSONSerializable] = {
134 "name": name,
135 "type": as_json_table_type(dtype),
136 }
138 if isinstance(dtype, CategoricalDtype):
139 cats = dtype.categories
140 ordered = dtype.ordered
142 field["constraints"] = {"enum": list(cats)}
143 field["ordered"] = ordered
144 elif isinstance(dtype, PeriodDtype):
145 field["freq"] = dtype.freq.freqstr
146 elif isinstance(dtype, DatetimeTZDtype):
147 if timezones.is_utc(dtype.tz):
148 field["tz"] = "UTC"
149 else:
150 zone = timezones.get_timezone(dtype.tz)
151 if isinstance(zone, str):
152 field["tz"] = zone
153 elif isinstance(zone, timezone):
154 # fixed-offset stdlib timezone, e.g. timezone(timedelta(hours=1));
155 # str gives a round-trippable "UTC+HH:MM" (GH#39537)
156 field["tz"] = str(zone)
157 elif isinstance(dtype, ExtensionDtype):
158 field["extDtype"] = dtype.name
159 return field
162def convert_json_field_to_pandas_type(field) -> str | CategoricalDtype:
163 """
164 Converts a JSON field descriptor into its corresponding NumPy / pandas type
166 Parameters
167 ----------
168 field
169 A JSON field descriptor
171 Returns
172 -------
173 dtype
175 Raises
176 ------
177 ValueError
178 If the type of the provided field is unknown or currently unsupported
180 Examples
181 --------
182 >>> convert_json_field_to_pandas_type({"name": "an_int", "type": "integer"})
183 'int64'
185 >>> convert_json_field_to_pandas_type(
186 ... {
187 ... "name": "a_categorical",
188 ... "type": "any",
189 ... "constraints": {"enum": ["a", "b", "c"]},
190 ... "ordered": True,
191 ... }
192 ... )
193 CategoricalDtype(categories=['a', 'b', 'c'], ordered=True, categories_dtype=str)
195 >>> convert_json_field_to_pandas_type({"name": "a_datetime", "type": "datetime"})
196 'datetime64[ns]'
198 >>> convert_json_field_to_pandas_type(
199 ... {"name": "a_datetime_with_tz", "type": "datetime", "tz": "US/Central"}
200 ... )
201 'datetime64[ns, US/Central]'
202 """
203 typ = field["type"]
204 if typ == "string":
205 return field.get("extDtype", None)
206 elif typ == "integer":
207 return field.get("extDtype", "int64")
208 elif typ == "number":
209 return field.get("extDtype", "float64")
210 elif typ == "boolean":
211 return field.get("extDtype", "bool")
212 elif typ == "duration":
213 return "timedelta64"
214 elif typ == "datetime":
215 if field.get("tz"):
216 return f"datetime64[ns, {field['tz']}]"
217 elif field.get("freq"):
218 # GH#9586 rename frequency M to ME for offsets
219 offset = to_offset(field["freq"])
220 freq = PeriodDtype(offset)._freqstr
221 # GH#47747 using datetime over period to minimize the change surface
222 return f"period[{freq}]"
223 else:
224 return "datetime64[ns]"
225 elif typ == "any":
226 if "constraints" in field and "ordered" in field:
227 return CategoricalDtype(
228 categories=field["constraints"]["enum"], ordered=field["ordered"]
229 )
230 elif "extDtype" in field:
231 return registry.find(field["extDtype"])
232 else:
233 return "object"
235 raise ValueError(f"Unsupported or invalid field type: {typ}")
238def build_table_schema(
239 data: DataFrame | Series,
240 index: bool = True,
241 primary_key: bool | None = None,
242 version: bool = True,
243) -> dict[str, JSONSerializable]:
244 """
245 Create a Table schema from ``data``.
247 This method is a utility to generate a JSON-serializable schema
248 representation of a pandas Series or DataFrame, compatible with the
249 Table Schema specification. It enables structured data to be shared
250 and validated in various applications, ensuring consistency and
251 interoperability.
253 Parameters
254 ----------
255 data : Series or DataFrame
256 The input data for which the table schema is to be created.
257 index : bool, default True
258 Whether to include ``data.index`` in the schema.
259 primary_key : bool or None, default True
260 Column names to designate as the primary key.
261 The default `None` will set `'primaryKey'` to the index
262 level or levels if the index is unique.
263 version : bool, default True
264 Whether to include a field `pandas_version` with the version
265 of pandas that last revised the table schema. This version
266 can be different from the installed pandas version.
268 Returns
269 -------
270 dict
271 A dictionary representing the Table schema.
273 See Also
274 --------
275 DataFrame.to_json : Convert the object to a JSON string.
276 read_json : Convert a JSON string to pandas object.
278 Notes
279 -----
280 See `Table Schema
281 <https://pandas.pydata.org/docs/user_guide/io.html#table-schema>`__ for
282 conversion types.
283 Timedeltas as converted to ISO8601 duration format with
284 9 decimal places after the seconds field for nanosecond precision.
286 Categoricals are converted to the `any` dtype, and use the `enum` field
287 constraint to list the allowed values. The `ordered` attribute is included
288 in an `ordered` field.
290 Examples
291 --------
292 >>> from pandas.io.json._table_schema import build_table_schema
293 >>> df = pd.DataFrame(
294 ... {'A': [1, 2, 3],
295 ... 'B': ['a', 'b', 'c'],
296 ... 'C': pd.date_range('2016-01-01', freq='D', periods=3),
297 ... }, index=pd.Index(range(3), name='idx'))
298 >>> build_table_schema(df)
299 {'fields': \
300[{'name': 'idx', 'type': 'integer'}, \
301{'name': 'A', 'type': 'integer'}, \
302{'name': 'B', 'type': 'string', 'extDtype': 'str'}, \
303{'name': 'C', 'type': 'datetime'}], \
304'primaryKey': ['idx'], \
305'pandas_version': '1.4.0'}
306 """
307 if index is True:
308 data = set_default_names(data)
310 schema: dict[str, Any] = {}
311 fields = []
313 if index:
314 if data.index.nlevels > 1:
315 data.index = cast("MultiIndex", data.index)
316 for level, name in zip(data.index.levels, data.index.names, strict=True):
317 new_field = convert_pandas_type_to_json_field(level)
318 new_field["name"] = name
319 fields.append(new_field)
320 else:
321 fields.append(convert_pandas_type_to_json_field(data.index))
323 if data.ndim > 1:
324 for column, s in data.items():
325 fields.append(convert_pandas_type_to_json_field(s))
326 else:
327 fields.append(convert_pandas_type_to_json_field(data))
329 schema["fields"] = fields
330 if index and data.index.is_unique and primary_key is None:
331 if data.index.nlevels == 1:
332 schema["primaryKey"] = [data.index.name]
333 else:
334 schema["primaryKey"] = data.index.names
335 elif primary_key is not None:
336 schema["primaryKey"] = primary_key
338 if version:
339 schema["pandas_version"] = TABLE_SCHEMA_VERSION
340 return schema
343def parse_table_schema(json, precise_float: bool) -> DataFrame:
344 """
345 Builds a DataFrame from a given schema
347 Parameters
348 ----------
349 json :
350 A JSON table schema
351 precise_float : bool
352 Flag controlling precision when decoding string to double values, as
353 dictated by ``read_json``
355 Returns
356 -------
357 df : DataFrame
359 Raises
360 ------
361 NotImplementedError
362 If the JSON table schema contains either timezone or timedelta data
364 Notes
365 -----
366 Because :func:`DataFrame.to_json` uses the string 'index' to denote a
367 name-less :class:`Index`, this function sets the name of the returned
368 :class:`DataFrame` to ``None`` when said string is encountered with a
369 normal :class:`Index`. For a :class:`MultiIndex`, the same limitation
370 applies to any strings beginning with 'level_'. Therefore, an
371 :class:`Index` name of 'index' and :class:`MultiIndex` names starting
372 with 'level_' are not supported.
374 See Also
375 --------
376 build_table_schema : Inverse function.
377 pandas.read_json
378 """
379 table = ujson_loads(json, precise_float=precise_float)
380 col_order = [field["name"] for field in table["schema"]["fields"]]
381 df = DataFrame(table["data"], columns=col_order)[col_order]
383 dtypes = {
384 field["name"]: convert_json_field_to_pandas_type(field)
385 for field in table["schema"]["fields"]
386 }
388 # No ISO constructor for Timedelta as of yet, so need to raise
389 if "timedelta64" in dtypes.values():
390 raise NotImplementedError(
391 'table="orient" can not yet read ISO-formatted Timedelta data'
392 )
394 with option_context("future.distinguish_nan_and_na", False):
395 df = df.astype(dtypes)
397 if "primaryKey" in table["schema"]:
398 df = df.set_index(table["schema"]["primaryKey"])
399 if len(df.index.names) == 1:
400 if df.index.name == "index":
401 df.index.name = None
402 else:
403 df.index.names = [
404 None if x.startswith("level_") else x for x in df.index.names
405 ]
407 return df