Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Table Schema builders
4https://specs.frictionlessdata.io/table-schema/
5"""
7from __future__ import annotations
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 cast,
13)
14import warnings
16from pandas._config import option_context
18from pandas._libs import lib
19from pandas._libs.json import ujson_loads
20from pandas._libs.tslibs import timezones
21from pandas.util._exceptions import find_stack_level
23from pandas.core.dtypes.base import _registry as registry
24from pandas.core.dtypes.common import (
25 is_bool_dtype,
26 is_integer_dtype,
27 is_numeric_dtype,
28 is_string_dtype,
29)
30from pandas.core.dtypes.dtypes import (
31 CategoricalDtype,
32 DatetimeTZDtype,
33 ExtensionDtype,
34 PeriodDtype,
35)
37from pandas import DataFrame
38import pandas.core.common as com
40from pandas.tseries.frequencies import to_offset
42if TYPE_CHECKING:
43 from pandas._typing import (
44 DtypeObj,
45 JSONSerializable,
46 )
48 from pandas import Series
49 from pandas.core.indexes.multi import MultiIndex
52TABLE_SCHEMA_VERSION = "1.4.0"
55def as_json_table_type(x: DtypeObj) -> str:
56 """
57 Convert a NumPy / pandas type to its corresponding json_table.
59 Parameters
60 ----------
61 x : np.dtype or ExtensionDtype
63 Returns
64 -------
65 str
66 the Table Schema data types
68 Notes
69 -----
70 This table shows the relationship between NumPy / pandas dtypes,
71 and Table Schema dtypes.
73 ============== =================
74 Pandas type Table Schema type
75 ============== =================
76 int64 integer
77 float64 number
78 bool boolean
79 datetime64[ns] datetime
80 timedelta64[ns] duration
81 object str
82 categorical any
83 =============== =================
84 """
85 if is_integer_dtype(x):
86 return "integer"
87 elif is_bool_dtype(x):
88 return "boolean"
89 elif is_numeric_dtype(x):
90 return "number"
91 elif lib.is_np_dtype(x, "M") or isinstance(x, (DatetimeTZDtype, PeriodDtype)):
92 return "datetime"
93 elif lib.is_np_dtype(x, "m"):
94 return "duration"
95 elif is_string_dtype(x):
96 return "string"
97 else:
98 return "any"
101def set_default_names(data):
102 """Sets index names to 'index' for regular, or 'level_x' for Multi"""
103 if com.all_not_none(*data.index.names):
104 nms = data.index.names
105 if len(nms) == 1 and data.index.name == "index":
106 warnings.warn(
107 "Index name of 'index' is not round-trippable.",
108 stacklevel=find_stack_level(),
109 )
110 elif len(nms) > 1 and any(x.startswith("level_") for x in nms):
111 warnings.warn(
112 "Index names beginning with 'level_' are not round-trippable.",
113 stacklevel=find_stack_level(),
114 )
115 return data
117 data = data.copy(deep=False)
118 if data.index.nlevels > 1:
119 data.index.names = com.fill_missing_names(data.index.names)
120 else:
121 data.index.name = data.index.name or "index"
122 return data
125def convert_pandas_type_to_json_field(arr) -> dict[str, JSONSerializable]:
126 dtype = arr.dtype
127 name: JSONSerializable
128 if arr.name is None:
129 name = "values"
130 else:
131 name = arr.name
132 field: dict[str, JSONSerializable] = {
133 "name": name,
134 "type": as_json_table_type(dtype),
135 }
137 if isinstance(dtype, CategoricalDtype):
138 cats = dtype.categories
139 ordered = dtype.ordered
141 field["constraints"] = {"enum": list(cats)}
142 field["ordered"] = ordered
143 elif isinstance(dtype, PeriodDtype):
144 field["freq"] = dtype.freq.freqstr
145 elif isinstance(dtype, DatetimeTZDtype):
146 if timezones.is_utc(dtype.tz):
147 field["tz"] = "UTC"
148 else:
149 zone = timezones.get_timezone(dtype.tz)
150 if isinstance(zone, str):
151 field["tz"] = zone
152 elif isinstance(dtype, ExtensionDtype):
153 field["extDtype"] = dtype.name
154 return field
157def convert_json_field_to_pandas_type(field) -> str | CategoricalDtype:
158 """
159 Converts a JSON field descriptor into its corresponding NumPy / pandas type
161 Parameters
162 ----------
163 field
164 A JSON field descriptor
166 Returns
167 -------
168 dtype
170 Raises
171 ------
172 ValueError
173 If the type of the provided field is unknown or currently unsupported
175 Examples
176 --------
177 >>> convert_json_field_to_pandas_type({"name": "an_int", "type": "integer"})
178 'int64'
180 >>> convert_json_field_to_pandas_type(
181 ... {
182 ... "name": "a_categorical",
183 ... "type": "any",
184 ... "constraints": {"enum": ["a", "b", "c"]},
185 ... "ordered": True,
186 ... }
187 ... )
188 CategoricalDtype(categories=['a', 'b', 'c'], ordered=True, categories_dtype=str)
190 >>> convert_json_field_to_pandas_type({"name": "a_datetime", "type": "datetime"})
191 'datetime64[ns]'
193 >>> convert_json_field_to_pandas_type(
194 ... {"name": "a_datetime_with_tz", "type": "datetime", "tz": "US/Central"}
195 ... )
196 'datetime64[ns, US/Central]'
197 """
198 typ = field["type"]
199 if typ == "string":
200 return field.get("extDtype", None)
201 elif typ == "integer":
202 return field.get("extDtype", "int64")
203 elif typ == "number":
204 return field.get("extDtype", "float64")
205 elif typ == "boolean":
206 return field.get("extDtype", "bool")
207 elif typ == "duration":
208 return "timedelta64"
209 elif typ == "datetime":
210 if field.get("tz"):
211 return f"datetime64[ns, {field['tz']}]"
212 elif field.get("freq"):
213 # GH#9586 rename frequency M to ME for offsets
214 offset = to_offset(field["freq"])
215 freq = PeriodDtype(offset)._freqstr
216 # GH#47747 using datetime over period to minimize the change surface
217 return f"period[{freq}]"
218 else:
219 return "datetime64[ns]"
220 elif typ == "any":
221 if "constraints" in field and "ordered" in field:
222 return CategoricalDtype(
223 categories=field["constraints"]["enum"], ordered=field["ordered"]
224 )
225 elif "extDtype" in field:
226 return registry.find(field["extDtype"])
227 else:
228 return "object"
230 raise ValueError(f"Unsupported or invalid field type: {typ}")
233def build_table_schema(
234 data: DataFrame | Series,
235 index: bool = True,
236 primary_key: bool | None = None,
237 version: bool = True,
238) -> dict[str, JSONSerializable]:
239 """
240 Create a Table schema from ``data``.
242 This method is a utility to generate a JSON-serializable schema
243 representation of a pandas Series or DataFrame, compatible with the
244 Table Schema specification. It enables structured data to be shared
245 and validated in various applications, ensuring consistency and
246 interoperability.
248 Parameters
249 ----------
250 data : Series or DataFrame
251 The input data for which the table schema is to be created.
252 index : bool, default True
253 Whether to include ``data.index`` in the schema.
254 primary_key : bool or None, default True
255 Column names to designate as the primary key.
256 The default `None` will set `'primaryKey'` to the index
257 level or levels if the index is unique.
258 version : bool, default True
259 Whether to include a field `pandas_version` with the version
260 of pandas that last revised the table schema. This version
261 can be different from the installed pandas version.
263 Returns
264 -------
265 dict
266 A dictionary representing the Table schema.
268 See Also
269 --------
270 DataFrame.to_json : Convert the object to a JSON string.
271 read_json : Convert a JSON string to pandas object.
273 Notes
274 -----
275 See `Table Schema
276 <https://pandas.pydata.org/docs/user_guide/io.html#table-schema>`__ for
277 conversion types.
278 Timedeltas as converted to ISO8601 duration format with
279 9 decimal places after the seconds field for nanosecond precision.
281 Categoricals are converted to the `any` dtype, and use the `enum` field
282 constraint to list the allowed values. The `ordered` attribute is included
283 in an `ordered` field.
285 Examples
286 --------
287 >>> from pandas.io.json._table_schema import build_table_schema
288 >>> df = pd.DataFrame(
289 ... {'A': [1, 2, 3],
290 ... 'B': ['a', 'b', 'c'],
291 ... 'C': pd.date_range('2016-01-01', freq='D', periods=3),
292 ... }, index=pd.Index(range(3), name='idx'))
293 >>> build_table_schema(df)
294 {'fields': \
295[{'name': 'idx', 'type': 'integer'}, \
296{'name': 'A', 'type': 'integer'}, \
297{'name': 'B', 'type': 'string', 'extDtype': 'str'}, \
298{'name': 'C', 'type': 'datetime'}], \
299'primaryKey': ['idx'], \
300'pandas_version': '1.4.0'}
301 """
302 if index is True:
303 data = set_default_names(data)
305 schema: dict[str, Any] = {}
306 fields = []
308 if index:
309 if data.index.nlevels > 1:
310 data.index = cast("MultiIndex", data.index)
311 for level, name in zip(data.index.levels, data.index.names, strict=True):
312 new_field = convert_pandas_type_to_json_field(level)
313 new_field["name"] = name
314 fields.append(new_field)
315 else:
316 fields.append(convert_pandas_type_to_json_field(data.index))
318 if data.ndim > 1:
319 for column, s in data.items():
320 fields.append(convert_pandas_type_to_json_field(s))
321 else:
322 fields.append(convert_pandas_type_to_json_field(data))
324 schema["fields"] = fields
325 if index and data.index.is_unique and primary_key is None:
326 if data.index.nlevels == 1:
327 schema["primaryKey"] = [data.index.name]
328 else:
329 schema["primaryKey"] = data.index.names
330 elif primary_key is not None:
331 schema["primaryKey"] = primary_key
333 if version:
334 schema["pandas_version"] = TABLE_SCHEMA_VERSION
335 return schema
338def parse_table_schema(json, precise_float: bool) -> DataFrame:
339 """
340 Builds a DataFrame from a given schema
342 Parameters
343 ----------
344 json :
345 A JSON table schema
346 precise_float : bool
347 Flag controlling precision when decoding string to double values, as
348 dictated by ``read_json``
350 Returns
351 -------
352 df : DataFrame
354 Raises
355 ------
356 NotImplementedError
357 If the JSON table schema contains either timezone or timedelta data
359 Notes
360 -----
361 Because :func:`DataFrame.to_json` uses the string 'index' to denote a
362 name-less :class:`Index`, this function sets the name of the returned
363 :class:`DataFrame` to ``None`` when said string is encountered with a
364 normal :class:`Index`. For a :class:`MultiIndex`, the same limitation
365 applies to any strings beginning with 'level_'. Therefore, an
366 :class:`Index` name of 'index' and :class:`MultiIndex` names starting
367 with 'level_' are not supported.
369 See Also
370 --------
371 build_table_schema : Inverse function.
372 pandas.read_json
373 """
374 table = ujson_loads(json, precise_float=precise_float)
375 col_order = [field["name"] for field in table["schema"]["fields"]]
376 df = DataFrame(table["data"], columns=col_order)[col_order]
378 dtypes = {
379 field["name"]: convert_json_field_to_pandas_type(field)
380 for field in table["schema"]["fields"]
381 }
383 # No ISO constructor for Timedelta as of yet, so need to raise
384 if "timedelta64" in dtypes.values():
385 raise NotImplementedError(
386 'table="orient" can not yet read ISO-formatted Timedelta data'
387 )
389 with option_context("future.distinguish_nan_and_na", False):
390 df = df.astype(dtypes)
392 if "primaryKey" in table["schema"]:
393 df = df.set_index(table["schema"]["primaryKey"])
394 if len(df.index.names) == 1:
395 if df.index.name == "index":
396 df.index.name = None
397 else:
398 df.index.names = [
399 None if x.startswith("level_") else x for x in df.index.names
400 ]
402 return df