Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/json/_json.py: 23%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from abc import (
4 ABC,
5 abstractmethod,
6)
7from collections import abc
8from itertools import islice
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 Generic,
13 Literal,
14 Self,
15 TypeVar,
16 final,
17 overload,
18)
19import warnings
21import numpy as np
23from pandas._config import option_context
25from pandas._libs import lib
26from pandas._libs.json import (
27 ujson_dumps,
28 ujson_loads,
29)
30from pandas._libs.tslibs import iNaT
31from pandas.compat._optional import import_optional_dependency
32from pandas.errors import (
33 AbstractMethodError,
34 OutOfBoundsDatetime,
35)
36from pandas.util._decorators import set_module
37from pandas.util._validators import check_dtype_backend
39from pandas.core.dtypes.common import (
40 ensure_str,
41 is_string_dtype,
42 pandas_dtype,
43)
44from pandas.core.dtypes.dtypes import PeriodDtype
46from pandas import (
47 ArrowDtype,
48 DataFrame,
49 Index,
50 MultiIndex,
51 Series,
52 isna,
53 notna,
54 to_datetime,
55)
56from pandas.core.reshape.concat import concat
58from pandas.io._util import arrow_table_to_pandas
59from pandas.io.common import (
60 IOHandles,
61 dedup_names,
62 get_handle,
63 is_potential_multi_index,
64 stringify_path,
65)
66from pandas.io.json._normalize import convert_to_line_delimits
67from pandas.io.json._table_schema import (
68 build_table_schema,
69 parse_table_schema,
70 set_default_names,
71)
72from pandas.io.parsers.readers import validate_integer
74if TYPE_CHECKING:
75 from collections.abc import (
76 Callable,
77 Hashable,
78 Mapping,
79 )
80 from types import TracebackType
82 from pandas._typing import (
83 CompressionOptions,
84 DtypeArg,
85 DtypeBackend,
86 FilePath,
87 IndexLabel,
88 JSONEngine,
89 JSONSerializable,
90 ReadBuffer,
91 StorageOptions,
92 WriteBuffer,
93 )
95 from pandas.core.generic import NDFrame
97FrameSeriesStrT = TypeVar("FrameSeriesStrT", bound=Literal["frame", "series"])
100# interface to/from
101@overload
102def to_json(
103 path_or_buf: FilePath | WriteBuffer[str] | WriteBuffer[bytes],
104 obj: NDFrame,
105 orient: str | None = ...,
106 date_format: str = ...,
107 double_precision: int = ...,
108 force_ascii: bool = ...,
109 date_unit: str = ...,
110 default_handler: Callable[[Any], JSONSerializable] | None = ...,
111 lines: bool = ...,
112 compression: CompressionOptions = ...,
113 index: bool | None = ...,
114 indent: int = ...,
115 storage_options: StorageOptions = ...,
116 mode: Literal["a", "w"] = ...,
117) -> None: ...
120@overload
121def to_json(
122 path_or_buf: None,
123 obj: NDFrame,
124 orient: str | None = ...,
125 date_format: str = ...,
126 double_precision: int = ...,
127 force_ascii: bool = ...,
128 date_unit: str = ...,
129 default_handler: Callable[[Any], JSONSerializable] | None = ...,
130 lines: bool = ...,
131 compression: CompressionOptions = ...,
132 index: bool | None = ...,
133 indent: int = ...,
134 storage_options: StorageOptions = ...,
135 mode: Literal["a", "w"] = ...,
136) -> str: ...
139def to_json(
140 path_or_buf: FilePath | WriteBuffer[str] | WriteBuffer[bytes] | None,
141 obj: NDFrame,
142 orient: str | None = None,
143 date_format: str = "epoch",
144 double_precision: int = 10,
145 force_ascii: bool = True,
146 date_unit: str = "ms",
147 default_handler: Callable[[Any], JSONSerializable] | None = None,
148 lines: bool = False,
149 compression: CompressionOptions = "infer",
150 index: bool | None = None,
151 indent: int = 0,
152 storage_options: StorageOptions | None = None,
153 mode: Literal["a", "w"] = "w",
154) -> str | None:
155 if orient in ["records", "values"] and index is True:
156 raise ValueError(
157 "'index=True' is only valid when 'orient' is 'split', 'table', "
158 "'index', or 'columns'."
159 )
160 elif orient in ["index", "columns"] and index is False:
161 raise ValueError(
162 "'index=False' is only valid when 'orient' is 'split', 'table', "
163 "'records', or 'values'."
164 )
165 elif index is None:
166 # will be ignored for orient='records' and 'values'
167 index = True
169 if lines and orient != "records":
170 raise ValueError("'lines' keyword only valid when 'orient' is records")
172 if mode not in ["a", "w"]:
173 msg = (
174 f"mode={mode} is not a valid option."
175 "Only 'w' and 'a' are currently supported."
176 )
177 raise ValueError(msg)
179 if mode == "a" and (not lines or orient != "records"):
180 msg = (
181 "mode='a' (append) is only supported when "
182 "lines is True and orient is 'records'"
183 )
184 raise ValueError(msg)
186 if orient == "table" and isinstance(obj, Series):
187 obj = obj.to_frame(name=obj.name or "values")
189 if date_format == "epoch":
190 # for epoch (numeric) format, convert datetime-likes to the desired
191 # unit up front, such that the C ObjToJSON code can simply write out
192 # the integer values without worrying about conversion
193 if date_unit not in ["s", "ms", "us", "ns"]:
194 raise ValueError(f"Invalid value '{date_unit}' for option 'date_unit'")
195 if isinstance(obj, DataFrame):
196 copied = False
197 cols = np.nonzero(obj.dtypes.map(lambda dt: dt.kind in ["M", "m"]))[0]
198 if len(cols):
199 obj = obj.copy(deep=False)
200 copied = True
201 for col in cols:
202 obj.isetitem(col, obj.iloc[:, col].dt.as_unit(date_unit))
203 if obj.index.dtype.kind in "Mm":
204 if not copied:
205 obj = obj.copy(deep=False)
206 copied = True
207 obj.index = Series(obj.index).dt.as_unit(date_unit)
208 if obj.columns.dtype.kind in "Mm":
209 if not copied:
210 obj = obj.copy(deep=False)
211 copied = True
212 obj.columns = Series(obj.columns).dt.as_unit(date_unit)
213 elif isinstance(obj, Series):
214 if obj.dtype.kind in "Mm":
215 obj = obj.copy(deep=False)
216 obj = obj.dt.as_unit(date_unit)
217 if obj.index.dtype.kind in "Mm":
218 obj = obj.copy(deep=False)
219 obj.index = Series(obj.index).dt.as_unit(date_unit)
221 writer: type[Writer]
222 if orient == "table" and isinstance(obj, DataFrame):
223 writer = JSONTableWriter
224 elif isinstance(obj, Series):
225 writer = SeriesWriter
226 elif isinstance(obj, DataFrame):
227 writer = FrameWriter
228 else:
229 raise NotImplementedError("'obj' should be a Series or a DataFrame")
231 s = writer(
232 obj,
233 orient=orient,
234 date_format=date_format,
235 double_precision=double_precision,
236 ensure_ascii=force_ascii,
237 date_unit=date_unit,
238 default_handler=default_handler,
239 index=index,
240 indent=indent,
241 ).write()
243 if lines:
244 s = convert_to_line_delimits(s)
246 if path_or_buf is not None:
247 # apply compression and byte/text conversion
248 with get_handle(
249 path_or_buf, mode, compression=compression, storage_options=storage_options
250 ) as handles:
251 handles.handle.write(s)
252 else:
253 return s
254 return None
257class Writer(ABC):
258 _default_orient: str
260 def __init__(
261 self,
262 obj: NDFrame,
263 orient: str | None,
264 date_format: str,
265 double_precision: int,
266 ensure_ascii: bool,
267 date_unit: str,
268 index: bool,
269 default_handler: Callable[[Any], JSONSerializable] | None = None,
270 indent: int = 0,
271 ) -> None:
272 self.obj = obj
274 if orient is None:
275 orient = self._default_orient
277 self.orient = orient
278 self.date_format = date_format
279 self.double_precision = double_precision
280 self.ensure_ascii = ensure_ascii
281 self.date_unit = date_unit
282 self.default_handler = default_handler
283 self.index = index
284 self.indent = indent
285 self._format_axes()
287 def _format_axes(self) -> None:
288 raise AbstractMethodError(self)
290 def write(self) -> str:
291 iso_dates = self.date_format == "iso"
292 return ujson_dumps(
293 self.obj_to_write,
294 orient=self.orient,
295 double_precision=self.double_precision,
296 ensure_ascii=self.ensure_ascii,
297 date_unit=self.date_unit,
298 iso_dates=iso_dates,
299 default_handler=self.default_handler,
300 indent=self.indent,
301 )
303 @property
304 @abstractmethod
305 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
306 """Object to write in JSON format."""
309class SeriesWriter(Writer):
310 _default_orient = "index"
312 @property
313 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
314 if not self.index and self.orient == "split":
315 return {"name": self.obj.name, "data": self.obj.values}
316 else:
317 return self.obj
319 def _format_axes(self) -> None:
320 if not self.obj.index.is_unique and self.orient == "index":
321 raise ValueError(f"Series index must be unique for orient='{self.orient}'")
324class FrameWriter(Writer):
325 _default_orient = "columns"
327 @property
328 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
329 if not self.index and self.orient == "split":
330 obj_to_write = self.obj.to_dict(orient="split")
331 del obj_to_write["index"]
332 else:
333 obj_to_write = self.obj
334 return obj_to_write
336 def _format_axes(self) -> None:
337 """
338 Try to format axes if they are datelike.
339 """
340 if not self.obj.index.is_unique and self.orient in ("index", "columns"):
341 raise ValueError(
342 f"DataFrame index must be unique for orient='{self.orient}'."
343 )
344 if not self.obj.columns.is_unique and self.orient in (
345 "index",
346 "columns",
347 "records",
348 ):
349 raise ValueError(
350 f"DataFrame columns must be unique for orient='{self.orient}'."
351 )
354class JSONTableWriter(FrameWriter):
355 _default_orient = "records"
357 def __init__(
358 self,
359 obj,
360 orient: str | None,
361 date_format: str,
362 double_precision: int,
363 ensure_ascii: bool,
364 date_unit: str,
365 index: bool,
366 default_handler: Callable[[Any], JSONSerializable] | None = None,
367 indent: int = 0,
368 ) -> None:
369 """
370 Adds a `schema` attribute with the Table Schema, resets
371 the index (can't do in caller, because the schema inference needs
372 to know what the index is, forces orient to records, and forces
373 date_format to 'iso'.
374 """
375 super().__init__(
376 obj,
377 orient,
378 date_format,
379 double_precision,
380 ensure_ascii,
381 date_unit,
382 index,
383 default_handler=default_handler,
384 indent=indent,
385 )
387 if date_format != "iso":
388 msg = (
389 "Trying to write with `orient='table'` and "
390 f"`date_format='{date_format}'`. Table Schema requires dates "
391 "to be formatted with `date_format='iso'`"
392 )
393 raise ValueError(msg)
395 self.schema = build_table_schema(obj, index=self.index)
396 if self.index:
397 obj = set_default_names(obj)
399 # NotImplemented on a column MultiIndex
400 if obj.ndim == 2 and isinstance(obj.columns, MultiIndex):
401 raise NotImplementedError(
402 "orient='table' is not supported for MultiIndex columns"
403 )
405 # TODO: Do this timedelta properly in objToJSON.c See GH #15137
406 if ((obj.ndim == 1) and (obj.name in set(obj.index.names))) or len(
407 obj.columns.intersection(obj.index.names)
408 ):
409 msg = "Overlapping names between the index and columns"
410 raise ValueError(msg)
412 timedeltas = obj.select_dtypes(include=["timedelta"]).columns
413 copied = False
414 if len(timedeltas):
415 obj = obj.copy()
416 copied = True
417 obj[timedeltas] = obj[timedeltas].map(lambda x: x.isoformat())
419 # exclude index from obj if index=False
420 if not self.index:
421 self.obj = obj.reset_index(drop=True)
422 else:
423 # Convert PeriodIndex to datetimes before serializing
424 if isinstance(obj.index.dtype, PeriodDtype):
425 if not copied:
426 obj = obj.copy(deep=False)
427 obj.index = obj.index.to_timestamp()
428 self.obj = obj.reset_index(drop=False)
429 self.date_format = "iso"
430 self.orient = "records"
431 self.index = index
433 @property
434 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
435 return {"schema": self.schema, "data": self.obj}
438@overload
439def read_json(
440 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
441 *,
442 orient: str | None = ...,
443 typ: Literal["frame"] = ...,
444 dtype: DtypeArg | None = ...,
445 convert_axes: bool | None = ...,
446 convert_dates: bool | list[str] = ...,
447 keep_default_dates: bool = ...,
448 precise_float: bool = ...,
449 date_unit: str | None = ...,
450 encoding: str | None = ...,
451 encoding_errors: str | None = ...,
452 lines: bool = ...,
453 chunksize: int,
454 compression: CompressionOptions = ...,
455 nrows: int | None = ...,
456 storage_options: StorageOptions = ...,
457 dtype_backend: DtypeBackend | lib.NoDefault = ...,
458 engine: JSONEngine = ...,
459) -> JsonReader[Literal["frame"]]: ...
462@overload
463def read_json(
464 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
465 *,
466 orient: str | None = ...,
467 typ: Literal["series"],
468 dtype: DtypeArg | None = ...,
469 convert_axes: bool | None = ...,
470 convert_dates: bool | list[str] = ...,
471 keep_default_dates: bool = ...,
472 precise_float: bool = ...,
473 date_unit: str | None = ...,
474 encoding: str | None = ...,
475 encoding_errors: str | None = ...,
476 lines: bool = ...,
477 chunksize: int,
478 compression: CompressionOptions = ...,
479 nrows: int | None = ...,
480 storage_options: StorageOptions = ...,
481 dtype_backend: DtypeBackend | lib.NoDefault = ...,
482 engine: JSONEngine = ...,
483) -> JsonReader[Literal["series"]]: ...
486@overload
487def read_json(
488 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
489 *,
490 orient: str | None = ...,
491 typ: Literal["series"],
492 dtype: DtypeArg | None = ...,
493 convert_axes: bool | None = ...,
494 convert_dates: bool | list[str] = ...,
495 keep_default_dates: bool = ...,
496 precise_float: bool = ...,
497 date_unit: str | None = ...,
498 encoding: str | None = ...,
499 encoding_errors: str | None = ...,
500 lines: bool = ...,
501 chunksize: None = ...,
502 compression: CompressionOptions = ...,
503 nrows: int | None = ...,
504 storage_options: StorageOptions = ...,
505 dtype_backend: DtypeBackend | lib.NoDefault = ...,
506 engine: JSONEngine = ...,
507) -> Series: ...
510@overload
511def read_json(
512 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
513 *,
514 orient: str | None = ...,
515 typ: Literal["frame"] = ...,
516 dtype: DtypeArg | None = ...,
517 convert_axes: bool | None = ...,
518 convert_dates: bool | list[str] = ...,
519 keep_default_dates: bool = ...,
520 precise_float: bool = ...,
521 date_unit: str | None = ...,
522 encoding: str | None = ...,
523 encoding_errors: str | None = ...,
524 lines: bool = ...,
525 chunksize: None = ...,
526 compression: CompressionOptions = ...,
527 nrows: int | None = ...,
528 storage_options: StorageOptions = ...,
529 dtype_backend: DtypeBackend | lib.NoDefault = ...,
530 engine: JSONEngine = ...,
531) -> DataFrame: ...
534@set_module("pandas")
535def read_json(
536 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
537 *,
538 orient: str | None = None,
539 typ: Literal["frame", "series"] = "frame",
540 dtype: DtypeArg | None = None,
541 convert_axes: bool | None = None,
542 convert_dates: bool | list[str] = True,
543 keep_default_dates: bool = True,
544 precise_float: bool = False,
545 date_unit: str | None = None,
546 encoding: str | None = None,
547 encoding_errors: str | None = "strict",
548 lines: bool = False,
549 chunksize: int | None = None,
550 compression: CompressionOptions = "infer",
551 nrows: int | None = None,
552 storage_options: StorageOptions | None = None,
553 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
554 engine: JSONEngine = "ujson",
555) -> DataFrame | Series | JsonReader:
556 """
557 Convert a JSON string to pandas object.
559 This method reads JSON files or JSON-like data and converts them into pandas
560 objects. It supports a variety of input formats, including line-delimited JSON,
561 compressed files, and various data representations (table, records, index-based,
562 etc.). When `chunksize` is specified, an iterator is returned instead of loading
563 the entire data into memory.
565 Parameters
566 ----------
567 path_or_buf : a str path, path object or file-like object
568 Any valid string path is acceptable. The string could be a URL. Valid
569 URL schemes include http, ftp, s3, and file. For file URLs, a host is
570 expected. A local file could be:
571 ``file://localhost/path/to/table.json``.
573 If you want to pass in a path object, pandas accepts any
574 ``os.PathLike``.
576 By file-like object, we refer to objects with a ``read()`` method,
577 such as a file handle (e.g. via builtin ``open`` function)
578 or ``StringIO``.
580 orient : str, optional
581 Indication of expected JSON string format.
582 Compatible JSON strings can be produced by ``to_json()`` with a
583 corresponding orient value.
584 The set of possible orients is:
586 - ``'split'`` : dict like
587 ``{index -> [index], columns -> [columns], data -> [values]}``
588 - ``'records'`` : list like
589 ``[{column -> value}, ... , {column -> value}]``
590 - ``'index'`` : dict like ``{index -> {column -> value}}``
591 - ``'columns'`` : dict like ``{column -> {index -> value}}``
592 - ``'values'`` : just the values array
593 - ``'table'`` : dict like ``{'schema': {schema}, 'data': {data}}``
595 The allowed and default values depend on the value
596 of the `typ` parameter.
598 * when ``typ == 'series'``,
600 - allowed orients are ``{'split','records','index'}``
601 - default is ``'index'``
602 - The Series index must be unique for orient ``'index'``.
604 * when ``typ == 'frame'``,
606 - allowed orients are ``{'split','records','index',
607 'columns','values', 'table'}``
608 - default is ``'columns'``
609 - The DataFrame index must be unique for orients ``'index'`` and
610 ``'columns'``.
611 - The DataFrame columns must be unique for orients ``'index'``,
612 ``'columns'``, and ``'records'``.
614 typ : {'frame', 'series'}, default 'frame'
615 The type of object to recover.
617 dtype : bool or dict, default None
618 If True, infer dtypes; if a dict of column to dtype, then use those;
619 if False, then don't infer dtypes at all, applies only to the data.
621 For all ``orient`` values except ``'table'``, default is True.
623 convert_axes : bool, default None
624 Try to convert the axes to the proper dtypes.
626 For all ``orient`` values except ``'table'``, default is True.
628 convert_dates : bool or list of str, default True
629 If True then default datelike columns may be converted (depending on
630 keep_default_dates).
631 If False, no dates will be converted.
632 If a list of column names, then those columns will be converted and
633 default datelike columns may also be converted (depending on
634 keep_default_dates).
636 keep_default_dates : bool, default True
637 If parsing dates (convert_dates is not False), then try to parse the
638 default datelike columns.
639 A column label is datelike if
641 * it ends with ``'_at'``,
643 * it ends with ``'_time'``,
645 * it begins with ``'timestamp'``,
647 * it is ``'modified'``, or
649 * it is ``'date'``.
651 precise_float : bool, default False
652 Set to enable usage of higher precision (strtod) function when
653 decoding string to double values. Default (False) is to use fast but
654 less precise builtin functionality.
656 date_unit : str, default None
657 The timestamp unit to detect if converting dates. The default behaviour
658 is to try and detect the correct precision, but if this is not desired
659 then pass one of 's', 'ms', 'us' or 'ns' to force parsing only seconds,
660 milliseconds, microseconds or nanoseconds respectively.
662 encoding : str, default is 'utf-8'
663 The encoding to use to decode py3 bytes.
665 encoding_errors : str, optional, default "strict"
666 How encoding errors are treated. `List of possible values
667 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
669 lines : bool, default False
670 Read the file as a json object per line.
672 chunksize : int, optional
673 Return JsonReader object for iteration.
674 See the `line-delimited json docs
675 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#line-delimited-json>`_
676 for more information on ``chunksize``.
677 This can only be passed if `lines=True`.
678 If this is None, the file will be read into memory all at once.
680 compression : str or dict, default 'infer'
681 For on-the-fly decompression of on-disk data. If 'infer' and 'path_or_buf' is
682 path-like, then detect compression from the following extensions: '.gz',
683 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
684 (otherwise no compression).
685 If using 'zip' or 'tar', the ZIP file must contain only one data file to be
686 read in.
687 Set to ``None`` for no decompression.
688 Can also be a dict with key ``'method'`` set
689 to one of {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``}
690 and other key-value pairs are forwarded to
691 ``zipfile.ZipFile``, ``gzip.GzipFile``,
692 ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
693 ``tarfile.TarFile``, respectively.
694 As an example, the following could be passed for Zstandard decompression using a
695 custom compression dictionary:
696 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
698 nrows : int, optional
699 The number of lines from the line-delimited jsonfile that has to be read.
700 This can only be passed if `lines=True`.
701 If this is None, all the rows will be returned.
703 storage_options : dict, optional
704 Extra options that make sense for a particular storage connection, e.g.
705 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
706 are forwarded to ``urllib.request.Request`` as header options. For other
707 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
708 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
709 details, and for more examples on storage options refer `here
710 <https://pandas.pydata.org/docs/user_guide/io.html?
711 highlight=storage_options#reading-writing-remote-files>`_.
713 dtype_backend : {'numpy_nullable', 'pyarrow'}
714 Back-end data type applied to the resultant :class:`DataFrame`
715 (still experimental). If not specified, the default behavior
716 is to not use nullable data types. If specified, the behavior
717 is as follows:
719 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
720 * ``"pyarrow"``: returns pyarrow-backed nullable
721 :class:`ArrowDtype` :class:`DataFrame`
723 .. versionadded:: 2.0
725 engine : {"ujson", "pyarrow"}, default "ujson"
726 Parser engine to use. The ``"pyarrow"`` engine is only available when
727 ``lines=True``.
729 .. versionadded:: 2.0
731 Returns
732 -------
733 Series, DataFrame, or pandas.api.typing.JsonReader
734 A JsonReader is returned when ``chunksize`` is not ``0`` or ``None``.
735 Otherwise, the type returned depends on the value of ``typ``.
737 See Also
738 --------
739 DataFrame.to_json : Convert a DataFrame to a JSON string.
740 Series.to_json : Convert a Series to a JSON string.
741 json_normalize : Normalize semi-structured JSON data into a flat table.
743 Notes
744 -----
745 Specific to ``orient='table'``, if a :class:`DataFrame` with a literal
746 :class:`Index` name of `index` gets written with :func:`to_json`, the
747 subsequent read operation will incorrectly set the :class:`Index` name to
748 ``None``. This is because `index` is also used by :func:`DataFrame.to_json`
749 to denote a missing :class:`Index` name, and the subsequent
750 :func:`read_json` operation cannot distinguish between the two. The same
751 limitation is encountered with a :class:`MultiIndex` and any names
752 beginning with ``'level_'``.
754 Examples
755 --------
756 >>> from io import StringIO
757 >>> df = pd.DataFrame(
758 ... [["a", "b"], ["c", "d"]],
759 ... index=["row 1", "row 2"],
760 ... columns=["col 1", "col 2"],
761 ... )
763 Encoding/decoding a Dataframe using ``'split'`` formatted JSON:
765 >>> df.to_json(orient="split")
766 '{"columns":["col 1","col 2"],"index":["row 1","row 2"],"data":[["a","b"],["c","d"]]}'
768 >>> pd.read_json(StringIO(_), orient="split") # noqa: F821
769 col 1 col 2
770 row 1 a b
771 row 2 c d
773 Encoding/decoding a Dataframe using ``'index'`` formatted JSON:
775 >>> df.to_json(orient="index")
776 '{"row 1":{"col 1":"a","col 2":"b"},"row 2":{"col 1":"c","col 2":"d"}}'
778 >>> pd.read_json(StringIO(_), orient="index") # noqa: F821
779 col 1 col 2
780 row 1 a b
781 row 2 c d
783 Encoding/decoding a Dataframe using ``'records'`` formatted JSON.
784 Note that index labels are not preserved with this encoding.
786 >>> df.to_json(orient="records")
787 '[{"col 1":"a","col 2":"b"},{"col 1":"c","col 2":"d"}]'
789 >>> pd.read_json(StringIO(_), orient="records") # noqa: F821
790 col 1 col 2
791 0 a b
792 1 c d
794 Encoding with Table Schema
796 >>> df.to_json(orient="table")
797 '{"schema":{"fields":[{"name":"index","type":"string","extDtype":"str"},{"name":"col 1","type":"string","extDtype":"str"},{"name":"col 2","type":"string","extDtype":"str"}],"primaryKey":["index"],"pandas_version":"1.4.0"},"data":[{"index":"row 1","col 1":"a","col 2":"b"},{"index":"row 2","col 1":"c","col 2":"d"}]}'
799 The following example uses ``dtype_backend="numpy_nullable"``
801 >>> data = '''{"index": {"0": 0, "1": 1},
802 ... "a": {"0": 1, "1": null},
803 ... "b": {"0": 2.5, "1": 4.5},
804 ... "c": {"0": true, "1": false},
805 ... "d": {"0": "a", "1": "b"},
806 ... "e": {"0": 1577.2, "1": 1577.1}}'''
807 >>> pd.read_json(StringIO(data), dtype_backend="numpy_nullable")
808 index a b c d e
809 0 0 1 2.5 True a 1577.2
810 1 1 <NA> 4.5 False b 1577.1
811 """ # noqa: E501
812 if orient == "table" and dtype:
813 raise ValueError("cannot pass both dtype and orient='table'")
814 if orient == "table" and convert_axes:
815 raise ValueError("cannot pass both convert_axes and orient='table'")
817 check_dtype_backend(dtype_backend)
819 if dtype is None and orient != "table":
820 # error: Incompatible types in assignment (expression has type "bool", variable
821 # has type "Union[ExtensionDtype, str, dtype[Any], Type[str], Type[float],
822 # Type[int], Type[complex], Type[bool], Type[object], Dict[Hashable,
823 # Union[ExtensionDtype, Union[str, dtype[Any]], Type[str], Type[float],
824 # Type[int], Type[complex], Type[bool], Type[object]]], None]")
825 dtype = True # type: ignore[assignment]
826 if convert_axes is None and orient != "table":
827 convert_axes = True
829 json_reader = JsonReader(
830 path_or_buf,
831 orient=orient,
832 typ=typ,
833 dtype=dtype,
834 convert_axes=convert_axes,
835 convert_dates=convert_dates,
836 keep_default_dates=keep_default_dates,
837 precise_float=precise_float,
838 date_unit=date_unit,
839 encoding=encoding,
840 lines=lines,
841 chunksize=chunksize,
842 compression=compression,
843 nrows=nrows,
844 storage_options=storage_options,
845 encoding_errors=encoding_errors,
846 dtype_backend=dtype_backend,
847 engine=engine,
848 )
850 if chunksize:
851 return json_reader
852 else:
853 return json_reader.read()
856@set_module("pandas.api.typing")
857class JsonReader(abc.Iterator, Generic[FrameSeriesStrT]):
858 """
859 JsonReader provides an interface for reading in a JSON file.
861 If initialized with ``lines=True`` and ``chunksize``, can be iterated over
862 ``chunksize`` lines at a time. Otherwise, calling ``read`` reads in the
863 whole document.
864 """
866 def __init__(
867 self,
868 filepath_or_buffer,
869 orient,
870 typ: FrameSeriesStrT,
871 dtype,
872 convert_axes: bool | None,
873 convert_dates,
874 keep_default_dates: bool,
875 precise_float: bool,
876 date_unit,
877 encoding,
878 lines: bool,
879 chunksize: int | None,
880 compression: CompressionOptions,
881 nrows: int | None,
882 storage_options: StorageOptions | None = None,
883 encoding_errors: str | None = "strict",
884 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
885 engine: JSONEngine = "ujson",
886 ) -> None:
887 self.orient = orient
888 self.typ = typ
889 self.dtype = dtype
890 self.convert_axes = convert_axes
891 self.convert_dates = convert_dates
892 self.keep_default_dates = keep_default_dates
893 self.precise_float = precise_float
894 self.date_unit = date_unit
895 self.encoding = encoding
896 self.engine = engine
897 self.compression = compression
898 self.storage_options = storage_options
899 self.lines = lines
900 self.chunksize = chunksize
901 self.nrows_seen = 0
902 self.nrows = nrows
903 self.encoding_errors = encoding_errors
904 self.handles: IOHandles[str] | None = None
905 self.dtype_backend = dtype_backend
907 if self.engine not in {"pyarrow", "ujson"}:
908 raise ValueError(
909 f"The engine type {self.engine} is currently not supported."
910 )
911 if self.chunksize is not None:
912 self.chunksize = validate_integer("chunksize", self.chunksize, 1)
913 if not self.lines:
914 raise ValueError("chunksize can only be passed if lines=True")
915 if self.engine == "pyarrow":
916 raise ValueError(
917 "currently pyarrow engine doesn't support chunksize parameter"
918 )
919 if self.nrows is not None:
920 self.nrows = validate_integer("nrows", self.nrows, 0)
921 if not self.lines:
922 raise ValueError("nrows can only be passed if lines=True")
923 if self.engine == "pyarrow":
924 if not self.lines:
925 raise ValueError(
926 "currently pyarrow engine only supports "
927 "the line-delimited JSON format"
928 )
929 self.data = filepath_or_buffer
930 elif self.engine == "ujson":
931 data = self._get_data_from_filepath(filepath_or_buffer)
932 # If self.chunksize, we prepare the data for the `__next__` method.
933 # Otherwise, we read it into memory for the `read` method.
934 if not (self.chunksize or self.nrows):
935 with self:
936 self.data = data.read()
937 else:
938 self.data = data
940 def _get_data_from_filepath(self, filepath_or_buffer):
941 """
942 The function read_json accepts three input types:
943 1. filepath (string-like)
944 2. file-like object (e.g. open file object, StringIO)
945 """
946 filepath_or_buffer = stringify_path(filepath_or_buffer)
947 try:
948 self.handles = get_handle(
949 filepath_or_buffer,
950 "r",
951 encoding=self.encoding,
952 compression=self.compression,
953 storage_options=self.storage_options,
954 errors=self.encoding_errors,
955 )
956 except OSError as err:
957 raise FileNotFoundError(
958 f"File {filepath_or_buffer} does not exist"
959 ) from err
960 filepath_or_buffer = self.handles.handle
961 return filepath_or_buffer
963 def _combine_lines(self, lines) -> str:
964 """
965 Combines a list of JSON objects into one JSON object.
966 """
967 return (
968 f"[{','.join([line for line in (line.strip() for line in lines) if line])}]"
969 )
971 @overload
972 def read(self: JsonReader[Literal["frame"]]) -> DataFrame: ...
974 @overload
975 def read(self: JsonReader[Literal["series"]]) -> Series: ...
977 @overload
978 def read(self: JsonReader[Literal["frame", "series"]]) -> DataFrame | Series: ...
980 def read(self) -> DataFrame | Series:
981 """
982 Read the whole JSON input into a pandas object.
983 """
984 obj: DataFrame | Series
985 with self:
986 if self.engine == "pyarrow":
987 obj = self._read_pyarrow()
988 elif self.engine == "ujson":
989 obj = self._read_ujson()
991 return obj
993 def _read_pyarrow(self) -> DataFrame:
994 """
995 Read JSON using the pyarrow engine.
996 """
997 pyarrow_json = import_optional_dependency("pyarrow.json")
998 options = None
1000 if isinstance(self.dtype, dict):
1001 pa = import_optional_dependency("pyarrow")
1002 fields = []
1003 for field, dtype in self.dtype.items():
1004 pd_dtype = pandas_dtype(dtype)
1005 if isinstance(pd_dtype, ArrowDtype):
1006 fields.append((field, pd_dtype.pyarrow_dtype))
1008 schema = pa.schema(fields)
1009 options = pyarrow_json.ParseOptions(
1010 explicit_schema=schema, unexpected_field_behavior="infer"
1011 )
1013 pa_table = pyarrow_json.read_json(self.data, parse_options=options)
1014 df = arrow_table_to_pandas(pa_table, dtype_backend=self.dtype_backend)
1016 return df
1018 def _read_ujson(self) -> DataFrame | Series:
1019 """
1020 Read JSON using the ujson engine.
1021 """
1022 obj: DataFrame | Series
1023 if self.lines:
1024 if self.chunksize:
1025 obj = concat(self)
1026 elif self.nrows:
1027 lines = list(islice(self.data, self.nrows))
1028 lines_json = self._combine_lines(lines)
1029 obj = self._get_object_parser(lines_json)
1030 else:
1031 data = ensure_str(self.data)
1032 data_lines = data.split("\n")
1033 obj = self._get_object_parser(self._combine_lines(data_lines))
1034 else:
1035 obj = self._get_object_parser(self.data)
1036 if self.dtype_backend is not lib.no_default:
1037 with option_context("future.distinguish_nan_and_na", False):
1038 return obj.convert_dtypes(
1039 infer_objects=False, dtype_backend=self.dtype_backend
1040 )
1041 else:
1042 return obj
1044 def _get_object_parser(self, json: str) -> DataFrame | Series:
1045 """
1046 Parses a json document into a pandas object.
1047 """
1048 typ = self.typ
1049 dtype = self.dtype
1050 kwargs = {
1051 "orient": self.orient,
1052 "dtype": self.dtype,
1053 "convert_axes": self.convert_axes,
1054 "convert_dates": self.convert_dates,
1055 "keep_default_dates": self.keep_default_dates,
1056 "precise_float": self.precise_float,
1057 "date_unit": self.date_unit,
1058 "dtype_backend": self.dtype_backend,
1059 }
1060 if typ == "frame":
1061 return FrameParser(json, **kwargs).parse()
1062 elif typ == "series":
1063 if not isinstance(dtype, bool):
1064 kwargs["dtype"] = dtype
1065 return SeriesParser(json, **kwargs).parse()
1066 else:
1067 raise ValueError(f"{typ=} must be 'frame' or 'series'.")
1069 def close(self) -> None:
1070 """
1071 If we opened a stream earlier, in _get_data_from_filepath, we should
1072 close it.
1074 If an open stream or file was passed, we leave it open.
1075 """
1076 if self.handles is not None:
1077 self.handles.close()
1079 def __iter__(self) -> Self:
1080 return self
1082 @overload
1083 def __next__(self: JsonReader[Literal["frame"]]) -> DataFrame: ...
1085 @overload
1086 def __next__(self: JsonReader[Literal["series"]]) -> Series: ...
1088 @overload
1089 def __next__(
1090 self: JsonReader[Literal["frame", "series"]],
1091 ) -> DataFrame | Series: ...
1093 def __next__(self) -> DataFrame | Series:
1094 if self.nrows and self.nrows_seen >= self.nrows:
1095 self.close()
1096 raise StopIteration
1098 lines = list(islice(self.data, self.chunksize))
1099 if not lines:
1100 self.close()
1101 raise StopIteration
1103 try:
1104 lines_json = self._combine_lines(lines)
1105 obj = self._get_object_parser(lines_json)
1107 # Make sure that the returned objects have the right index.
1108 obj.index = range(self.nrows_seen, self.nrows_seen + len(obj))
1109 self.nrows_seen += len(obj)
1110 except Exception as ex:
1111 self.close()
1112 raise ex
1114 if self.dtype_backend is not lib.no_default:
1115 with option_context("future.distinguish_nan_and_na", False):
1116 return obj.convert_dtypes(
1117 infer_objects=False, dtype_backend=self.dtype_backend
1118 )
1119 else:
1120 return obj
1122 def __enter__(self) -> Self:
1123 return self
1125 def __exit__(
1126 self,
1127 exc_type: type[BaseException] | None,
1128 exc_value: BaseException | None,
1129 traceback: TracebackType | None,
1130 ) -> None:
1131 self.close()
1134class Parser:
1135 _split_keys: tuple[str, ...]
1136 _default_orient: str
1138 _STAMP_UNITS = ("s", "ms", "us", "ns")
1139 _MIN_STAMPS = {
1140 "s": 31536000,
1141 "ms": 31536000000,
1142 "us": 31536000000000,
1143 "ns": 31536000000000000,
1144 }
1145 json: str
1147 def __init__(
1148 self,
1149 json: str,
1150 orient,
1151 dtype: DtypeArg | None = None,
1152 convert_axes: bool = True,
1153 convert_dates: bool | list[str] = True,
1154 keep_default_dates: bool = False,
1155 precise_float: bool = False,
1156 date_unit=None,
1157 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1158 ) -> None:
1159 self.json = json
1161 if orient is None:
1162 orient = self._default_orient
1164 self.orient = orient
1166 self.dtype = dtype
1168 if date_unit is not None:
1169 date_unit = date_unit.lower()
1170 if date_unit not in self._STAMP_UNITS:
1171 raise ValueError(f"date_unit must be one of {self._STAMP_UNITS}")
1172 self.min_stamp = self._MIN_STAMPS[date_unit]
1173 else:
1174 self.min_stamp = self._MIN_STAMPS["s"]
1176 self.precise_float = precise_float
1177 self.convert_axes = convert_axes
1178 self.convert_dates = convert_dates
1179 self.date_unit = date_unit
1180 self.keep_default_dates = keep_default_dates
1181 self.dtype_backend = dtype_backend
1183 @final
1184 def check_keys_split(self, decoded: dict) -> None:
1185 """
1186 Checks that dict has only the appropriate keys for orient='split'.
1187 """
1188 bad_keys = set(decoded.keys()).difference(set(self._split_keys))
1189 if bad_keys:
1190 bad_keys_joined = ", ".join(bad_keys)
1191 raise ValueError(f"JSON data had unexpected key(s): {bad_keys_joined}")
1193 @final
1194 def parse(self) -> DataFrame | Series:
1195 obj = self._parse()
1197 if self.convert_axes:
1198 obj = self._convert_axes(obj)
1199 obj = self._try_convert_types(obj)
1200 return obj
1202 def _parse(self) -> DataFrame | Series:
1203 raise AbstractMethodError(self)
1205 @final
1206 def _convert_axes(self, obj: DataFrame | Series) -> DataFrame | Series:
1207 """
1208 Try to convert axes.
1209 """
1210 for axis_name in obj._AXIS_ORDERS:
1211 ax = obj._get_axis(axis_name)
1212 ser = Series(ax, dtype=ax.dtype, copy=False)
1213 new_ser, result = self._try_convert_data(
1214 name=axis_name,
1215 data=ser,
1216 use_dtypes=False,
1217 convert_dates=True,
1218 is_axis=True,
1219 )
1220 if result:
1221 new_axis = Index(new_ser, dtype=new_ser.dtype, copy=False)
1222 setattr(obj, axis_name, new_axis)
1223 return obj
1225 def _try_convert_types(self, obj):
1226 raise AbstractMethodError(self)
1228 @final
1229 def _try_convert_data(
1230 self,
1231 name: Hashable,
1232 data: Series,
1233 use_dtypes: bool = True,
1234 convert_dates: bool | list[str] = True,
1235 is_axis: bool = False,
1236 ) -> tuple[Series, bool]:
1237 """
1238 Try to parse a Series into a column by inferring dtype.
1239 """
1240 org_data = data
1241 # don't try to coerce, unless a force conversion
1242 if use_dtypes:
1243 if not self.dtype:
1244 if all(notna(data)):
1245 return data, False
1247 filled = data.fillna(np.nan)
1249 return filled, True
1251 elif self.dtype is True:
1252 pass
1253 elif not _should_convert_dates(
1254 convert_dates, self.keep_default_dates, name
1255 ):
1256 # convert_dates takes precedence over columns listed in dtypes
1257 dtype = (
1258 self.dtype.get(name) if isinstance(self.dtype, dict) else self.dtype
1259 )
1260 if dtype is not None:
1261 try:
1262 return data.astype(dtype), True
1263 except (TypeError, ValueError):
1264 return data, False
1266 if convert_dates:
1267 new_data = self._try_convert_to_date(data)
1268 if new_data is not data:
1269 return new_data, True
1271 converted = False
1272 if self.dtype_backend is not lib.no_default and not is_axis:
1273 # Fall through for conversion later on
1274 return data, True
1275 elif is_string_dtype(data.dtype):
1276 # try float
1277 try:
1278 data = data.astype("float64")
1279 converted = True
1280 except (TypeError, ValueError):
1281 pass
1283 if data.dtype.kind == "f" and data.dtype != "float64":
1284 # coerce floats to 64
1285 try:
1286 data = data.astype("float64")
1287 converted = True
1288 except (TypeError, ValueError):
1289 pass
1291 # don't coerce 0-len data
1292 if len(data) and data.dtype in ("float", "object"):
1293 # coerce ints if we can
1294 try:
1295 new_data = org_data.astype("int64")
1296 if (new_data == data).all():
1297 data = new_data
1298 converted = True
1299 except (TypeError, ValueError, OverflowError):
1300 pass
1302 if data.dtype == "int" and data.dtype != "int64":
1303 # coerce ints to 64
1304 try:
1305 data = data.astype("int64")
1306 converted = True
1307 except (TypeError, ValueError):
1308 pass
1310 # if we have an index, we want to preserve dtypes
1311 if name == "index" and len(data):
1312 if self.orient == "split":
1313 return data, False
1315 return data, converted
1317 @final
1318 def _try_convert_to_date(self, data: Series) -> Series:
1319 """
1320 Try to parse an ndarray like into a date column.
1322 Try to coerce object in epoch/iso formats and integer/float in epoch
1323 formats.
1324 """
1325 # no conversion on empty
1326 if not len(data):
1327 return data
1329 new_data = data
1331 if new_data.dtype == "object" or new_data.dtype == "string": # noqa: PLR1714
1332 try:
1333 new_data = data.astype("int64")
1334 except OverflowError:
1335 return data
1336 except (TypeError, ValueError):
1337 pass
1339 # ignore numbers that are out of range
1340 if issubclass(new_data.dtype.type, np.number):
1341 in_range = (
1342 isna(new_data._values)
1343 | (new_data > self.min_stamp)
1344 | (new_data._values == iNaT)
1345 )
1346 if not in_range.all():
1347 return data
1349 if new_data.dtype == "string":
1350 with warnings.catch_warnings():
1351 # ignore "Could not infer format" warnings from to_datetime
1352 # which is incorrectly raised for non-date strings
1353 warnings.simplefilter("ignore", UserWarning)
1354 for format in (None, "iso8601", "mixed"):
1355 try:
1356 return to_datetime(new_data, errors="raise", format=format)
1357 except Exception:
1358 pass
1359 else:
1360 # numeric or mixed objects
1361 date_units = (self.date_unit,) if self.date_unit else self._STAMP_UNITS
1362 for date_unit in date_units:
1363 try:
1364 # In case of multiple possible units, infer the likely unit
1365 # based on the first unit for which the parsed dates fit
1366 # within the nanoseconds bounds
1367 # -> do as_unit cast to ensure OutOfBounds error
1368 data = to_datetime(new_data, errors="raise", unit=date_unit)
1369 _ = data.dt.as_unit("ns")
1370 break
1371 except OutOfBoundsDatetime:
1372 continue
1373 except (ValueError, OverflowError, TypeError):
1374 pass
1375 return data
1378class SeriesParser(Parser):
1379 _default_orient = "index"
1380 _split_keys = ("name", "index", "data")
1382 def _parse(self) -> Series:
1383 data = ujson_loads(self.json, precise_float=self.precise_float)
1385 if self.orient == "split":
1386 decoded = {str(k): v for k, v in data.items()}
1387 self.check_keys_split(decoded)
1388 return Series(**decoded)
1389 else:
1390 return Series(data)
1392 def _try_convert_types(self, obj: Series) -> Series:
1393 obj, _ = self._try_convert_data("data", obj, convert_dates=self.convert_dates)
1394 return obj
1397class FrameParser(Parser):
1398 _default_orient = "columns"
1399 _split_keys = ("columns", "index", "data")
1401 def _parse(self) -> DataFrame:
1402 json = self.json
1403 orient = self.orient
1405 if orient == "split":
1406 decoded = {
1407 str(k): v
1408 for k, v in ujson_loads(json, precise_float=self.precise_float).items()
1409 }
1410 self.check_keys_split(decoded)
1411 orig_names = [
1412 (tuple(col) if isinstance(col, list) else col)
1413 for col in decoded["columns"]
1414 ]
1415 decoded["columns"] = dedup_names(
1416 orig_names,
1417 is_potential_multi_index(orig_names, None),
1418 )
1419 return DataFrame(dtype=None, **decoded)
1420 elif orient == "index":
1421 return DataFrame.from_dict(
1422 ujson_loads(json, precise_float=self.precise_float),
1423 dtype=None,
1424 orient="index",
1425 )
1426 elif orient == "table":
1427 return parse_table_schema(json, precise_float=self.precise_float)
1428 else:
1429 # includes orient == "columns"
1430 return DataFrame(
1431 ujson_loads(json, precise_float=self.precise_float), dtype=None
1432 )
1434 def _try_convert_types(self, obj: DataFrame) -> DataFrame:
1435 arrays = []
1436 for col_label, series in obj.items():
1437 result, _ = self._try_convert_data(
1438 col_label,
1439 series,
1440 convert_dates=_should_convert_dates(
1441 self.convert_dates,
1442 keep_default_dates=self.keep_default_dates,
1443 col=col_label,
1444 ),
1445 )
1446 arrays.append(result.array)
1447 return DataFrame._from_arrays(
1448 arrays, obj.columns, obj.index, verify_integrity=False
1449 )
1452def _should_convert_dates(
1453 convert_dates: bool | list[str],
1454 keep_default_dates: bool,
1455 col: Hashable,
1456) -> bool:
1457 """
1458 Return bool whether a DataFrame column should be cast to datetime.
1459 """
1460 if convert_dates is False:
1461 # convert_dates=True means follow keep_default_dates
1462 return False
1463 elif not isinstance(convert_dates, bool) and col in set(convert_dates):
1464 return True
1465 elif not keep_default_dates:
1466 return False
1467 elif not isinstance(col, str):
1468 return False
1469 col_lower = col.lower()
1470 if (
1471 col_lower.endswith(("_at", "_time"))
1472 or col_lower in {"modified", "date", "datetime"}
1473 or col_lower.startswith("timestamp")
1474 ):
1475 return True
1476 return False