Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/json/_json.py: 20%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1from __future__ import annotations
3from abc import (
4 ABC,
5 abstractmethod,
6)
7from collections import abc
8from itertools import islice
9from typing import (
10 TYPE_CHECKING,
11 Any,
12 Generic,
13 Literal,
14 Self,
15 TypeVar,
16 final,
17 overload,
18)
19import warnings
21import numpy as np
23from pandas._config import option_context
25from pandas._libs import lib
26from pandas._libs._ujson import (
27 ujson_dumps,
28 ujson_loads,
29)
30from pandas._libs.tslibs import iNaT
31from pandas.compat._optional import import_optional_dependency
32from pandas.errors import (
33 AbstractMethodError,
34 OutOfBoundsDatetime,
35 Pandas4Warning,
36)
37from pandas.util._decorators import set_module
38from pandas.util._exceptions import find_stack_level
39from pandas.util._validators import check_dtype_backend
41from pandas.core.dtypes.common import (
42 ensure_str,
43 is_string_dtype,
44 pandas_dtype,
45)
46from pandas.core.dtypes.dtypes import (
47 PeriodDtype,
48 SparseDtype,
49)
51from pandas import (
52 ArrowDtype,
53 DataFrame,
54 Index,
55 MultiIndex,
56 Series,
57 isna,
58 notna,
59 to_datetime,
60)
61from pandas.core.reshape.concat import concat
63from pandas.io._util import arrow_table_to_pandas
64from pandas.io.common import (
65 IOHandles,
66 dedup_names,
67 get_handle,
68 is_potential_multi_index,
69 stringify_path,
70)
71from pandas.io.json._normalize import convert_to_line_delimits
72from pandas.io.json._table_schema import (
73 build_table_schema,
74 parse_table_schema,
75 set_default_names,
76)
77from pandas.io.parsers.readers import validate_integer
79if TYPE_CHECKING:
80 from collections.abc import (
81 Callable,
82 Hashable,
83 Mapping,
84 )
85 from types import TracebackType
87 from pandas._typing import (
88 CompressionOptions,
89 DtypeArg,
90 DtypeBackend,
91 DtypeObj,
92 FilePath,
93 IndexLabel,
94 JSONEngine,
95 JSONSerializable,
96 ReadBuffer,
97 StorageOptions,
98 WriteBuffer,
99 )
101 from pandas.core.generic import NDFrame
103FrameSeriesStrT = TypeVar("FrameSeriesStrT", bound=Literal["frame", "series"])
106def _has_dt_accessor(dtype: DtypeObj) -> bool:
107 """
108 Whether this dtype takes the ``.dt.as_unit`` conversion below.
110 SparseDtype reports the subtype's kind, so the kind check alone is not
111 enough. This is not an exhaustive test for a usable ``.dt.as_unit`` -- it
112 only rules out what is known to reach here. Datetime-likes it returns False
113 for are scaled to date_unit by the C encoder instead.
114 """
115 return dtype.kind in "Mm" and not isinstance(dtype, SparseDtype)
118# interface to/from
119@overload
120def to_json(
121 path_or_buf: FilePath | WriteBuffer[str] | WriteBuffer[bytes],
122 obj: NDFrame,
123 orient: str | None = ...,
124 date_format: str = ...,
125 double_precision: int = ...,
126 force_ascii: bool = ...,
127 date_unit: str = ...,
128 default_handler: Callable[[Any], JSONSerializable] | None = ...,
129 lines: bool = ...,
130 compression: CompressionOptions = ...,
131 index: bool | None = ...,
132 indent: int = ...,
133 storage_options: StorageOptions = ...,
134 mode: Literal["a", "w"] = ...,
135) -> None: ...
138@overload
139def to_json(
140 path_or_buf: None,
141 obj: NDFrame,
142 orient: str | None = ...,
143 date_format: str = ...,
144 double_precision: int = ...,
145 force_ascii: bool = ...,
146 date_unit: str = ...,
147 default_handler: Callable[[Any], JSONSerializable] | None = ...,
148 lines: bool = ...,
149 compression: CompressionOptions = ...,
150 index: bool | None = ...,
151 indent: int = ...,
152 storage_options: StorageOptions = ...,
153 mode: Literal["a", "w"] = ...,
154) -> str: ...
157def to_json(
158 path_or_buf: FilePath | WriteBuffer[str] | WriteBuffer[bytes] | None,
159 obj: NDFrame,
160 orient: str | None = None,
161 date_format: str = "epoch",
162 double_precision: int = 10,
163 force_ascii: bool = True,
164 date_unit: str = "ms",
165 default_handler: Callable[[Any], JSONSerializable] | None = None,
166 lines: bool = False,
167 compression: CompressionOptions = "infer",
168 index: bool | None = None,
169 indent: int = 0,
170 storage_options: StorageOptions | None = None,
171 mode: Literal["a", "w"] = "w",
172) -> str | None:
173 if orient in ["records", "values"] and index is True:
174 raise ValueError(
175 "'index=True' is only valid when 'orient' is 'split', 'table', "
176 "'index', or 'columns'."
177 )
178 elif orient in ["index", "columns"] and index is False:
179 raise ValueError(
180 "'index=False' is only valid when 'orient' is 'split', 'table', "
181 "'records', or 'values'."
182 )
183 elif index is None:
184 # will be ignored for orient='records' and 'values'
185 index = True
187 if lines and orient != "records":
188 raise ValueError("'lines' keyword only valid when 'orient' is records")
190 if mode not in ["a", "w"]:
191 msg = (
192 f"mode={mode} is not a valid option."
193 "Only 'w' and 'a' are currently supported."
194 )
195 raise ValueError(msg)
197 if mode == "a" and (not lines or orient != "records"):
198 msg = (
199 "mode='a' (append) is only supported when "
200 "lines is True and orient is 'records'"
201 )
202 raise ValueError(msg)
204 if orient == "table" and isinstance(obj, Series):
205 obj = obj.to_frame(name=obj.name or "values")
207 if date_format == "epoch":
208 # for epoch (numeric) format, convert datetime-likes to the desired
209 # unit up front, such that the C ObjToJSON code can simply write out
210 # the integer values without worrying about conversion
211 if date_unit not in ["s", "ms", "us", "ns"]:
212 raise ValueError(f"Invalid value '{date_unit}' for option 'date_unit'")
213 if isinstance(obj, DataFrame):
214 copied = False
215 cols = np.nonzero(obj.dtypes.map(_has_dt_accessor))[0]
216 if len(cols):
217 obj = obj.copy(deep=False)
218 copied = True
219 for col in cols:
220 obj.isetitem(col, obj.iloc[:, col].dt.as_unit(date_unit))
221 if _has_dt_accessor(obj.index.dtype):
222 if not copied:
223 obj = obj.copy(deep=False)
224 copied = True
225 obj.index = Series(obj.index).dt.as_unit(date_unit)
226 if _has_dt_accessor(obj.columns.dtype):
227 if not copied:
228 obj = obj.copy(deep=False)
229 copied = True
230 obj.columns = Series(obj.columns).dt.as_unit(date_unit)
231 elif isinstance(obj, Series):
232 if _has_dt_accessor(obj.dtype):
233 obj = obj.copy(deep=False)
234 obj = obj.dt.as_unit(date_unit)
235 if _has_dt_accessor(obj.index.dtype):
236 obj = obj.copy(deep=False)
237 obj.index = Series(obj.index).dt.as_unit(date_unit)
239 writer: type[Writer]
240 if orient == "table" and isinstance(obj, DataFrame):
241 writer = JSONTableWriter
242 elif isinstance(obj, Series):
243 writer = SeriesWriter
244 elif isinstance(obj, DataFrame):
245 writer = FrameWriter
246 else:
247 raise NotImplementedError("'obj' should be a Series or a DataFrame")
249 s = writer(
250 obj,
251 orient=orient,
252 date_format=date_format,
253 double_precision=double_precision,
254 ensure_ascii=force_ascii,
255 date_unit=date_unit,
256 default_handler=default_handler,
257 index=index,
258 indent=indent,
259 ).write()
261 if lines:
262 s = convert_to_line_delimits(s)
264 if path_or_buf is not None:
265 # apply compression and byte/text conversion
266 with get_handle(
267 path_or_buf, mode, compression=compression, storage_options=storage_options
268 ) as handles:
269 handles.handle.write(s)
270 else:
271 return s
272 return None
275class Writer(ABC):
276 _default_orient: str
278 def __init__(
279 self,
280 obj: NDFrame,
281 orient: str | None,
282 date_format: str,
283 double_precision: int,
284 ensure_ascii: bool,
285 date_unit: str,
286 index: bool,
287 default_handler: Callable[[Any], JSONSerializable] | None = None,
288 indent: int = 0,
289 ) -> None:
290 self.obj = obj
292 if orient is None:
293 orient = self._default_orient
295 self.orient = orient
296 self.date_format = date_format
297 self.double_precision = double_precision
298 self.ensure_ascii = ensure_ascii
299 self.date_unit = date_unit
300 self.default_handler = default_handler
301 self.index = index
302 self.indent = indent
303 self._format_axes()
305 def _format_axes(self) -> None:
306 raise AbstractMethodError(self)
308 def write(self) -> str:
309 iso_dates = self.date_format == "iso"
310 try:
311 return ujson_dumps(
312 self.obj_to_write,
313 orient=self.orient,
314 double_precision=self.double_precision,
315 ensure_ascii=self.ensure_ascii,
316 date_unit=self.date_unit,
317 iso_dates=iso_dates,
318 default_handler=self.default_handler,
319 indent=self.indent,
320 )
321 except OverflowError as err:
322 # GH#36211 the C encoder recurses without bound on object types it
323 # does not understand.
324 if "Maximum recursion level reached" not in str(err):
325 raise
326 raise ValueError(
327 "Unable to serialize object to JSON: encountered an "
328 "unsupported object type; convert the column to a supported "
329 "type (e.g. str) before calling to_json."
330 ) from err
332 @property
333 @abstractmethod
334 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
335 """Object to write in JSON format."""
338class SeriesWriter(Writer):
339 _default_orient = "index"
341 @property
342 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
343 if not self.index and self.orient == "split":
344 return {"name": self.obj.name, "data": self.obj.values}
345 else:
346 return self.obj
348 def _format_axes(self) -> None:
349 if not self.obj.index.is_unique and self.orient == "index":
350 raise ValueError(f"Series index must be unique for orient='{self.orient}'")
353class FrameWriter(Writer):
354 _default_orient = "columns"
356 @property
357 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
358 if not self.index and self.orient == "split":
359 obj_to_write = self.obj.to_dict(orient="split")
360 del obj_to_write["index"]
361 else:
362 obj_to_write = self.obj
363 return obj_to_write
365 def _format_axes(self) -> None:
366 """
367 Try to format axes if they are datelike.
368 """
369 if not self.obj.index.is_unique and self.orient in ("index", "columns"):
370 raise ValueError(
371 f"DataFrame index must be unique for orient='{self.orient}'."
372 )
373 if not self.obj.columns.is_unique and self.orient in (
374 "index",
375 "columns",
376 "records",
377 ):
378 raise ValueError(
379 f"DataFrame columns must be unique for orient='{self.orient}'."
380 )
383class JSONTableWriter(FrameWriter):
384 _default_orient = "records"
386 def __init__(
387 self,
388 obj,
389 orient: str | None,
390 date_format: str,
391 double_precision: int,
392 ensure_ascii: bool,
393 date_unit: str,
394 index: bool,
395 default_handler: Callable[[Any], JSONSerializable] | None = None,
396 indent: int = 0,
397 ) -> None:
398 """
399 Adds a `schema` attribute with the Table Schema, resets
400 the index (can't do in caller, because the schema inference needs
401 to know what the index is, forces orient to records, and forces
402 date_format to 'iso'.
403 """
404 super().__init__(
405 obj,
406 orient,
407 date_format,
408 double_precision,
409 ensure_ascii,
410 date_unit,
411 index,
412 default_handler=default_handler,
413 indent=indent,
414 )
416 if date_format != "iso":
417 msg = (
418 "Trying to write with `orient='table'` and "
419 f"`date_format='{date_format}'`. Table Schema requires dates "
420 "to be formatted with `date_format='iso'`"
421 )
422 raise ValueError(msg)
424 self.schema = build_table_schema(obj, index=self.index)
425 if self.index:
426 obj = set_default_names(obj)
428 # NotImplemented on a column MultiIndex
429 if obj.ndim == 2 and isinstance(obj.columns, MultiIndex):
430 raise NotImplementedError(
431 "orient='table' is not supported for MultiIndex columns"
432 )
434 # TODO: Do this timedelta properly in objToJSON.c See GH #15137
435 if ((obj.ndim == 1) and (obj.name in set(obj.index.names))) or len(
436 obj.columns.intersection(obj.index.names)
437 ):
438 msg = "Overlapping names between the index and columns"
439 raise ValueError(msg)
441 timedeltas = obj.select_dtypes(include=["timedelta"]).columns
442 copied = False
443 if len(timedeltas):
444 obj = obj.copy()
445 copied = True
446 obj[timedeltas] = obj[timedeltas].map(lambda x: x.isoformat())
448 # exclude index from obj if index=False
449 if not self.index:
450 self.obj = obj.reset_index(drop=True)
451 else:
452 # Convert PeriodIndex to datetimes before serializing
453 if isinstance(obj.index.dtype, PeriodDtype):
454 if not copied:
455 obj = obj.copy(deep=False)
456 obj.index = obj.index.to_timestamp()
457 self.obj = obj.reset_index(drop=False)
458 self.date_format = "iso"
459 self.orient = "records"
460 self.index = index
462 @property
463 def obj_to_write(self) -> NDFrame | Mapping[IndexLabel, Any]:
464 return {"schema": self.schema, "data": self.obj}
467def _validate_pyarrow_engine_options(
468 *,
469 typ,
470 dtype,
471 convert_axes,
472 convert_dates,
473 keep_default_dates,
474 precise_float,
475 date_unit,
476 encoding,
477 encoding_errors,
478 compression,
479 storage_options,
480) -> None:
481 """
482 Reject ``read_json`` arguments the ``"pyarrow"`` engine cannot honor.
484 The pyarrow engine hands the input straight to ``pyarrow.json.read_json``,
485 so any argument that pandas would otherwise apply itself is silently
486 ignored, producing a wrong result rather than an error. Raise instead.
488 Called from ``read_json`` before ``convert_axes`` and ``dtype`` are
489 resolved from ``None`` to ``True`` (and ``convert_dates`` /
490 ``keep_default_dates`` from ``lib.no_default``), so that an
491 explicitly-passed value is distinguishable from the default.
492 """
494 def raise_unsupported(argname: str) -> None:
495 raise ValueError(
496 f"The {argname!r} option is not supported with the 'pyarrow' engine"
497 )
499 if typ != "frame":
500 raise_unsupported("typ")
501 if convert_axes is not None:
502 raise_unsupported("convert_axes")
503 if convert_dates is not True and convert_dates is not lib.no_default:
504 raise_unsupported("convert_dates")
505 if keep_default_dates is not True and keep_default_dates is not lib.no_default:
506 raise_unsupported("keep_default_dates")
507 if precise_float is not False:
508 raise_unsupported("precise_float")
509 if date_unit is not None:
510 raise_unsupported("date_unit")
511 if storage_options is not None:
512 raise_unsupported("storage_options")
513 if compression != "infer":
514 # pyarrow infers compression from the file extension itself; pandas
515 # cannot honor an explicit method (or dict) or disable inference.
516 raise_unsupported("compression")
517 if encoding is not None and encoding.lower() not in ("utf-8", "utf8"):
518 # pyarrow assumes UTF-8 and ignores the encoding argument entirely.
519 raise_unsupported("encoding")
520 if encoding_errors is not None and encoding_errors != "strict":
521 raise_unsupported("encoding_errors")
523 if isinstance(dtype, dict):
524 # GH#59516 supports a mapping of ArrowDtype; any other dtype in the
525 # mapping is silently dropped, so reject it.
526 for field, field_dtype in dtype.items():
527 if not isinstance(pandas_dtype(field_dtype), ArrowDtype):
528 raise ValueError(
529 "The 'pyarrow' engine only supports ArrowDtype values in "
530 f"the 'dtype' mapping, got {field_dtype!r} for column "
531 f"{field!r}"
532 )
533 elif dtype is not None:
534 # a scalar dtype (including True/False) applies to every column but is
535 # silently ignored by the pyarrow engine.
536 raise_unsupported("dtype")
539@overload
540def read_json(
541 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
542 *,
543 orient: str | None = ...,
544 typ: Literal["frame"] = ...,
545 dtype: DtypeArg | None = ...,
546 convert_axes: bool | None = ...,
547 convert_dates: bool | list[str] | lib.NoDefault = ...,
548 keep_default_dates: bool | lib.NoDefault = ...,
549 precise_float: bool = ...,
550 date_unit: str | None = ...,
551 encoding: str | None = ...,
552 encoding_errors: str | None = ...,
553 lines: bool = ...,
554 chunksize: int,
555 compression: CompressionOptions = ...,
556 nrows: int | None = ...,
557 storage_options: StorageOptions = ...,
558 dtype_backend: DtypeBackend | lib.NoDefault = ...,
559 engine: JSONEngine = ...,
560) -> JsonReader[Literal["frame"]]: ...
563@overload
564def read_json(
565 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
566 *,
567 orient: str | None = ...,
568 typ: Literal["series"],
569 dtype: DtypeArg | None = ...,
570 convert_axes: bool | None = ...,
571 convert_dates: bool | list[str] | lib.NoDefault = ...,
572 keep_default_dates: bool | lib.NoDefault = ...,
573 precise_float: bool = ...,
574 date_unit: str | None = ...,
575 encoding: str | None = ...,
576 encoding_errors: str | None = ...,
577 lines: bool = ...,
578 chunksize: int,
579 compression: CompressionOptions = ...,
580 nrows: int | None = ...,
581 storage_options: StorageOptions = ...,
582 dtype_backend: DtypeBackend | lib.NoDefault = ...,
583 engine: JSONEngine = ...,
584) -> JsonReader[Literal["series"]]: ...
587@overload
588def read_json(
589 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
590 *,
591 orient: str | None = ...,
592 typ: Literal["series"],
593 dtype: DtypeArg | None = ...,
594 convert_axes: bool | None = ...,
595 convert_dates: bool | list[str] | lib.NoDefault = ...,
596 keep_default_dates: bool | lib.NoDefault = ...,
597 precise_float: bool = ...,
598 date_unit: str | None = ...,
599 encoding: str | None = ...,
600 encoding_errors: str | None = ...,
601 lines: bool = ...,
602 chunksize: None = ...,
603 compression: CompressionOptions = ...,
604 nrows: int | None = ...,
605 storage_options: StorageOptions = ...,
606 dtype_backend: DtypeBackend | lib.NoDefault = ...,
607 engine: JSONEngine = ...,
608) -> Series: ...
611@overload
612def read_json(
613 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
614 *,
615 orient: str | None = ...,
616 typ: Literal["frame"] = ...,
617 dtype: DtypeArg | None = ...,
618 convert_axes: bool | None = ...,
619 convert_dates: bool | list[str] | lib.NoDefault = ...,
620 keep_default_dates: bool | lib.NoDefault = ...,
621 precise_float: bool = ...,
622 date_unit: str | None = ...,
623 encoding: str | None = ...,
624 encoding_errors: str | None = ...,
625 lines: bool = ...,
626 chunksize: None = ...,
627 compression: CompressionOptions = ...,
628 nrows: int | None = ...,
629 storage_options: StorageOptions = ...,
630 dtype_backend: DtypeBackend | lib.NoDefault = ...,
631 engine: JSONEngine = ...,
632) -> DataFrame: ...
635@set_module("pandas")
636def read_json(
637 path_or_buf: FilePath | ReadBuffer[str] | ReadBuffer[bytes],
638 *,
639 orient: str | None = None,
640 typ: Literal["frame", "series"] = "frame",
641 dtype: DtypeArg | None = None,
642 convert_axes: bool | None = None,
643 convert_dates: bool | list[str] | lib.NoDefault = lib.no_default,
644 keep_default_dates: bool | lib.NoDefault = lib.no_default,
645 precise_float: bool = False,
646 date_unit: str | None = None,
647 encoding: str | None = None,
648 encoding_errors: str | None = "strict",
649 lines: bool = False,
650 chunksize: int | None = None,
651 compression: CompressionOptions = "infer",
652 nrows: int | None = None,
653 storage_options: StorageOptions | None = None,
654 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
655 engine: JSONEngine = "ujson",
656) -> DataFrame | Series | JsonReader[Any]:
657 """
658 Convert a JSON string to pandas object.
660 This method reads JSON files or JSON-like data and converts them into pandas
661 objects. It supports a variety of input formats, including line-delimited JSON,
662 compressed files, and various data representations (table, records, index-based,
663 etc.). When `chunksize` is specified, an iterator is returned instead of loading
664 the entire data into memory.
666 Parameters
667 ----------
668 path_or_buf : a str path, path object or file-like object
669 Any valid string path is acceptable. The string could be a URL. Valid
670 URL schemes include http, ftp, s3, and file. For file URLs, a host is
671 expected. A local file could be:
672 ``file://localhost/path/to/table.json``.
674 Certain URL schemes may require additional packages. For example, S3
675 URLs require the ``s3fs`` library. See
676 :ref:`install.optional_dependencies` for a full list.
678 If you want to pass in a path object, pandas accepts any
679 ``os.PathLike``.
681 By file-like object, we refer to objects with a ``read()`` method,
682 such as a file handle (e.g. via builtin ``open`` function)
683 or ``StringIO``.
685 orient : str, optional
686 Indication of expected JSON string format.
687 Compatible JSON strings can be produced by ``to_json()`` with a
688 corresponding orient value.
689 When ``typ == 'frame'``, the set of possible orients is:
691 - ``'split'`` : dict like
692 ``{index -> [index], columns -> [columns], data -> [values]}``
693 - ``'records'`` : list like
694 ``[{column -> value}, ... , {column -> value}]``
695 - ``'index'`` : dict like ``{index -> {column -> value}}``
696 - ``'columns'`` : dict like ``{column -> {index -> value}}``
697 - ``'values'`` : just the values array
698 - ``'table'`` : dict like ``{'schema': {schema}, 'data': {data}}``
700 When ``typ == 'series'``, the shapes differ, since a Series has no
701 columns:
703 - ``'split'`` : dict like
704 ``{name -> name, index -> [index], data -> [values]}``
705 - ``'records'`` : list like ``[value, ... , value]``
706 - ``'index'`` : dict like ``{index -> value}``
708 The allowed and default values depend on the value
709 of the `typ` parameter.
711 * when ``typ == 'series'``,
713 - allowed orients are ``{'split','records','index'}``
714 - default is ``'index'``
715 - The Series index must be unique for orient ``'index'``.
717 * when ``typ == 'frame'``,
719 - allowed orients are ``{'split','records','index',
720 'columns','values', 'table'}``
721 - default is ``'columns'``
722 - The DataFrame index must be unique for orients ``'index'`` and
723 ``'columns'``.
724 - The DataFrame columns must be unique for orients ``'index'``,
725 ``'columns'``, and ``'records'``.
727 typ : {'frame', 'series'}, default 'frame'
728 The type of object to recover.
730 dtype : bool or dict, default None
731 If True, infer dtypes; if a dict of column to dtype, then use those;
732 if False, then don't infer dtypes at all, applies only to the data.
734 For all ``orient`` values except ``'table'``, default is True.
736 convert_axes : bool, default None
737 Try to convert the axes to the proper dtypes.
739 For all ``orient`` values except ``'table'``, default is True.
741 convert_dates : bool or list of str, default True
742 If True then default datelike columns may be converted (depending on
743 keep_default_dates).
744 If False, no dates will be converted.
745 If a list of column names, then those columns will be converted and
746 default datelike columns may also be converted (depending on
747 keep_default_dates).
749 .. deprecated:: 3.1.0
750 Pass ``dtype=False`` to disable type conversion, or parse date
751 columns with :func:`~pandas.to_datetime` after reading.
753 keep_default_dates : bool, default True
754 If parsing dates (convert_dates is not False), then try to parse the
755 default datelike columns.
756 A column label is datelike if
758 * it ends with ``'_at'``,
760 * it ends with ``'_time'``,
762 * it begins with ``'timestamp'``,
764 * it is ``'modified'``, or
766 * it is ``'date'``.
768 .. deprecated:: 3.1.0
769 Pass ``dtype=False`` to disable type conversion, or parse date
770 columns with :func:`~pandas.to_datetime` after reading.
772 precise_float : bool, default False
773 Set to enable usage of higher precision (strtod) function when
774 decoding string to double values. Default (False) is to use fast but
775 less precise builtin functionality.
777 date_unit : str, default None
778 The timestamp unit to detect if converting dates. The default behaviour
779 is to try and detect the correct precision, but if this is not desired
780 then pass one of 's', 'ms', 'us' or 'ns' to force parsing only seconds,
781 milliseconds, microseconds or nanoseconds respectively.
783 encoding : str, default is 'utf-8'
784 The encoding to use to decode py3 bytes.
786 encoding_errors : str, optional, default "strict"
787 How encoding errors are treated. `List of possible values
788 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
790 lines : bool, default False
791 Read the file as a json object per line.
793 chunksize : int, optional
794 Return JsonReader object for iteration.
795 See the `line-delimited json docs
796 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#line-delimited-json>`_
797 for more information on ``chunksize``.
798 This can only be passed if `lines=True`.
799 If this is None, the file will be read into memory all at once.
801 compression : str or dict, default 'infer'
802 For on-the-fly decompression of on-disk data. If 'infer' and 'path_or_buf' is
803 path-like, then detect compression from the following extensions: '.gz',
804 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
805 (otherwise no compression).
806 If using 'zip' or 'tar', the ZIP file must contain only one data file to be
807 read in.
808 Set to ``None`` for no decompression.
809 Can also be a dict with key ``'method'`` set
810 to one of {``'zip'``, ``'gzip'``, ``'bz2'``, ``'zstd'``, ``'xz'``, ``'tar'``}
811 and other key-value pairs are forwarded to
812 ``zipfile.ZipFile``, ``gzip.GzipFile``,
813 ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
814 ``tarfile.TarFile``, respectively.
815 As an example, the following could be passed for Zstandard decompression using a
816 custom compression dictionary:
817 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
819 nrows : int, optional
820 The number of lines from the line-delimited jsonfile that has to be read.
821 This can only be passed if `lines=True`.
822 If this is None, all the rows will be returned.
824 storage_options : dict, optional
825 Extra options that make sense for a particular storage connection, e.g.
826 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
827 are forwarded to ``urllib.request.Request`` as header options. For other
828 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
829 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
830 details, and for more examples on storage options refer `here
831 <https://pandas.pydata.org/docs/user_guide/io.html?
832 highlight=storage_options#reading-writing-remote-files>`_.
834 dtype_backend : {'numpy_nullable', 'pyarrow'}
835 Back-end data type applied to the resultant :class:`DataFrame`
836 (still experimental). If not specified, the default behavior
837 is to not use nullable data types. If specified, the behavior
838 is as follows:
840 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
841 * ``"pyarrow"``: returns pyarrow-backed nullable
842 :class:`ArrowDtype` :class:`DataFrame`
844 .. versionadded:: 2.0
846 engine : {"ujson", "pyarrow"}, default "ujson"
847 Parser engine to use. The ``"pyarrow"`` engine is only available when
848 ``lines=True``, and only supports ``typ="frame"`` with default parsing
849 options; passing an option it cannot apply (for example ``convert_dates``,
850 ``date_unit``, ``precise_float``, ``encoding``, or a non-``ArrowDtype``
851 ``dtype``) raises ``ValueError``.
853 .. versionadded:: 2.0
855 Returns
856 -------
857 Series, DataFrame, or pandas.api.typing.JsonReader
858 A JsonReader is returned when ``chunksize`` is not ``0`` or ``None``.
859 Otherwise, the type returned depends on the value of ``typ``.
861 See Also
862 --------
863 DataFrame.to_json : Convert a DataFrame to a JSON string.
864 Series.to_json : Convert a Series to a JSON string.
865 json_normalize : Normalize semi-structured JSON data into a flat table.
867 Notes
868 -----
869 Specific to ``orient='table'``, if a :class:`DataFrame` with a literal
870 :class:`Index` name of `index` gets written with :func:`to_json`, the
871 subsequent read operation will incorrectly set the :class:`Index` name to
872 ``None``. This is because `index` is also used by :func:`DataFrame.to_json`
873 to denote a missing :class:`Index` name, and the subsequent
874 :func:`read_json` operation cannot distinguish between the two. The same
875 limitation is encountered with a :class:`MultiIndex` and any names
876 beginning with ``'level_'``.
878 Examples
879 --------
880 >>> from io import StringIO
881 >>> df = pd.DataFrame(
882 ... [["a", "b"], ["c", "d"]],
883 ... index=["row 1", "row 2"],
884 ... columns=["col 1", "col 2"],
885 ... )
887 Encoding/decoding a Dataframe using ``'split'`` formatted JSON:
889 >>> df.to_json(orient="split")
890 '{"columns":["col 1","col 2"],"index":["row 1","row 2"],"data":[["a","b"],["c","d"]]}'
892 >>> pd.read_json(StringIO(_), orient="split") # noqa: F821
893 col 1 col 2
894 row 1 a b
895 row 2 c d
897 Encoding/decoding a Dataframe using ``'index'`` formatted JSON:
899 >>> df.to_json(orient="index")
900 '{"row 1":{"col 1":"a","col 2":"b"},"row 2":{"col 1":"c","col 2":"d"}}'
902 >>> pd.read_json(StringIO(_), orient="index") # noqa: F821
903 col 1 col 2
904 row 1 a b
905 row 2 c d
907 Encoding/decoding a Dataframe using ``'records'`` formatted JSON.
908 Note that index labels are not preserved with this encoding.
910 >>> df.to_json(orient="records")
911 '[{"col 1":"a","col 2":"b"},{"col 1":"c","col 2":"d"}]'
913 >>> pd.read_json(StringIO(_), orient="records") # noqa: F821
914 col 1 col 2
915 0 a b
916 1 c d
918 Encoding with Table Schema
920 >>> df.to_json(orient="table")
921 '{"schema":{"fields":[{"name":"index","type":"string","extDtype":"str"},{"name":"col 1","type":"string","extDtype":"str"},{"name":"col 2","type":"string","extDtype":"str"}],"primaryKey":["index"],"pandas_version":"1.4.0"},"data":[{"index":"row 1","col 1":"a","col 2":"b"},{"index":"row 2","col 1":"c","col 2":"d"}]}'
923 The following example uses ``dtype_backend="numpy_nullable"``
925 >>> data = '''{"index": {"0": 0, "1": 1},
926 ... "a": {"0": 1, "1": null},
927 ... "b": {"0": 2.5, "1": 4.5},
928 ... "c": {"0": true, "1": false},
929 ... "d": {"0": "a", "1": "b"},
930 ... "e": {"0": 1577.2, "1": 1577.1}}'''
931 >>> pd.read_json(StringIO(data), dtype_backend="numpy_nullable")
932 index a b c d e
933 0 0 1 2.5 True a 1577.2
934 1 1 <NA> 4.5 False b 1577.1
935 """ # noqa: E501
936 if orient == "table" and dtype:
937 raise ValueError("cannot pass both dtype and orient='table'")
938 if orient == "table" and convert_axes:
939 raise ValueError("cannot pass both convert_axes and orient='table'")
941 check_dtype_backend(dtype_backend)
943 if engine == "pyarrow":
944 # Validate before dtype/convert_axes are resolved from None to True
945 # below, so an explicitly-passed value can be told apart from the
946 # default.
947 _validate_pyarrow_engine_options(
948 typ=typ,
949 dtype=dtype,
950 convert_axes=convert_axes,
951 convert_dates=convert_dates,
952 keep_default_dates=keep_default_dates,
953 precise_float=precise_float,
954 date_unit=date_unit,
955 encoding=encoding,
956 encoding_errors=encoding_errors,
957 compression=compression,
958 storage_options=storage_options,
959 )
961 if convert_dates is not lib.no_default:
962 warnings.warn(
963 "The 'convert_dates' keyword in read_json is deprecated and will be "
964 "removed in a future version. Pass dtype=False to disable type "
965 "conversion, or parse date columns with pd.to_datetime after reading.",
966 Pandas4Warning,
967 stacklevel=find_stack_level(),
968 )
969 else:
970 convert_dates = True
972 if keep_default_dates is not lib.no_default:
973 warnings.warn(
974 "The 'keep_default_dates' keyword in read_json is deprecated and will "
975 "be removed in a future version. Pass dtype=False to disable type "
976 "conversion, or parse date columns with pd.to_datetime after reading.",
977 Pandas4Warning,
978 stacklevel=find_stack_level(),
979 )
980 else:
981 keep_default_dates = True
983 if dtype is None and orient != "table":
984 # error: Incompatible types in assignment (expression has type "bool", variable
985 # has type "Union[ExtensionDtype, str, dtype[Any], Type[str], Type[float],
986 # Type[int], Type[complex], Type[bool], Type[object], Dict[Hashable,
987 # Union[ExtensionDtype, Union[str, dtype[Any]], Type[str], Type[float],
988 # Type[int], Type[complex], Type[bool], Type[object]]], None]")
989 dtype = True # type: ignore[assignment]
990 if convert_axes is None and orient != "table":
991 convert_axes = True
993 json_reader = JsonReader(
994 path_or_buf,
995 orient=orient,
996 typ=typ,
997 dtype=dtype,
998 convert_axes=convert_axes,
999 convert_dates=convert_dates,
1000 keep_default_dates=keep_default_dates,
1001 precise_float=precise_float,
1002 date_unit=date_unit,
1003 encoding=encoding,
1004 lines=lines,
1005 chunksize=chunksize,
1006 compression=compression,
1007 nrows=nrows,
1008 storage_options=storage_options,
1009 encoding_errors=encoding_errors,
1010 dtype_backend=dtype_backend,
1011 engine=engine,
1012 )
1014 if chunksize:
1015 return json_reader
1016 else:
1017 return json_reader.read()
1020@set_module("pandas.api.typing")
1021class JsonReader(abc.Iterator[DataFrame | Series], Generic[FrameSeriesStrT]):
1022 """
1023 JsonReader provides an interface for reading in a JSON file.
1025 If initialized with ``lines=True`` and ``chunksize``, can be iterated over
1026 ``chunksize`` lines at a time. Otherwise, calling ``read`` reads in the
1027 whole document.
1029 A ``JsonReader`` is returned by :func:`~pandas.read_json` when
1030 ``chunksize`` is passed; it is not usually instantiated directly.
1032 Parameters
1033 ----------
1034 filepath_or_buffer : a str path, path object or file-like object
1035 Any valid string path is acceptable. The string could be a URL. Valid
1036 URL schemes include http, ftp, s3, and file. pandas accepts any
1037 ``os.PathLike``, or file-like object with a ``read()`` method, such
1038 as a file handle or ``StringIO``.
1039 orient : str
1040 Indication of expected JSON string format. Compatible JSON strings
1041 can be produced by ``to_json()`` with a corresponding orient value.
1042 See :func:`~pandas.read_json` for the set of possible orients.
1043 typ : {"frame", "series"}
1044 The type of object to recover.
1045 dtype : bool or dict
1046 If True, infer dtypes; if a dict of column to dtype, then use those;
1047 if False, then don't infer dtypes at all, applies only to the data.
1048 convert_axes : bool or None
1049 Try to convert the axes to the proper dtypes.
1050 convert_dates : bool or list of str
1051 If True then default datelike columns may be converted (depending on
1052 ``keep_default_dates``). If False, no dates will be converted. If a
1053 list of column names, then those columns will be converted and
1054 default datelike columns may also be converted (depending on
1055 ``keep_default_dates``).
1056 keep_default_dates : bool
1057 If parsing dates (``convert_dates`` is not False), then try to parse
1058 the default datelike columns. See :func:`~pandas.read_json` for the
1059 criteria used to determine whether a column label is datelike.
1060 precise_float : bool
1061 Set to enable usage of higher precision (strtod) function when
1062 decoding string to double values.
1063 date_unit : str or None
1064 The timestamp unit to detect if converting dates. The default
1065 behaviour is to try and detect the correct precision, but if this
1066 is not desired then pass one of 's', 'ms', 'us' or 'ns' to force
1067 parsing only seconds, milliseconds, microseconds or nanoseconds
1068 respectively.
1069 encoding : str or None
1070 The encoding to use to decode py3 bytes.
1071 lines : bool
1072 Read the file as a json object per line.
1073 chunksize : int or None
1074 Return a ``JsonReader`` for iteration, yielding ``chunksize`` lines
1075 at a time. Can only be passed if ``lines=True``. If this is None,
1076 the file will be read into memory all at once.
1077 compression : str or dict
1078 For on-the-fly decompression of on-disk data, see
1079 :func:`~pandas.read_json` for the accepted values.
1080 nrows : int or None
1081 The number of lines from the line-delimited json file that has to
1082 be read. Can only be passed if ``lines=True``. If this is None, all
1083 the rows will be returned.
1084 storage_options : dict, optional
1085 Extra options that make sense for a particular storage connection,
1086 e.g. host, port, username, password, etc. See
1087 :func:`~pandas.read_json` for more details.
1088 encoding_errors : str, optional, default "strict"
1089 How encoding errors are treated. `List of possible values
1090 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
1091 dtype_backend : {"numpy_nullable", "pyarrow"}
1092 Back-end data type applied to the resultant :class:`DataFrame`
1093 (still experimental). See :func:`~pandas.read_json` for more
1094 details.
1095 engine : {"ujson", "pyarrow"}, default "ujson"
1096 Parser engine to use. See :func:`~pandas.read_json` for the
1097 restrictions on the ``"pyarrow"`` engine.
1099 See Also
1100 --------
1101 read_json : Convert a JSON string to pandas object.
1103 Examples
1104 --------
1105 >>> from io import StringIO
1106 >>> df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
1107 >>> reader = pd.read_json(
1108 ... StringIO(df.to_json(orient="records", lines=True)),
1109 ... lines=True,
1110 ... chunksize=1,
1111 ... )
1112 >>> for chunk in reader:
1113 ... print(chunk)
1114 a b
1115 0 1 3
1116 a b
1117 1 2 4
1118 """
1120 def __init__(
1121 self,
1122 filepath_or_buffer,
1123 orient,
1124 typ: FrameSeriesStrT,
1125 dtype,
1126 convert_axes: bool | None,
1127 convert_dates,
1128 keep_default_dates: bool,
1129 precise_float: bool,
1130 date_unit,
1131 encoding,
1132 lines: bool,
1133 chunksize: int | None,
1134 compression: CompressionOptions,
1135 nrows: int | None,
1136 storage_options: StorageOptions | None = None,
1137 encoding_errors: str | None = "strict",
1138 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1139 engine: JSONEngine = "ujson",
1140 ) -> None:
1141 self.orient = orient
1142 self.typ = typ
1143 self.dtype = dtype
1144 self.convert_axes = convert_axes
1145 self.convert_dates = convert_dates
1146 self.keep_default_dates = keep_default_dates
1147 self.precise_float = precise_float
1148 self.date_unit = date_unit
1149 self.encoding = encoding
1150 self.engine = engine
1151 self.compression = compression
1152 self.storage_options = storage_options
1153 self.lines = lines
1154 self.chunksize = chunksize
1155 self.nrows_seen = 0
1156 self.nrows = nrows
1157 self.encoding_errors = encoding_errors
1158 self.handles: IOHandles[str] | None = None
1159 self.dtype_backend = dtype_backend
1161 if self.engine not in {"pyarrow", "ujson"}:
1162 raise ValueError(
1163 f"The engine type {self.engine} is currently not supported."
1164 )
1165 if self.chunksize is not None:
1166 self.chunksize = validate_integer("chunksize", self.chunksize, 1)
1167 if not self.lines:
1168 raise ValueError("chunksize can only be passed if lines=True")
1169 if self.engine == "pyarrow":
1170 raise ValueError(
1171 "currently pyarrow engine doesn't support chunksize parameter"
1172 )
1173 if self.nrows is not None:
1174 self.nrows = validate_integer("nrows", self.nrows, 0)
1175 if not self.lines:
1176 raise ValueError("nrows can only be passed if lines=True")
1177 if self.engine == "pyarrow":
1178 raise NotImplementedError(
1179 "currently pyarrow engine doesn't support nrows parameter"
1180 )
1181 if self.engine == "pyarrow":
1182 if not self.lines:
1183 raise ValueError(
1184 "currently pyarrow engine only supports "
1185 "the line-delimited JSON format"
1186 )
1187 self.data = filepath_or_buffer
1188 elif self.engine == "ujson":
1189 data = self._get_data_from_filepath(filepath_or_buffer)
1190 # If self.chunksize, we prepare the data for the `__next__` method.
1191 # Otherwise, we read it into memory for the `read` method.
1192 if self.chunksize is None and self.nrows is None:
1193 with self:
1194 self.data = data.read()
1195 else:
1196 self.data = data
1198 def _get_data_from_filepath(self, filepath_or_buffer):
1199 """
1200 The function read_json accepts three input types:
1201 1. filepath (string-like)
1202 2. file-like object (e.g. open file object, StringIO)
1203 """
1204 filepath_or_buffer = stringify_path(filepath_or_buffer)
1205 self.handles = get_handle(
1206 filepath_or_buffer,
1207 "r",
1208 encoding=self.encoding,
1209 compression=self.compression,
1210 storage_options=self.storage_options,
1211 errors=self.encoding_errors,
1212 )
1213 filepath_or_buffer = self.handles.handle
1214 return filepath_or_buffer
1216 def _combine_lines(self, lines) -> str:
1217 """
1218 Combines a list of JSON objects into one JSON object.
1219 """
1220 return (
1221 f"[{','.join([line for line in (line.strip() for line in lines) if line])}]"
1222 )
1224 @overload
1225 def read(self: JsonReader[Literal["frame"]]) -> DataFrame: ...
1227 @overload
1228 def read(self: JsonReader[Literal["series"]]) -> Series: ...
1230 @overload
1231 def read(self: JsonReader[Literal["frame", "series"]]) -> DataFrame | Series: ...
1233 def read(self) -> DataFrame | Series:
1234 """
1235 Read the whole JSON input into a pandas object.
1237 Unlike iterating over the reader, this reads the entire underlying
1238 JSON input in a single call, regardless of whether ``chunksize``
1239 was specified when this reader was created.
1241 Returns
1242 -------
1243 DataFrame or Series
1244 The parsed pandas object, depending on the ``typ`` passed when
1245 this reader was created.
1247 See Also
1248 --------
1249 read_json : Convert a JSON string to pandas object.
1251 Examples
1252 --------
1253 >>> from io import StringIO
1254 >>> df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
1255 >>> reader = pd.read_json(
1256 ... StringIO(df.to_json(orient="records", lines=True)),
1257 ... lines=True,
1258 ... chunksize=2,
1259 ... )
1260 >>> reader.read()
1261 a b
1262 0 1 3
1263 1 2 4
1264 """
1265 obj: DataFrame | Series
1266 with self:
1267 if self.engine == "pyarrow":
1268 obj = self._read_pyarrow()
1269 elif self.engine == "ujson":
1270 obj = self._read_ujson()
1272 return obj
1274 def _read_pyarrow(self) -> DataFrame:
1275 """
1276 Read JSON using the pyarrow engine.
1277 """
1278 pyarrow_json = import_optional_dependency("pyarrow.json")
1279 options = None
1281 if isinstance(self.dtype, dict):
1282 pa = import_optional_dependency("pyarrow")
1283 fields = []
1284 for field, dtype in self.dtype.items():
1285 pd_dtype = pandas_dtype(dtype)
1286 if isinstance(pd_dtype, ArrowDtype):
1287 fields.append((field, pd_dtype.pyarrow_dtype))
1289 schema = pa.schema(fields)
1290 options = pyarrow_json.ParseOptions(
1291 explicit_schema=schema, unexpected_field_behavior="infer"
1292 )
1294 pa_table = pyarrow_json.read_json(self.data, parse_options=options)
1295 df = arrow_table_to_pandas(pa_table, dtype_backend=self.dtype_backend)
1297 return df
1299 def _read_ujson(self) -> DataFrame | Series:
1300 """
1301 Read JSON using the ujson engine.
1302 """
1303 obj: DataFrame | Series
1304 if self.lines:
1305 if self.chunksize:
1306 chunks = list(self)
1307 if chunks:
1308 obj = concat(chunks)
1309 else:
1310 obj = self._get_object_parser(self._combine_lines([]))
1311 elif self.nrows is not None:
1312 lines = list(islice(self.data, self.nrows))
1313 lines_json = self._combine_lines(lines)
1314 obj = self._get_object_parser(lines_json)
1315 else:
1316 data = ensure_str(self.data)
1317 data_lines = data.split("\n")
1318 obj = self._get_object_parser(self._combine_lines(data_lines))
1319 else:
1320 obj = self._get_object_parser(self.data)
1321 if self.dtype_backend is not lib.no_default:
1322 with option_context("future.distinguish_nan_and_na", False):
1323 return obj.convert_dtypes(dtype_backend=self.dtype_backend)
1324 else:
1325 return obj
1327 def _get_object_parser(self, json: str) -> DataFrame | Series:
1328 """
1329 Parses a json document into a pandas object.
1330 """
1331 typ = self.typ
1332 dtype = self.dtype
1333 kwargs = {
1334 "orient": self.orient,
1335 "dtype": self.dtype,
1336 "convert_axes": self.convert_axes,
1337 "convert_dates": self.convert_dates,
1338 "keep_default_dates": self.keep_default_dates,
1339 "precise_float": self.precise_float,
1340 "date_unit": self.date_unit,
1341 "dtype_backend": self.dtype_backend,
1342 }
1343 if typ == "frame":
1344 return FrameParser(json, **kwargs).parse()
1345 elif typ == "series":
1346 if not isinstance(dtype, bool):
1347 kwargs["dtype"] = dtype
1348 return SeriesParser(json, **kwargs).parse()
1349 else:
1350 raise ValueError(f"{typ=} must be 'frame' or 'series'.")
1352 def close(self) -> None:
1353 """
1354 Close the underlying file handle, if one was opened by the reader.
1356 If an open stream or file-like object was passed to the reader
1357 rather than a path, it is left open and this method does not
1358 close it.
1360 See Also
1361 --------
1362 read_json : Convert a JSON string to pandas object.
1364 Examples
1365 --------
1366 >>> from io import StringIO
1367 >>> df = pd.DataFrame({"a": [1, 2], "b": [3, 4]})
1368 >>> reader = pd.read_json(
1369 ... StringIO(df.to_json(orient="records", lines=True)),
1370 ... lines=True,
1371 ... chunksize=1,
1372 ... )
1373 >>> reader.close()
1374 """
1375 if self.handles is not None:
1376 self.handles.close()
1378 def __iter__(self) -> Self:
1379 return self
1381 @overload
1382 def __next__(self: JsonReader[Literal["frame"]]) -> DataFrame: ...
1384 @overload
1385 def __next__(self: JsonReader[Literal["series"]]) -> Series: ...
1387 @overload
1388 def __next__(
1389 self: JsonReader[Literal["frame", "series"]],
1390 ) -> DataFrame | Series: ...
1392 def __next__(self) -> DataFrame | Series:
1393 chunk_size: int | None
1394 if self.nrows is not None:
1395 remaining = self.nrows - self.nrows_seen
1396 if remaining <= 0:
1397 self.close()
1398 raise StopIteration
1399 if self.chunksize is None:
1400 chunk_size = remaining
1401 else:
1402 chunk_size = min(self.chunksize, remaining)
1403 else:
1404 chunk_size = self.chunksize
1406 lines = list(islice(self.data, chunk_size))
1407 if not lines:
1408 self.close()
1409 raise StopIteration
1411 try:
1412 lines_json = self._combine_lines(lines)
1413 obj = self._get_object_parser(lines_json)
1415 # Make sure that the returned objects have the right index.
1416 obj.index = range(self.nrows_seen, self.nrows_seen + len(obj))
1417 self.nrows_seen += len(obj)
1418 except Exception as ex:
1419 self.close()
1420 raise ex
1422 if self.dtype_backend is not lib.no_default:
1423 with option_context("future.distinguish_nan_and_na", False):
1424 return obj.convert_dtypes(dtype_backend=self.dtype_backend)
1425 else:
1426 return obj
1428 def __enter__(self) -> Self:
1429 return self
1431 def __exit__(
1432 self,
1433 exc_type: type[BaseException] | None,
1434 exc_value: BaseException | None,
1435 traceback: TracebackType | None,
1436 ) -> None:
1437 self.close()
1440class Parser:
1441 _split_keys: tuple[str, ...]
1442 _default_orient: str
1444 _STAMP_UNITS = ("s", "ms", "us", "ns")
1445 _MIN_STAMPS = {
1446 "s": 31536000,
1447 "ms": 31536000000,
1448 "us": 31536000000000,
1449 "ns": 31536000000000000,
1450 }
1451 json: str
1453 def __init__(
1454 self,
1455 json: str,
1456 orient,
1457 dtype: DtypeArg | None = None,
1458 convert_axes: bool = True,
1459 convert_dates: bool | list[str] = True,
1460 keep_default_dates: bool = False,
1461 precise_float: bool = False,
1462 date_unit=None,
1463 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1464 ) -> None:
1465 self.json = json
1467 if orient is None:
1468 orient = self._default_orient
1470 self.orient = orient
1472 self.dtype = dtype
1474 if date_unit is not None:
1475 date_unit = date_unit.lower()
1476 if date_unit not in self._STAMP_UNITS:
1477 raise ValueError(f"date_unit must be one of {self._STAMP_UNITS}")
1478 self.min_stamp = self._MIN_STAMPS[date_unit]
1479 else:
1480 self.min_stamp = self._MIN_STAMPS["s"]
1482 self.precise_float = precise_float
1483 self.convert_axes = convert_axes
1484 self.convert_dates = convert_dates
1485 self.date_unit = date_unit
1486 self.keep_default_dates = keep_default_dates
1487 self.dtype_backend = dtype_backend
1489 @final
1490 def check_keys_split(self, decoded: dict[str, Any]) -> None:
1491 """
1492 Checks that dict has only the appropriate keys for orient='split'.
1493 """
1494 bad_keys = set(decoded.keys()).difference(set(self._split_keys))
1495 if bad_keys:
1496 bad_keys_joined = ", ".join(bad_keys)
1497 raise ValueError(f"JSON data had unexpected key(s): {bad_keys_joined}")
1499 @final
1500 def parse(self) -> DataFrame | Series:
1501 obj = self._parse()
1503 if self.convert_axes:
1504 obj = self._convert_axes(obj)
1505 obj = self._try_convert_types(obj)
1506 return obj
1508 def _parse(self) -> DataFrame | Series:
1509 raise AbstractMethodError(self)
1511 @final
1512 def _convert_axes(self, obj: DataFrame | Series) -> DataFrame | Series:
1513 """
1514 Try to convert axes.
1515 """
1516 for axis_name in obj._AXIS_ORDERS:
1517 ax = obj._get_axis(axis_name)
1518 ser = Series(ax, dtype=ax.dtype, copy=False)
1519 new_ser, result = self._try_convert_data(
1520 name=axis_name,
1521 data=ser,
1522 use_dtypes=False,
1523 convert_dates=True,
1524 is_axis=True,
1525 )
1526 if result:
1527 new_axis = Index(new_ser, dtype=new_ser.dtype, copy=False)
1528 setattr(obj, axis_name, new_axis)
1529 return obj
1531 def _try_convert_types(self, obj):
1532 raise AbstractMethodError(self)
1534 @final
1535 def _try_convert_data(
1536 self,
1537 name: Hashable,
1538 data: Series,
1539 use_dtypes: bool = True,
1540 convert_dates: bool | list[str] = True,
1541 is_axis: bool = False,
1542 ) -> tuple[Series, bool]:
1543 """
1544 Try to parse a Series into a column by inferring dtype.
1545 """
1546 org_data = data
1547 # don't try to coerce, unless a force conversion
1548 if use_dtypes:
1549 if not self.dtype:
1550 if all(notna(data)):
1551 return data, False
1553 filled = data.fillna(np.nan)
1555 return filled, True
1556 # error: Non-overlapping identity check (left operand type: "ExtensionDtype
1557 # | str | dtype[Any] | type[object] | Mapping[Hashable, ExtensionDtype
1558 # | str | dtype[Any] | type[str] | type[complex] | type[bool]
1559 # | type[object]]", right operand type: "Literal[True]")
1560 elif self.dtype is True: # type: ignore[comparison-overlap]
1561 pass
1562 elif not _should_convert_dates(
1563 convert_dates, self.keep_default_dates, name
1564 ):
1565 # convert_dates takes precedence over columns listed in dtypes
1566 dtype = (
1567 self.dtype.get(name) if isinstance(self.dtype, dict) else self.dtype
1568 )
1569 if dtype is not None:
1570 try:
1571 return data.astype(dtype), True
1572 except (TypeError, ValueError):
1573 return data, False
1575 if convert_dates:
1576 new_data = self._try_convert_to_date(data)
1577 if new_data is not data:
1578 return new_data, True
1580 converted = False
1581 if self.dtype_backend is not lib.no_default and not is_axis:
1582 # Fall through for conversion later on
1583 return data, True
1584 elif is_string_dtype(data.dtype):
1585 # try float
1586 try:
1587 data = data.astype("float64")
1588 converted = True
1589 except (TypeError, ValueError):
1590 pass
1592 if data.dtype.kind == "f" and data.dtype != "float64":
1593 # coerce floats to 64
1594 try:
1595 data = data.astype("float64")
1596 converted = True
1597 except (TypeError, ValueError):
1598 pass
1600 # don't coerce 0-len data
1601 if len(data) and data.dtype in ("float", "object"):
1602 # coerce ints if we can
1603 try:
1604 new_data = org_data.astype("int64")
1605 if (new_data == data).all():
1606 data = new_data
1607 converted = True
1608 except (TypeError, ValueError, OverflowError):
1609 pass
1611 if data.dtype == "int" and data.dtype != "int64":
1612 # coerce ints to 64
1613 try:
1614 data = data.astype("int64")
1615 converted = True
1616 except (TypeError, ValueError):
1617 pass
1619 # if we have an index, we want to preserve dtypes
1620 if name == "index" and len(data):
1621 if self.orient == "split":
1622 return data, False
1624 return data, converted
1626 @final
1627 def _try_convert_to_date(self, data: Series) -> Series:
1628 """
1629 Try to parse an ndarray like into a date column.
1631 Try to coerce object in epoch/iso formats and integer/float in epoch
1632 formats.
1633 """
1634 # no conversion on empty
1635 if not len(data):
1636 return data
1638 new_data = data
1640 if new_data.dtype == "object" or new_data.dtype == "string": # noqa: PLR1714
1641 try:
1642 new_data = data.astype("int64")
1643 except OverflowError:
1644 return data
1645 except (TypeError, ValueError):
1646 pass
1648 # ignore numbers that are out of range
1649 if issubclass(new_data.dtype.type, np.number):
1650 in_range = (
1651 isna(new_data._values)
1652 | (new_data > self.min_stamp)
1653 | (new_data._values == iNaT)
1654 )
1655 if not in_range.all():
1656 return data
1658 if new_data.dtype == "string":
1659 for format in (None, "ISO8601", "mixed"):
1660 converted = None
1661 with warnings.catch_warnings(record=True) as record:
1662 warnings.simplefilter("always")
1663 try:
1664 converted = to_datetime(new_data, errors="raise", format=format)
1665 except Exception:
1666 pass
1667 if converted is None:
1668 continue
1669 if format != "ISO8601" and _is_default_filled(converted):
1670 # only a strict ISO8601 parse proves a year-1 result came
1671 # from the strings themselves rather than from dateutil's
1672 # defaults, so keep looking
1673 continue
1674 _reemit_parse_warnings(record, ignore_user_warnings=True)
1675 return converted
1676 else:
1677 # numeric or mixed objects
1678 date_units = (self.date_unit,) if self.date_unit else self._STAMP_UNITS
1679 converted = None
1680 in_ns_bounds = False
1681 for date_unit in date_units:
1682 with warnings.catch_warnings(record=True) as record:
1683 warnings.simplefilter("always")
1684 try:
1685 # In case of multiple possible units, infer the likely unit
1686 # based on the first unit for which the parsed dates fit
1687 # within the nanoseconds bounds
1688 # -> do as_unit cast to ensure OutOfBounds error
1689 converted = to_datetime(
1690 new_data, errors="raise", unit=date_unit
1691 )
1692 _ = converted.dt.as_unit("ns")
1693 in_ns_bounds = True
1694 except (
1695 OutOfBoundsDatetime,
1696 ValueError,
1697 OverflowError,
1698 TypeError,
1699 ):
1700 pass
1701 if converted is not None and in_ns_bounds:
1702 _reemit_parse_warnings(record)
1703 data = converted
1704 break
1705 else:
1706 if converted is not None and (
1707 not _is_default_filled(converted)
1708 or _parses_strict_iso8601(new_data)
1709 ):
1710 # all units failed to cast to ns (eg with mixed string / int)
1711 # but to_datetime still returned a result -> use this this
1712 # result (with the last unit) and re-emit any warning
1713 _reemit_parse_warnings(record)
1714 data = converted
1715 return data
1718def _is_default_filled(converted: Series) -> bool:
1719 """
1720 Whether a speculative string date parse only succeeded by filling in defaults.
1722 GH#63473: ``to_datetime`` falls back to ``dateutil`` for strings it cannot
1723 match against a format, and ``dateutil`` takes the year absent from the
1724 string from ``datetime(1, 1, 1)``. Strings like ``"Jan"`` or ``"1st"`` parse
1725 that way, so accepting the result would silently turn ordinary labels into
1726 year 1 timestamps. Strings carrying only a time of day are filled from
1727 *today* instead and so are not caught here.
1729 A year-1 result is genuine when the strings parse under a strict ISO8601
1730 format, which rejects ``"Jan"`` outright; callers check that separately,
1731 either by having attempted ``format="ISO8601"`` themselves or via
1732 ``_parses_strict_iso8601``.
1733 """
1734 if converted.dtype.kind != "M":
1735 # defensive; the parses attempted above all give datetime64
1736 return False
1737 return bool((converted.dt.year == 1).any())
1740def _parses_strict_iso8601(data: Series) -> bool:
1741 """
1742 Whether every entry of ``data`` parses under a strict ISO8601 format.
1744 Used to tell a genuine out-of-nanosecond-bounds date apart from a
1745 default-filled parse on the object-dtype path, which -- unlike the
1746 string-dtype path -- does not attempt ``format="ISO8601"`` itself.
1747 """
1748 with warnings.catch_warnings():
1749 # a speculative probe whose result is discarded; anything it emits
1750 # would be about a parse the caller never adopts
1751 warnings.simplefilter("ignore")
1752 try:
1753 to_datetime(data, errors="raise", format="ISO8601")
1754 except Exception:
1755 return False
1756 return True
1759def _reemit_parse_warnings(
1760 record: list[warnings.WarningMessage],
1761 *,
1762 ignore_user_warnings: bool = False,
1763) -> None:
1764 """
1765 Re-emit the warnings captured from a date parse ``_try_convert_to_date`` may
1766 discard.
1768 GH#50907: it parses the same values several ways and keeps at most one
1769 result. The quarterly-string deprecation asks the user to change a call, so
1770 it may only be emitted for the parse whose result they actually receive.
1771 Unrelated warnings keep the visibility they had before.
1772 """
1773 for warning in record:
1774 if ignore_user_warnings and issubclass(warning.category, UserWarning):
1775 # "Could not infer format", incorrectly raised for non-date strings
1776 continue
1777 warnings.warn(
1778 warning.message,
1779 warning.category,
1780 stacklevel=find_stack_level(),
1781 )
1784class SeriesParser(Parser):
1785 _default_orient = "index"
1786 _split_keys = ("name", "index", "data")
1788 def _parse(self) -> Series:
1789 data = ujson_loads(self.json, precise_float=self.precise_float)
1791 if self.orient == "split":
1792 decoded = {str(k): v for k, v in data.items()}
1793 self.check_keys_split(decoded)
1794 return Series(**decoded)
1795 else:
1796 return Series(data)
1798 def _try_convert_types(self, obj: Series) -> Series:
1799 obj, _ = self._try_convert_data("data", obj, convert_dates=self.convert_dates)
1800 return obj
1803class FrameParser(Parser):
1804 _default_orient = "columns"
1805 _split_keys = ("columns", "index", "data")
1807 def _parse(self) -> DataFrame:
1808 json = self.json
1809 orient = self.orient
1811 if orient == "split":
1812 decoded = {
1813 str(k): v
1814 for k, v in ujson_loads(json, precise_float=self.precise_float).items()
1815 }
1816 self.check_keys_split(decoded)
1817 orig_names = [
1818 (tuple(col) if isinstance(col, list) else col)
1819 for col in decoded["columns"]
1820 ]
1821 decoded["columns"] = dedup_names(
1822 orig_names,
1823 is_potential_multi_index(orig_names, None),
1824 )
1825 return DataFrame(dtype=None, **decoded)
1826 elif orient == "index":
1827 return DataFrame.from_dict(
1828 ujson_loads(json, precise_float=self.precise_float),
1829 dtype=None,
1830 orient="index",
1831 )
1832 elif orient == "table":
1833 return parse_table_schema(json, precise_float=self.precise_float)
1834 else:
1835 # includes orient == "columns"
1836 return DataFrame(
1837 ujson_loads(json, precise_float=self.precise_float), dtype=None
1838 )
1840 def _try_convert_types(self, obj: DataFrame) -> DataFrame:
1841 arrays = []
1842 for col_label, series in obj.items():
1843 result, _ = self._try_convert_data(
1844 col_label,
1845 series,
1846 convert_dates=_should_convert_dates(
1847 self.convert_dates,
1848 keep_default_dates=self.keep_default_dates,
1849 col=col_label,
1850 ),
1851 )
1852 arrays.append(result.array)
1853 return DataFrame._from_arrays(
1854 arrays, obj.columns, obj.index, verify_integrity=False
1855 )
1858def _should_convert_dates(
1859 convert_dates: bool | list[str],
1860 keep_default_dates: bool,
1861 col: Hashable,
1862) -> bool:
1863 """
1864 Return bool whether a DataFrame column should be cast to datetime.
1865 """
1866 if convert_dates is False:
1867 # convert_dates=True means follow keep_default_dates
1868 return False
1869 elif not isinstance(convert_dates, bool) and col in set(convert_dates):
1870 return True
1871 elif not keep_default_dates:
1872 return False
1873 elif not isinstance(col, str):
1874 return False
1875 col_lower = col.lower()
1876 if (
1877 col_lower.endswith(("_at", "_time"))
1878 or col_lower in {"modified", "date", "datetime"}
1879 or col_lower.startswith("timestamp")
1880 ):
1881 return True
1882 return False