Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/pytables.py: 17%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2High level interface to PyTables for reading and writing pandas data structures
3to disk
4"""
6from __future__ import annotations
8from contextlib import suppress
9import copy
10from datetime import (
11 date,
12 tzinfo,
13)
14import itertools
15import os
16import re
17from textwrap import dedent
18from typing import (
19 TYPE_CHECKING,
20 Any,
21 Final,
22 Literal,
23 Self,
24 TypeAlias,
25 cast,
26 overload,
27)
28import warnings
30import numpy as np
32from pandas._config import (
33 using_string_dtype,
34)
35import pandas._config.config as cf
36from pandas._config.config import _global_config as config
38from pandas._libs import (
39 lib,
40 writers as libwriters,
41)
42from pandas._libs.lib import is_string_array
43from pandas._libs.tslibs import timezones
44from pandas.compat import (
45 HAS_PYARROW,
46 PY312,
47)
48from pandas.compat._optional import import_optional_dependency
49from pandas.compat.pickle_compat import patch_pickle
50from pandas.errors import (
51 AttributeConflictWarning,
52 ClosedFileError,
53 Pandas4Warning,
54 PerformanceWarning,
55 PossibleDataLossError,
56)
57from pandas.util._decorators import (
58 cache_readonly,
59 set_module,
60)
61from pandas.util._exceptions import find_stack_level
63from pandas.core.dtypes.common import (
64 ensure_object,
65 is_bool_dtype,
66 is_complex_dtype,
67 is_list_like,
68 is_string_dtype,
69 needs_i8_conversion,
70)
71from pandas.core.dtypes.dtypes import (
72 CategoricalDtype,
73 DatetimeTZDtype,
74 ExtensionDtype,
75 PeriodDtype,
76)
77from pandas.core.dtypes.missing import array_equivalent
79from pandas import (
80 CategoricalIndex,
81 DataFrame,
82 DatetimeIndex,
83 Index,
84 MultiIndex,
85 PeriodIndex,
86 RangeIndex,
87 Series,
88 StringDtype,
89 TimedeltaIndex,
90 concat,
91 isna,
92)
93from pandas.core.arrays import (
94 Categorical,
95 DatetimeArray,
96 PeriodArray,
97)
98from pandas.core.arrays.datetimes import tz_to_dtype
99from pandas.core.arrays.string_ import BaseStringArray
100import pandas.core.common as com
101from pandas.core.computation.pytables import (
102 JointConditionBinOp,
103 PyTablesExpr,
104 maybe_expression,
105)
106from pandas.core.construction import (
107 array as pd_array,
108 extract_array,
109)
110from pandas.core.indexes.api import ensure_index
112from pandas.io.common import stringify_path
113from pandas.io.formats.printing import (
114 adjoin,
115 pprint_thing,
116)
118if TYPE_CHECKING:
119 from collections.abc import (
120 Callable,
121 Hashable,
122 Iterator,
123 )
124 from types import (
125 ModuleType,
126 TracebackType,
127 )
129 from tables import (
130 Col,
131 File,
132 Node,
133 )
135 from pandas._typing import (
136 AnyArrayLike,
137 ArrayLike,
138 AxisInt,
139 DtypeArg,
140 FilePath,
141 TimeUnit,
142 npt,
143 )
145 from pandas.core.internals import Block
148# encoding
149_default_encoding = "UTF-8"
152def _ensure_encoding(encoding: str | None) -> str:
153 # set the encoding if we need
154 if encoding is None:
155 encoding = _default_encoding
157 return encoding
160def _ensure_str(name):
161 """
162 Ensure that an index / column name is a str (python 3); otherwise they
163 may be np.string dtype. Non-string dtypes are passed through unchanged.
165 https://github.com/pandas-dev/pandas/issues/13492
166 """
167 if isinstance(name, str):
168 name = str(name)
169 return name
172Term: TypeAlias = PyTablesExpr
175def _ensure_term(where, scope_level: int):
176 """
177 Ensure that the where is a Term or a list of Term.
179 This makes sure that we are capturing the scope of variables that are
180 passed create the terms here with a frame_level=2 (we are 2 levels down)
181 """
182 # only consider list/tuple here as an ndarray is automatically a coordinate
183 # list
184 level = scope_level + 1
185 if isinstance(where, (list, tuple)):
186 # Pre-3.12 the list comprehension created its own stack frame,
187 # requiring an extra +1 to reach the caller's scope. PEP 709
188 # inlined comprehensions in 3.12, removing that frame.
189 comp_adjustment = 0 if PY312 else 1
190 where = [
191 Term(term, scope_level=level + comp_adjustment)
192 if maybe_expression(term)
193 else term
194 for term in where
195 if term is not None
196 ]
197 elif maybe_expression(where):
198 where = Term(where, scope_level=level)
199 return where if where is None or len(where) else None
202attribute_conflict_doc: Final = """
203the [%s] attribute of the existing index is [%s] which conflicts with the new
204[%s], resetting the attribute to None
205"""
207performance_doc: Final = """
208your performance may suffer as PyTables will pickle object types that it cannot
209map directly to c-types [inferred_type->%s,key->%s] [items->%s]
210"""
212# formats
213_FORMAT_MAP = {"f": "fixed", "fixed": "fixed", "t": "table", "table": "table"}
215# axes map
216_AXES_MAP = {DataFrame: [0]}
218# register our configuration options
219dropna_doc: Final = """
220: boolean
221 drop ALL nan rows when appending to a table
222"""
223format_doc: Final = """
224: format
225 default format writing format, if None, then
226 put will default to 'fixed' and append will default to 'table'
227"""
229with cf.config_prefix("io.hdf"):
230 cf.register_option("dropna_table", False, dropna_doc, validator=cf.is_bool)
231 cf.register_option(
232 "default_format",
233 None,
234 format_doc,
235 validator=cf.is_one_of_factory(["fixed", "table", None]),
236 )
238cf.deprecate_option(
239 "io.hdf.dropna_table",
240 Pandas4Warning,
241 msg="io.hdf.dropna_table option is deprecated. Use DataFrame.dropna "
242 "before writing instead.",
243)
245# oh the troubles to reduce import time
246_table_mod: ModuleType | None = None
247_table_file_open_policy_is_strict = False
250_MISSING = object()
253def _set_attr_if_changed(attrs, name: str, value) -> None:
254 """
255 setattr on a PyTables AttributeSet only if the on-disk value differs.
257 Re-writing an HDF5 attribute to the same value is expensive — pytables
258 deletes and re-creates it, hitting the disk per attribute. On wide-table
259 appends this dominates runtime (GH#25839).
260 """
261 current = getattr(attrs, name, _MISSING)
262 if current is _MISSING:
263 setattr(attrs, name, value)
264 return
265 try:
266 equal = bool(current == value)
267 except (ValueError, TypeError):
268 equal = False
269 if not equal:
270 setattr(attrs, name, value)
273def _resolve_row_window(
274 start: int | None, stop: int | None, nrows: int
275) -> tuple[int, int]:
276 """
277 Resolve ``start``/``stop`` row bounds against a table holding ``nrows`` rows.
279 Slice semantics count a negative bound back from the end of the table *and*
280 clamp both bounds into ``[0, nrows]``, which is what a plain read does; the
281 row-coordinate paths have to agree with it.
282 """
283 start, stop, _ = slice(start, stop).indices(nrows)
284 return start, stop
287def _tables():
288 global _table_mod
289 global _table_file_open_policy_is_strict
290 if _table_mod is None:
291 import tables
293 _table_mod = tables
295 # set the file open policy
296 # return the file open policy; this changes as of pytables 3.1
297 # depending on the HDF5 version
298 with suppress(AttributeError):
299 _table_file_open_policy_is_strict = (
300 tables.file._FILE_OPEN_POLICY == "strict"
301 )
303 return _table_mod
306# interface to/from ###
309def to_hdf(
310 path_or_buf: FilePath | HDFStore,
311 key: str,
312 value: DataFrame | Series,
313 mode: str = "a",
314 complevel: int | None = None,
315 complib: str | None = None,
316 append: bool = False,
317 format: str | None = None,
318 index: bool = True,
319 min_itemsize: int | dict[str, int] | None = None,
320 nan_rep=None,
321 dropna: bool | lib.NoDefault | None = lib.no_default,
322 data_columns: Literal[True] | list[str] | None = None,
323 errors: str = "strict",
324 encoding: str = "UTF-8",
325) -> None:
326 """store this object, close it if we opened it"""
327 if append:
328 f = lambda store: store.append(
329 key,
330 value,
331 format=format,
332 index=index,
333 min_itemsize=min_itemsize,
334 nan_rep=nan_rep,
335 dropna=dropna,
336 data_columns=data_columns,
337 errors=errors,
338 encoding=encoding,
339 )
340 else:
341 # NB: dropna is not passed to `put`
342 f = lambda store: store.put(
343 key,
344 value,
345 format=format,
346 index=index,
347 min_itemsize=min_itemsize,
348 nan_rep=nan_rep,
349 data_columns=data_columns,
350 errors=errors,
351 encoding=encoding,
352 dropna=dropna,
353 track_times=False,
354 )
356 if isinstance(path_or_buf, HDFStore):
357 f(path_or_buf)
358 else:
359 path_or_buf = stringify_path(path_or_buf)
360 with HDFStore(
361 path_or_buf, mode=mode, complevel=complevel, complib=complib
362 ) as store:
363 f(store)
366@set_module("pandas")
367def read_hdf(
368 path_or_buf: FilePath | HDFStore,
369 key=None,
370 mode: str = "r",
371 errors: str = "strict",
372 where: str | list | None = None,
373 start: int | None = None,
374 stop: int | None = None,
375 columns: list[str] | None = None,
376 iterator: bool = False,
377 chunksize: int | None = None,
378 **kwargs,
379):
380 """
381 Read from the store, close it if we opened it.
383 Retrieve pandas object stored in file, optionally based on where
384 criteria. This function requires the
385 `PyTables <https://www.pytables.org/>`_ library.
387 .. note::
389 This function only reads HDF5 files written by pandas (via
390 :meth:`DataFrame.to_hdf`, :meth:`Series.to_hdf`, or :class:`HDFStore`),
391 which use a pandas-specific layout built on PyTables. Arbitrary HDF5
392 files produced by other tools such as ``h5py`` or plain PyTables are
393 not supported; use those libraries directly to read such files.
395 .. warning::
397 Pandas uses PyTables for reading and writing HDF5 files, which allows
398 serializing object-dtype data with pickle when using the "fixed" format.
399 Loading pickled data received from untrusted sources can be unsafe.
401 See: https://docs.python.org/3/library/pickle.html for more.
403 Parameters
404 ----------
405 path_or_buf : str, path object, pandas.HDFStore
406 Any valid string path is acceptable. Only supports the local file system,
407 remote URLs and file-like objects are not supported.
409 If you want to pass in a path object, pandas accepts any
410 ``os.PathLike``.
412 Alternatively, pandas accepts an open :class:`pandas.HDFStore` object.
414 key : object, optional
415 The group identifier in the store. Can be omitted if the HDF file
416 contains a single pandas object.
417 mode : {'r', 'r+', 'a'}, default 'r'
418 Mode to use when opening the file. Ignored if path_or_buf is a
419 :class:`pandas.HDFStore`. Default is 'r'.
420 errors : str, default 'strict'
421 Specifies how encoding and decoding errors are to be handled.
422 See the errors argument for :func:`open` for a full list
423 of options.
424 where : list, optional
425 A list of Term (or convertible) objects.
426 start : int, optional
427 Row number to start selection.
428 stop : int, optional
429 Row number to stop selection.
430 columns : list, optional
431 A list of columns names to return.
432 iterator : bool, optional
433 Return an iterator object.
434 chunksize : int, optional
435 Number of rows to include in an iteration when using an iterator.
436 **kwargs
437 Additional keyword arguments passed to HDFStore.
439 Returns
440 -------
441 object
442 The selected object. Return type depends on the object stored.
444 See Also
445 --------
446 DataFrame.to_hdf : Write an HDF file from a DataFrame.
447 HDFStore : Low-level access to HDF files.
449 Notes
450 -----
451 When ``errors="surrogatepass"``, ``pd.options.future.infer_string`` is true,
452 and PyArrow is installed, if a UTF-16 surrogate is encountered when decoding
453 to UTF-8, the resulting dtype will be
454 ``pd.StringDtype(storage="python", na_value=np.nan)``.
456 Examples
457 --------
458 >>> df = pd.DataFrame([[1, 1.0, "a"]], columns=["x", "y", "z"]) # doctest: +SKIP
459 >>> df.to_hdf("./store.h5", key="data") # doctest: +SKIP
460 >>> reread = pd.read_hdf("./store.h5") # doctest: +SKIP
461 """
462 if mode not in ["r", "r+", "a"]:
463 raise ValueError(
464 f"mode {mode} is not allowed while performing a read. "
465 f"Allowed modes are r, r+ and a."
466 )
467 # grab the scope
468 if where is not None:
469 where = _ensure_term(where, scope_level=1)
471 if isinstance(path_or_buf, HDFStore):
472 if not path_or_buf.is_open:
473 raise OSError("The HDFStore must be open for reading.")
475 store = path_or_buf
476 auto_close = False
477 else:
478 path_or_buf = stringify_path(path_or_buf)
479 if not isinstance(path_or_buf, str):
480 raise NotImplementedError(
481 "Support for generic buffers has not been implemented."
482 )
483 try:
484 exists = os.path.exists(path_or_buf)
486 # if filepath is too long
487 except (TypeError, ValueError):
488 exists = False
490 if not exists:
491 raise FileNotFoundError(f"File {path_or_buf} does not exist")
493 store = HDFStore(path_or_buf, mode=mode, errors=errors, **kwargs)
494 # can't auto open/close if we are using an iterator
495 # so delegate to the iterator
496 auto_close = True
498 read_succeeded = False
499 try:
500 if key is None:
501 groups = store.groups()
502 if len(groups) == 0:
503 raise ValueError(
504 "Dataset(s) incompatible with Pandas data types, "
505 "not table, or no datasets found in HDF5 file."
506 )
507 candidate_only_group = groups[0]
509 # For the HDF file to have only one dataset, all other groups
510 # should then be metadata groups for that candidate group. (This
511 # assumes that the groups() method enumerates parent groups
512 # before their children.)
513 for group_to_check in groups[1:]:
514 if not _is_metadata_of(group_to_check, candidate_only_group):
515 raise ValueError(
516 "key must be provided when HDF5 "
517 "file contains multiple datasets."
518 )
519 key = candidate_only_group._v_pathname
520 result = store.select(
521 key,
522 where=where,
523 start=start,
524 stop=stop,
525 columns=columns,
526 iterator=iterator,
527 chunksize=chunksize,
528 auto_close=auto_close,
529 )
530 read_succeeded = True
531 return result
532 finally:
533 # If the read failed for any reason, close the store when we were the
534 # ones who opened it, so the caller can reopen the file (GH#28430). On
535 # success the store is left as-is: an iterator result needs it open.
536 if not read_succeeded and not isinstance(path_or_buf, HDFStore):
537 with suppress(AttributeError):
538 store.close()
541def _is_metadata_of(group: Node, parent_group: Node) -> bool:
542 """Check if a given group is a metadata group for a given parent_group."""
543 if group._v_depth <= parent_group._v_depth:
544 return False
546 current = group
547 while current._v_depth > 1:
548 parent = current._v_parent
549 if parent == parent_group and current._v_name == "meta":
550 return True
551 current = current._v_parent
552 return False
555@set_module("pandas")
556class HDFStore:
557 """
558 Dict-like IO interface for storing pandas objects in PyTables.
560 Either Fixed or Table format.
562 .. note::
564 ``HDFStore`` uses a pandas-specific layout on top of PyTables and is
565 intended for round-tripping pandas objects. It cannot read arbitrary
566 HDF5 files produced by other tools such as ``h5py`` or plain PyTables;
567 use those libraries directly for general HDF5 interoperability.
569 .. warning::
571 Pandas uses PyTables for reading and writing HDF5 files, which allows
572 serializing object-dtype data with pickle when using the "fixed" format.
573 Loading pickled data received from untrusted sources can be unsafe.
575 See: https://docs.python.org/3/library/pickle.html for more.
577 Parameters
578 ----------
579 path : str
580 File path to HDF5 file.
581 mode : {'a', 'w', 'r', 'r+'}, default 'a'
583 ``'r'``
584 Read-only; no data can be modified.
585 ``'w'``
586 Write; a new file is created (an existing file with the same
587 name would be deleted).
588 ``'a'``
589 Append; an existing file is opened for reading and writing,
590 and if the file does not exist it is created.
591 ``'r+'``
592 It is similar to ``'a'``, but the file must already exist.
593 complevel : int, 0-9, default None
594 Specifies a compression level for data.
595 A value of 0 or None disables compression.
596 complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
597 Specifies the compression library to be used. This has no effect
598 unless ``complevel`` is set to a value greater than 0; passing
599 ``complib`` alone emits a ``UserWarning`` and produces an
600 uncompressed store.
601 These additional compressors for Blosc are supported
602 (default if no compressor specified: 'blosc:blosclz'):
603 {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
604 'blosc:zlib', 'blosc:zstd'}.
605 Specifying a compression library which is not available issues
606 a ValueError.
607 fletcher32 : bool, default False
608 If applying compression use the fletcher32 checksum.
609 **kwargs
610 These parameters will be passed to the PyTables open_file method.
612 Examples
613 --------
614 >>> bar = pd.DataFrame(np.random.randn(10, 4))
615 >>> store = pd.HDFStore("test.h5")
616 >>> store["foo"] = bar # write to HDF5
617 >>> bar = store["foo"] # retrieve
618 >>> store.close()
620 **Create or load HDF5 file in-memory**
622 When passing the `driver` option to the PyTables open_file method through
623 **kwargs, the HDF5 file is loaded or created in-memory and will only be
624 written when closed:
626 >>> bar = pd.DataFrame(np.random.randn(10, 4))
627 >>> store = pd.HDFStore("test.h5", driver="H5FD_CORE")
628 >>> store["foo"] = bar
629 >>> store.close() # only now, data is written to disk
630 """
632 _handle: File | None
633 _mode: str
635 def __init__(
636 self,
637 path,
638 mode: str = "a",
639 complevel: int | None = None,
640 complib=None,
641 fletcher32: bool = False,
642 **kwargs,
643 ) -> None:
644 if "format" in kwargs:
645 raise ValueError("format is not a defined argument for HDFStore")
647 tables = import_optional_dependency("tables")
649 if complib is not None and complib not in tables.filters.all_complibs:
650 raise ValueError(
651 f"complib only supports {tables.filters.all_complibs} compression."
652 )
654 if complib is None and complevel is not None:
655 complib = tables.filters.default_complib
657 if complib is not None and complevel is None:
658 # GH#29310 complib without complevel does not compress, because
659 # complevel defaults to 0. Warn rather than silently ignoring the
660 # requested library.
661 warnings.warn(
662 f"complib={complib!r} was specified without complevel; no "
663 "compression will be applied. Pass complevel (an int in 1-9) "
664 "to enable compression.",
665 UserWarning,
666 stacklevel=find_stack_level(),
667 )
669 self._path = stringify_path(path)
670 if mode is None:
671 mode = "a"
672 self._mode = mode
673 self._handle = None
674 self._complevel = complevel if complevel else 0
675 self._complib = complib
676 self._fletcher32 = fletcher32
677 self._filters = None
678 self.open(mode=mode, **kwargs)
680 def __fspath__(self) -> str:
681 return self._path
683 @property
684 def root(self):
685 """return the root node"""
686 self._check_if_open()
687 assert self._handle is not None # for mypy
688 return self._handle.root
690 @property
691 def filename(self) -> str:
692 return self._path
694 def __getitem__(self, key: str):
695 return self.get(key)
697 def __setitem__(self, key: str, value) -> None:
698 self.put(key, value, track_times=False)
700 def __delitem__(self, key: str) -> int | None:
701 return self.remove(key)
703 def __getattr__(self, name: str):
704 """allow attribute access to get stores"""
705 try:
706 return self.get(name)
707 except (KeyError, ClosedFileError):
708 pass
709 raise AttributeError(
710 f"'{type(self).__name__}' object has no attribute '{name}'"
711 )
713 def __contains__(self, key: str) -> bool:
714 """
715 check for existence of this key
716 can match the exact pathname or the pathnm w/o the leading '/'
717 """
718 node = self.get_node(key)
719 if node is not None:
720 name = node._v_pathname
721 if key in (name, name[1:]):
722 return True
723 return False
725 def __len__(self) -> int:
726 return len(self.groups())
728 def __repr__(self) -> str:
729 pstr = pprint_thing(self._path)
730 return f"{type(self)}\nFile path: {pstr}\n"
732 def __enter__(self) -> Self:
733 return self
735 def __exit__(
736 self,
737 exc_type: type[BaseException] | None,
738 exc_value: BaseException | None,
739 traceback: TracebackType | None,
740 ) -> None:
741 self.close()
743 def keys(self, include: str = "pandas") -> list[str]:
744 """
745 Return a list of keys corresponding to objects stored in HDFStore.
747 The keys are absolute path-names within the HDF5 file hierarchy.
748 By default only pandas objects are returned, but native HDF5 table
749 objects can be included as well.
751 Parameters
752 ----------
754 include : str, default 'pandas'
755 When kind equals 'pandas' return pandas objects.
756 When kind equals 'native' return native HDF5 Table objects.
758 Returns
759 -------
760 list
761 List of ABSOLUTE path-names (e.g. have the leading '/').
763 Raises
764 ------
765 raises ValueError if kind has an illegal value
767 See Also
768 --------
769 HDFStore.info : Prints detailed information on the store.
770 HDFStore.get_node : Returns the node with the key.
771 HDFStore.get_storer : Returns the storer object for a key.
773 Examples
774 --------
775 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
776 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
777 >>> store.put("data", df) # doctest: +SKIP
778 >>> store.get("data") # doctest: +SKIP
779 >>> print(store.keys()) # doctest: +SKIP
780 ['/data1', '/data2']
781 >>> store.close() # doctest: +SKIP
782 """
783 if include == "pandas":
784 return [n._v_pathname for n in self.groups()]
786 elif include == "native":
787 assert self._handle is not None # mypy
788 return [
789 n._v_pathname for n in self._handle.walk_nodes("/", classname="Table")
790 ]
791 raise ValueError(
792 f"`include` should be either 'pandas' or 'native' but is '{include}'"
793 )
795 def __iter__(self) -> Iterator[str]:
796 return iter(self.keys())
798 def items(self) -> Iterator[tuple[str, list]]:
799 """
800 iterate on key->group
801 """
802 for g in self.groups():
803 yield g._v_pathname, g
805 def open(self, mode: str = "a", **kwargs) -> None:
806 """
807 Open the file in the specified mode
809 Parameters
810 ----------
811 mode : {'a', 'w', 'r', 'r+'}, default 'a'
812 See HDFStore docstring or tables.open_file for info about modes
813 **kwargs
814 These parameters will be passed to the PyTables open_file method.
815 """
816 tables = _tables()
818 if self._mode != mode:
819 # if we are changing a write mode to read, ok
820 if self._mode in ["a", "w"] and mode in ["r", "r+"]:
821 pass
822 elif mode in ["w"]:
823 # this would truncate, raise here
824 if self.is_open:
825 raise PossibleDataLossError(
826 f"Re-opening the file [{self._path}] with mode [{self._mode}] "
827 "will delete the current file!"
828 )
830 self._mode = mode
832 # close and reopen the handle
833 if self.is_open:
834 self.close()
836 if self._complevel and self._complevel > 0:
837 self._filters = _tables().Filters(
838 self._complevel, self._complib, fletcher32=self._fletcher32
839 )
841 if _table_file_open_policy_is_strict and self.is_open:
842 msg = (
843 "Cannot open HDF5 file, which is already opened, "
844 "even in read-only mode."
845 )
846 raise ValueError(msg)
848 self._handle = tables.open_file(self._path, self._mode, **kwargs)
850 def close(self) -> None:
851 """
852 Close the PyTables file handle
853 """
854 if self._handle is not None:
855 self._handle.close()
856 self._handle = None
858 @property
859 def is_open(self) -> bool:
860 """
861 Return a boolean indicating whether the file is open.
863 ``HDFStore`` instances open the underlying PyTables file in their
864 constructor, but the file can be closed and reopened on the same
865 instance via :meth:`close` and :meth:`open`.
867 See Also
868 --------
869 HDFStore.open : Open the underlying file in the specified mode.
870 HDFStore.close : Close the underlying PyTables file handle.
872 Examples
873 --------
874 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
875 >>> store.is_open # doctest: +SKIP
876 True
877 >>> store.close() # doctest: +SKIP
878 >>> store.is_open # doctest: +SKIP
879 False
880 """
881 if self._handle is None:
882 return False
883 return bool(self._handle.isopen)
885 def flush(self, fsync: bool = False) -> None:
886 """
887 Force all buffered modifications to be written to disk.
889 Useful when sharing access between processes -- call ``flush`` (with
890 ``fsync=True`` if needed) before releasing a write lock so that
891 readers see the latest data.
893 Parameters
894 ----------
895 fsync : bool, default False
896 Call ``os.fsync()`` on the file handle to force writing to disk.
898 See Also
899 --------
900 HDFStore.close : Close the underlying PyTables file handle.
902 Notes
903 -----
904 Without ``fsync=True``, flushing may not guarantee that the OS writes
905 to disk. With fsync, the operation will block until the OS claims the
906 file has been written; however, other caching layers may still
907 interfere.
909 Examples
910 --------
911 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
912 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
913 >>> store.put("data", df) # doctest: +SKIP
914 >>> store.flush(fsync=True) # doctest: +SKIP
915 >>> store.close() # doctest: +SKIP
916 """
917 if self._handle is not None:
918 self._handle.flush()
919 if fsync:
920 with suppress(OSError):
921 os.fsync(self._handle.fileno())
923 def get(self, key: str):
924 """
925 Retrieve pandas object stored in file.
927 The object is read from the HDF5 file and returned as the
928 same type that was stored (e.g., DataFrame, Series).
930 Parameters
931 ----------
932 key : str
933 Object to retrieve from file. Raises KeyError if not found.
935 Returns
936 -------
937 object
938 Same type as object stored in file.
940 See Also
941 --------
942 HDFStore.get_node : Returns the node with the key.
943 HDFStore.get_storer : Returns the storer object for a key.
945 Examples
946 --------
947 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
948 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
949 >>> store.put("data", df) # doctest: +SKIP
950 >>> store.get("data") # doctest: +SKIP
951 >>> store.close() # doctest: +SKIP
952 """
953 with patch_pickle():
954 # GH#31167 Without this patch, pickle doesn't know how to unpickle
955 # old DateOffset objects now that they are cdef classes.
956 group = self.get_node(key)
957 if group is None:
958 raise KeyError(f"No object named {key} in the file")
959 return self._read_group(group)
961 def select(
962 self,
963 key: str,
964 where=None,
965 start=None,
966 stop=None,
967 columns=None,
968 iterator: bool = False,
969 chunksize: int | None = None,
970 auto_close: bool = False,
971 ):
972 """
973 Retrieve pandas object stored in file, optionally based on where criteria.
975 .. warning::
977 Pandas uses PyTables for reading and writing HDF5 files, which allows
978 serializing object-dtype data with pickle when using the "fixed" format.
979 Loading pickled data received from untrusted sources can be unsafe.
981 See: https://docs.python.org/3/library/pickle.html for more.
983 Parameters
984 ----------
985 key : str
986 Object being retrieved from file.
987 where : list or None
988 List of Term (or convertible) objects, optional.
989 start : int or None
990 Row number to start selection.
991 stop : int, default None
992 Row number to stop selection.
993 columns : list or None
994 A list of columns that if not None, will limit the return columns.
995 iterator : bool or False
996 Returns an iterator.
997 chunksize : int or None
998 Number or rows to include in iteration, return an iterator.
999 auto_close : bool or False
1000 Should automatically close the store when finished.
1002 Returns
1003 -------
1004 object
1005 Retrieved object from file.
1007 See Also
1008 --------
1009 HDFStore.select_as_coordinates : Returns the selection as an index.
1010 HDFStore.select_column : Returns a single column from the table.
1011 HDFStore.select_as_multiple : Retrieves pandas objects from multiple tables.
1013 Examples
1014 --------
1015 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1016 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1017 >>> store.put("data", df) # doctest: +SKIP
1018 >>> store.get("data") # doctest: +SKIP
1019 >>> print(store.keys()) # doctest: +SKIP
1020 ['/data1', '/data2']
1021 >>> store.select("/data1") # doctest: +SKIP
1022 A B
1023 0 1 2
1024 1 3 4
1025 >>> store.select("/data1", where="columns == A") # doctest: +SKIP
1026 A
1027 0 1
1028 1 3
1029 >>> store.close() # doctest: +SKIP
1030 """
1031 group = self.get_node(key)
1032 if group is None:
1033 raise KeyError(f"No object named {key} in the file")
1035 # create the storer and axes
1036 where = _ensure_term(where, scope_level=1)
1037 s = self._create_storer(group)
1038 s.infer_axes()
1040 # function to call on iteration
1041 def func(_start, _stop, _where):
1042 return s.read(start=_start, stop=_stop, where=_where, columns=columns)
1044 # create the iterator
1045 it = TableIterator(
1046 self,
1047 s,
1048 func,
1049 where=where,
1050 nrows=s.nrows,
1051 start=start,
1052 stop=stop,
1053 iterator=iterator,
1054 chunksize=chunksize,
1055 auto_close=auto_close,
1056 )
1058 return it.get_result()
1060 def select_as_coordinates(
1061 self,
1062 key: str,
1063 where=None,
1064 start: int | None = None,
1065 stop: int | None = None,
1066 ):
1067 """
1068 Return the selection as an Index.
1070 .. warning::
1072 Pandas uses PyTables for reading and writing HDF5 files, which allows
1073 serializing object-dtype data with pickle when using the "fixed" format.
1074 Loading pickled data received from untrusted sources can be unsafe.
1076 See: https://docs.python.org/3/library/pickle.html for more.
1078 Parameters
1079 ----------
1080 key : str
1081 Object being retrieved from file.
1082 where : list of Term (or convertible) objects, optional
1083 Conditions to apply to the selection. ``start`` and ``stop`` are
1084 applied to the table before ``where``.
1085 start : int, optional
1086 Row number to start selection.
1087 stop : int, optional
1088 Row number to stop selection.
1090 Returns
1091 -------
1092 Index
1093 Integer positions of the matching rows; can be passed as
1094 ``where`` to a subsequent ``select``.
1096 See Also
1097 --------
1098 HDFStore.select : Retrieve a stored object, optionally filtered by
1099 ``where``.
1100 HDFStore.select_as_multiple : Retrieve pandas objects from multiple
1101 tables.
1103 Examples
1104 --------
1105 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1106 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1107 >>> store.append(
1108 ... "data", df, format="table", data_columns=["A"]
1109 ... ) # doctest: +SKIP
1110 >>> store.select_as_coordinates("data", "A>1") # doctest: +SKIP
1111 >>> store.close() # doctest: +SKIP
1112 """
1113 where = _ensure_term(where, scope_level=1)
1114 tbl = self.get_storer(key)
1115 if not isinstance(tbl, Table):
1116 raise TypeError("can only read_coordinates with a table")
1117 return tbl.read_coordinates(where=where, start=start, stop=stop)
1119 def select_column(
1120 self,
1121 key: str,
1122 column: str,
1123 start: int | None = None,
1124 stop: int | None = None,
1125 ):
1126 """
1127 Return a single column from the table.
1129 This is generally only useful to select an indexable.
1131 .. warning::
1133 Pandas uses PyTables for reading and writing HDF5 files, which allows
1134 serializing object-dtype data with pickle when using the "fixed" format.
1135 Loading pickled data received from untrusted sources can be unsafe.
1137 See: https://docs.python.org/3/library/pickle.html for more.
1139 Parameters
1140 ----------
1141 key : str
1142 Object being retrieved from file.
1143 column : str
1144 The column of interest.
1145 start : int or None, default None
1146 Row number to start selection.
1147 stop : int or None, default None
1148 Row number to stop selection.
1150 Returns
1151 -------
1152 Series
1153 A ``Series`` of the column's values.
1155 Raises
1156 ------
1157 KeyError
1158 If the column is not found, or ``key`` is not a valid store.
1159 ValueError
1160 If the column cannot be extracted individually (not an
1161 indexable or a data column).
1163 See Also
1164 --------
1165 HDFStore.select : Retrieve a stored object, optionally filtered by
1166 ``where``.
1167 HDFStore.select_as_coordinates : Return the matching row coordinates
1168 as an Index.
1170 Examples
1171 --------
1172 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1173 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1174 >>> store.append(
1175 ... "data", df, format="table", data_columns=["A"]
1176 ... ) # doctest: +SKIP
1177 >>> store.select_column("data", "A") # doctest: +SKIP
1178 >>> store.close() # doctest: +SKIP
1179 """
1180 tbl = self.get_storer(key)
1181 if not isinstance(tbl, Table):
1182 raise TypeError("can only read_column with a table")
1183 return tbl.read_column(column=column, start=start, stop=stop)
1185 def select_as_multiple(
1186 self,
1187 keys,
1188 where=None,
1189 selector=None,
1190 columns=None,
1191 start=None,
1192 stop=None,
1193 iterator: bool = False,
1194 chunksize: int | None = None,
1195 auto_close: bool = False,
1196 ):
1197 """
1198 Retrieve pandas objects from multiple tables.
1200 .. warning::
1202 Pandas uses PyTables for reading and writing HDF5 files, which allows
1203 serializing object-dtype data with pickle when using the "fixed" format.
1204 Loading pickled data received from untrusted sources can be unsafe.
1206 See: https://docs.python.org/3/library/pickle.html for more.
1208 Parameters
1209 ----------
1210 keys : list of str
1211 Names of the tables to read.
1212 where : list, optional
1213 List of Term (or convertible) objects.
1214 selector : str, optional
1215 The table to apply the where criteria to. Defaults to ``keys[0]``.
1216 columns : list, optional
1217 Columns to return.
1218 start : int, optional
1219 Row number to start selection. Applied to each table before
1220 ``where`` is evaluated.
1221 stop : int, optional
1222 Row number to stop selection. Applied to each table before
1223 ``where`` is evaluated.
1224 iterator : bool, default False
1225 Return an iterator.
1226 chunksize : int, optional
1227 Number of rows to include in each iteration; implies
1228 ``iterator=True``.
1229 auto_close : bool, default False
1230 Should automatically close the store when finished.
1232 Returns
1233 -------
1234 DataFrame or TableIterator
1235 Concatenated result from the selected tables. A ``TableIterator``
1236 is returned instead when ``iterator=True`` or ``chunksize`` is
1237 given.
1239 Raises
1240 ------
1241 KeyError
1242 If ``keys`` or ``selector`` is not found, or ``keys`` is empty.
1243 TypeError
1244 If ``keys`` is not a list or tuple.
1245 ValueError
1246 If the tables do not all have the same number of rows.
1248 See Also
1249 --------
1250 HDFStore.append_to_multiple : Append to multiple tables, splitting a
1251 single object into a dict of column groups.
1252 HDFStore.select : Retrieve a single stored object.
1254 Examples
1255 --------
1256 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1257 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["C", "D"])
1258 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1259 >>> store.append("t1", df1, format="table") # doctest: +SKIP
1260 >>> store.append("t2", df2, format="table") # doctest: +SKIP
1261 >>> store.select_as_multiple(["t1", "t2"]) # doctest: +SKIP
1262 >>> store.close() # doctest: +SKIP
1263 """
1264 # default to single select
1265 where = _ensure_term(where, scope_level=1)
1266 if isinstance(keys, (list, tuple)) and len(keys) == 1:
1267 keys = keys[0]
1268 if isinstance(keys, str):
1269 return self.select(
1270 key=keys,
1271 where=where,
1272 columns=columns,
1273 start=start,
1274 stop=stop,
1275 iterator=iterator,
1276 chunksize=chunksize,
1277 auto_close=auto_close,
1278 )
1280 if not isinstance(keys, (list, tuple)):
1281 raise TypeError("keys must be a list/tuple")
1283 if not len(keys):
1284 raise ValueError("keys must have a non-zero length")
1286 if selector is None:
1287 selector = keys[0]
1289 # collect the tables
1290 tbls = [self.get_storer(k) for k in keys]
1291 s = self.get_storer(selector)
1293 # validate rows
1294 nrows = None
1295 for t, k in itertools.chain([(s, selector)], zip(tbls, keys, strict=True)):
1296 if t is None:
1297 raise KeyError(f"Invalid table [{k}]")
1298 if not t.is_table:
1299 raise TypeError(
1300 f"object [{t.pathname}] is not a table, and cannot be used in all "
1301 "select as multiple"
1302 )
1304 if nrows is None:
1305 nrows = t.nrows
1306 elif t.nrows != nrows:
1307 raise ValueError("all tables must have exactly the same nrows!")
1309 # The isinstance checks here are redundant with the check above,
1310 # but necessary for mypy; see GH#29757
1311 _tbls = [x for x in tbls if isinstance(x, Table)]
1313 # axis is the concentration axes
1314 axis = {t.non_index_axes[0][0] for t in _tbls}.pop()
1316 def func(_start, _stop, _where):
1317 # retrieve the objs, _where is always passed as a set of
1318 # coordinates here
1319 objs = [
1320 t.read(where=_where, columns=columns, start=_start, stop=_stop)
1321 for t in tbls
1322 ]
1324 # concat and return
1325 return concat(objs, axis=axis, verify_integrity=False)._consolidate()
1327 # create the iterator
1328 it = TableIterator(
1329 self,
1330 s,
1331 func,
1332 where=where,
1333 nrows=nrows,
1334 start=start,
1335 stop=stop,
1336 iterator=iterator,
1337 chunksize=chunksize,
1338 auto_close=auto_close,
1339 )
1341 return it.get_result(coordinates=True)
1343 def put(
1344 self,
1345 key: str,
1346 value: DataFrame | Series,
1347 format=None,
1348 index: bool = True,
1349 append: bool = False,
1350 complib=None,
1351 complevel: int | None = None,
1352 min_itemsize: int | dict[str, int] | None = None,
1353 nan_rep=None,
1354 data_columns: Literal[True] | list[str] | None = None,
1355 encoding=None,
1356 errors: str = "strict",
1357 track_times: bool | lib.NoDefault = lib.no_default,
1358 dropna: bool | lib.NoDefault = lib.no_default,
1359 ) -> None:
1360 """
1361 Store object in HDFStore.
1363 This method writes a pandas DataFrame or Series into an HDF5 file using
1364 either the fixed or table format. The `table` format allows additional
1365 operations like incremental appends and queries but may have performance
1366 trade-offs. The `fixed` format provides faster read/write operations but
1367 does not support appends or queries.
1369 Parameters
1370 ----------
1371 key : str
1372 Key of object to store in file.
1373 value : {Series, DataFrame}
1374 Value of object to store in file.
1375 format : 'fixed(f)|table(t)', default is 'fixed'
1376 Format to use when storing object in HDFStore. Value can be one of:
1378 ``'fixed'``
1379 Fixed format. Fast writing/reading. Not-appendable, nor searchable.
1380 ``'table'``
1381 Table format. Write as a PyTables Table structure which may perform
1382 worse but allow more flexible operations like searching / selecting
1383 subsets of the data.
1384 index : bool, default True
1385 Write DataFrame index as a column.
1386 append : bool, default False
1387 This will force Table format, append the input data to the existing.
1388 complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default None
1389 Compression library to use, only applied with ``format='table'``.
1390 None disables compression. See the ``complib`` parameter of
1391 :class:`HDFStore` for the full list of supported compressors.
1392 complevel : int, 0-9, default None
1393 Specifies a compression level for data.
1394 A value of 0 or None disables compression.
1395 min_itemsize : int, dict, or None
1396 Minimum number of bytes reserved for object columns.
1397 If int, all columns reserve 'min_itemsize' bytes per stored value.
1398 If dict, specific columns reserve 'min_itemsize' bytes per stored value.
1399 Strings are stored as encoded bytes. Since some characters require multiple
1400 bytes, required size may be larger than string length.
1401 nan_rep : str, optional
1402 String used on disk to represent missing values in string columns
1403 (``format="table"`` only).
1404 By default a sentinel that collides with no value in the column is
1405 used, so a literal ``"nan"`` round-trips unchanged; when this is
1406 passed, a value equal to it is read back as a missing value.
1407 data_columns : list of columns or True, default None
1408 List of columns to create as data columns, or True to use all columns.
1409 See `here
1410 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
1411 encoding : str, default None
1412 Provide an encoding for strings.
1413 errors : str, default 'strict'
1414 The error handling scheme to use for encoding errors.
1415 The default is 'strict' meaning that encoding errors raise a
1416 UnicodeEncodeError. Other possible values are 'ignore', 'replace' and
1417 'xmlcharrefreplace' as well as any other name registered with
1418 codecs.register_error that can handle UnicodeEncodeErrors.
1419 track_times : bool, default True
1420 Parameter is propagated to 'create_table' method of 'PyTables'.
1421 If set to False it enables to have the same h5 files (same hashes)
1422 independent on creation time.
1424 .. deprecated:: 3.1.0
1425 The default value of ``track_times`` will change from ``True``
1426 to ``False`` in a future version. Pass ``track_times=False``
1427 explicitly to silence this warning and get deterministic
1428 HDF5 files.
1429 dropna : bool, default False, optional
1430 Remove missing values.
1432 .. deprecated:: 3.1.0
1433 The ``dropna`` keyword is deprecated and will be removed in a
1434 future version. Use :meth:`DataFrame.dropna` before writing
1435 instead.
1437 See Also
1438 --------
1439 HDFStore.info : Prints detailed information on the store.
1440 HDFStore.get_storer : Returns the storer object for a key.
1442 Notes
1443 -----
1444 Writing an empty ``DataFrame`` or ``Series`` with ``format='table'``
1445 or ``append=True`` is a no-op: the store is not modified and a
1446 ``UserWarning`` is emitted. Use ``format='fixed'`` to store an empty
1447 object.
1449 Examples
1450 --------
1451 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1452 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1453 >>> store.put("data", df) # doctest: +SKIP
1454 """
1455 if dropna is not lib.no_default:
1456 warnings.warn(
1457 "The 'dropna' keyword in HDFStore.put is deprecated and "
1458 "will be removed in a future version. Use DataFrame.dropna "
1459 "before writing instead.",
1460 Pandas4Warning,
1461 stacklevel=find_stack_level(),
1462 )
1463 else:
1464 dropna = False
1465 if format is None:
1466 format = config["io"]["hdf"]["default_format"] or "fixed"
1467 format = self._validate_format(format)
1468 if track_times is lib.no_default:
1469 warnings.warn(
1470 "The default value of 'track_times' in HDFStore.put will "
1471 "change from True to False in a future version. Pass "
1472 "track_times=False explicitly to silence this warning and "
1473 "get deterministic HDF5 files.",
1474 Pandas4Warning,
1475 stacklevel=find_stack_level(),
1476 )
1477 track_times = True
1478 self._write_to_group(
1479 key,
1480 value,
1481 format=format,
1482 index=index,
1483 append=append,
1484 complib=complib,
1485 complevel=complevel,
1486 min_itemsize=min_itemsize,
1487 nan_rep=nan_rep,
1488 data_columns=data_columns,
1489 encoding=encoding,
1490 errors=errors,
1491 track_times=track_times,
1492 dropna=dropna,
1493 )
1495 def remove(self, key: str, where=None, start=None, stop=None) -> int | None:
1496 """
1497 Remove pandas object partially by specifying the where condition.
1499 If ``where`` is not provided, the entire object stored at ``key``
1500 (and any of its child nodes) is removed. When ``where`` is given,
1501 only matching rows are deleted, which requires the object to be in
1502 ``table`` format.
1504 Parameters
1505 ----------
1506 key : str
1507 Node to remove or delete rows from.
1508 where : list of Term (or convertible) objects, optional
1509 Conditions selecting which rows to remove.
1510 start : int, optional
1511 Row number to start selection.
1512 stop : int, optional
1513 Row number to stop selection.
1515 Returns
1516 -------
1517 int or None
1518 Number of rows removed (or ``None`` if the object is not a Table).
1520 Raises
1521 ------
1522 KeyError
1523 If ``key`` is not a valid store.
1525 See Also
1526 --------
1527 HDFStore.append : Append data to an existing table.
1528 HDFStore.put : Store an object in the file.
1530 Examples
1531 --------
1532 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1533 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1534 >>> store.put("data", df) # doctest: +SKIP
1535 >>> store.remove("data") # doctest: +SKIP
1536 >>> store.close() # doctest: +SKIP
1537 """
1538 where = _ensure_term(where, scope_level=1)
1539 try:
1540 s = self.get_storer(key)
1541 except KeyError:
1542 # the key is not a valid store, re-raising KeyError
1543 raise
1544 except AssertionError:
1545 # surface any assertion errors for e.g. debugging
1546 raise
1547 except Exception as err:
1548 # In tests we get here with ClosedFileError, TypeError, and
1549 # _table_mod.NoSuchNodeError. TODO: Catch only these?
1551 if where is not None:
1552 raise ValueError(
1553 "trying to remove a node with a non-None where clause!"
1554 ) from err
1556 # we are actually trying to remove a node (with children)
1557 node = self.get_node(key)
1558 if node is not None:
1559 node._f_remove(recursive=True)
1560 return None
1562 # remove the node
1563 if com.all_none(where, start, stop):
1564 s.group._f_remove(recursive=True)
1565 return None
1567 # delete from the table
1568 if not s.is_table:
1569 raise ValueError("can only remove with where on objects written as tables")
1570 return s.delete(where=where, start=start, stop=stop)
1572 def append(
1573 self,
1574 key: str,
1575 value: DataFrame | Series,
1576 format=None,
1577 axes=None,
1578 index: bool | list[str] = True,
1579 append: bool = True,
1580 complib=None,
1581 complevel: int | None = None,
1582 columns=None,
1583 min_itemsize: int | dict[str, int] | None = None,
1584 nan_rep=None,
1585 chunksize: int | None = None,
1586 expectedrows=None,
1587 dropna: bool | lib.NoDefault | None = lib.no_default,
1588 data_columns: Literal[True] | list[str] | None = None,
1589 encoding=None,
1590 errors: str = "strict",
1591 ) -> None:
1592 """
1593 Append to Table in file.
1595 Node must already exist and be Table format.
1597 Parameters
1598 ----------
1599 key : str
1600 Key of object to append.
1601 value : {Series, DataFrame}
1602 Value of object to append.
1603 format : 'table' is the default
1604 Format to use when storing object in HDFStore. Value can be one of:
1606 ``'table'``
1607 Table format. Write as a PyTables Table structure which may perform
1608 worse but allow more flexible operations like searching / selecting
1609 subsets of the data.
1610 axes : list, default None
1611 Single-element list selecting the axis to store as the table's
1612 indexed (appendable) axis: ``[0]`` for the index (the default) or
1613 ``[1]`` for the columns. Ignored when appending to an existing
1614 table, which keeps the axes it was created with.
1615 index : bool, default True
1616 Write DataFrame index as a column.
1617 append : bool, default True
1618 Append the input data to the existing.
1619 complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default None
1620 Compression library to use; None disables compression. See the
1621 ``complib`` parameter of :class:`HDFStore` for the full list of
1622 supported compressors.
1623 complevel : int, 0-9, default None
1624 Specifies a compression level for data.
1625 A value of 0 or None disables compression.
1626 columns : default None
1627 This parameter is currently not accepted, try data_columns.
1628 min_itemsize : int, dict, or None
1629 Minimum number of bytes reserved for object columns.
1630 If int, all columns reserve 'min_itemsize' bytes per stored value.
1631 If dict, specific columns reserve 'min_itemsize' bytes per stored value.
1632 Strings are stored as encoded bytes. Since some characters require multiple
1633 bytes, required size may be larger than string length.
1634 nan_rep : str, optional
1635 String used on disk to represent missing values in string columns.
1636 By default a sentinel that collides with no value in the column is
1637 used, so a literal ``"nan"`` round-trips unchanged; when this is
1638 passed, a value equal to it is read back as a missing value.
1639 Only used when the table is created; ignored on later appends,
1640 which reuse whatever the table already stores.
1641 chunksize : int, default 100000
1642 Number of rows to write in each chunk.
1643 expectedrows : int
1644 Expected TOTAL row size of this table.
1645 dropna : bool, default False, optional
1646 Do not write an ALL nan row to the store settable
1647 by the option 'io.hdf.dropna_table'.
1649 .. deprecated:: 3.1.0
1650 The ``dropna`` keyword is deprecated and will be removed in a
1651 future version. Use :meth:`DataFrame.dropna` before writing
1652 instead.
1654 data_columns : list of columns, or True, default None
1655 List of columns to create as indexed data columns for on-disk
1656 queries, or True to use all columns. By default only the axes
1657 of the object are indexed. See `here
1658 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
1659 encoding : default None
1660 Provide an encoding for str.
1661 errors : str, default 'strict'
1662 The error handling scheme to use for encoding errors.
1663 The default is 'strict' meaning that encoding errors raise a
1664 UnicodeEncodeError. Other possible values are 'ignore', 'replace' and
1665 'xmlcharrefreplace' as well as any other name registered with
1666 codecs.register_error that can handle UnicodeEncodeErrors.
1668 See Also
1669 --------
1670 HDFStore.append_to_multiple : Append to multiple tables.
1672 Notes
1673 -----
1674 Does *not* check if data being appended overlaps with existing
1675 data in the table, so be careful
1677 Appending an empty ``DataFrame`` or ``Series`` is a no-op: the store
1678 is not modified and a ``UserWarning`` is emitted.
1680 Examples
1681 --------
1682 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1683 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1684 >>> store.put("data", df1, format="table") # doctest: +SKIP
1685 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["A", "B"])
1686 >>> store.append("data", df2) # doctest: +SKIP
1687 >>> store.close() # doctest: +SKIP
1688 A B
1689 0 1 2
1690 1 3 4
1691 0 5 6
1692 1 7 8
1693 """
1694 if columns is not None:
1695 raise TypeError(
1696 "columns is not a supported keyword in append, try data_columns"
1697 )
1699 if dropna is not lib.no_default:
1700 warnings.warn(
1701 "The 'dropna' keyword in HDFStore.append is deprecated and "
1702 "will be removed in a future version. Use DataFrame.dropna "
1703 "before writing instead.",
1704 Pandas4Warning,
1705 stacklevel=find_stack_level(),
1706 )
1707 dropna = bool(dropna) if dropna is not None else False
1708 else:
1709 dropna = False
1710 if format is None:
1711 format = config["io"]["hdf"]["default_format"] or "table"
1712 format = self._validate_format(format)
1713 self._write_to_group(
1714 key,
1715 value,
1716 format=format,
1717 axes=axes,
1718 index=index,
1719 append=append,
1720 complib=complib,
1721 complevel=complevel,
1722 min_itemsize=min_itemsize,
1723 nan_rep=nan_rep,
1724 chunksize=chunksize,
1725 expectedrows=expectedrows,
1726 dropna=dropna,
1727 data_columns=data_columns,
1728 encoding=encoding,
1729 errors=errors,
1730 )
1732 def append_to_multiple(
1733 self,
1734 d: dict,
1735 value,
1736 selector,
1737 data_columns=None,
1738 axes=None,
1739 dropna: bool | lib.NoDefault = lib.no_default,
1740 **kwargs,
1741 ) -> None:
1742 """
1743 Append to multiple tables.
1745 Splits ``value`` column-wise according to ``d`` and appends each
1746 slice to the corresponding table. The ``selector`` table is the one
1747 you query against; its columns are made data_columns so they can be
1748 used in ``where`` clauses.
1750 Parameters
1751 ----------
1752 d : dict
1753 Mapping of table_name to table_columns. ``None`` is acceptable as
1754 the values for one node (that table will get all the remaining
1755 columns).
1756 value : DataFrame or Series
1757 Pandas object to split across the tables.
1758 selector : str
1759 Designates the indexable table; all of its columns will be made
1760 data_columns unless ``data_columns`` is passed, in which case
1761 those are used.
1762 data_columns : list of str or True, optional
1763 Columns to create as data columns, or ``True`` to use all columns.
1764 axes : default None
1765 This parameter is currently not accepted.
1766 dropna : bool, default False
1767 If ``True``, drop rows from all tables if any single row in each
1768 table has all NaN.
1770 .. deprecated:: 3.1.0
1771 The ``dropna`` keyword is deprecated and will be removed in a
1772 future version. Use :meth:`DataFrame.dropna` before writing
1773 instead.
1774 **kwargs
1775 Additional keyword arguments forwarded to :meth:`HDFStore.append`.
1777 See Also
1778 --------
1779 HDFStore.append : Append to a single table.
1780 HDFStore.select_as_multiple : Read from multiple tables with a
1781 single ``where``.
1783 Examples
1784 --------
1785 >>> df = pd.DataFrame({"A": [1, 2], "B": [3, 4], "C": [5, 6]}) # doctest: +SKIP
1786 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1787 >>> store.append_to_multiple(
1788 ... {"t1": ["A", "B"], "t2": None}, df, selector="t1"
1789 ... ) # doctest: +SKIP
1790 >>> store.close() # doctest: +SKIP
1791 """
1792 if axes is not None:
1793 raise TypeError(
1794 "axes is currently not accepted as a parameter to append_to_multiple; "
1795 "you can create the tables independently instead"
1796 )
1798 if not isinstance(d, dict):
1799 raise ValueError(
1800 "append_to_multiple must have a dictionary specified as the "
1801 "way to split the value"
1802 )
1804 if selector not in d:
1805 raise ValueError(
1806 "append_to_multiple requires a selector that is in passed dict"
1807 )
1809 # figure out the splitting axis (the non_index_axis)
1810 axis = next(iter(set(range(value.ndim)) - set(_AXES_MAP[type(value)])))
1812 # figure out how to split the value
1813 remain_key = None
1814 remain_values: list = []
1815 for k, v in d.items():
1816 if v is None:
1817 if remain_key is not None:
1818 raise ValueError(
1819 "append_to_multiple can only have one value in d that is None"
1820 )
1821 remain_key = k
1822 else:
1823 remain_values.extend(v)
1824 if remain_key is not None:
1825 ordered = value.axes[axis]
1826 ordd = ordered.difference(Index(remain_values))
1827 ordd = sorted(ordered.get_indexer(ordd))
1828 d[remain_key] = ordered.take(ordd)
1830 # data_columns
1831 if data_columns is None:
1832 data_columns = d[selector]
1834 # ensure rows are synchronized across the tables
1835 if dropna is not lib.no_default:
1836 warnings.warn(
1837 "The 'dropna' keyword in HDFStore.append_to_multiple is "
1838 "deprecated and will be removed in a future version. Use "
1839 "DataFrame.dropna before writing instead.",
1840 Pandas4Warning,
1841 stacklevel=find_stack_level(),
1842 )
1843 else:
1844 dropna = False
1845 if dropna:
1846 idxs = (value[cols].dropna(how="all").index for cols in d.values())
1847 valid_index = next(idxs)
1848 for index in idxs:
1849 valid_index = valid_index.intersection(index)
1850 value = value.loc[valid_index]
1852 min_itemsize = kwargs.pop("min_itemsize", None)
1854 # append
1855 for k, v in d.items():
1856 dc = data_columns if k == selector else None
1858 # compute the val
1859 val = value.reindex(v, axis=axis)
1861 filtered = (
1862 {key: value for (key, value) in min_itemsize.items() if key in v}
1863 if min_itemsize is not None
1864 else None
1865 )
1866 self.append(k, val, data_columns=dc, min_itemsize=filtered, **kwargs)
1868 def create_table_index(
1869 self,
1870 key: str,
1871 columns=None,
1872 optlevel: int | None = None,
1873 kind: str | None = None,
1874 ) -> None:
1875 """
1876 Create a pytables index on the table.
1878 Indexes are automatically created on indexable axes and
1879 ``data_columns`` during ``append``/``put`` when ``index=True``
1880 (the default). This method lets you add or rebuild indexes after
1881 the fact, which is **highly encouraged** because it greatly speeds
1882 up ``select`` calls that filter on the indexed dimension.
1884 Parameters
1885 ----------
1886 key : str
1887 Object stored in the file to index.
1888 columns : None, bool, or listlike[str]
1889 Indicate which columns to create an index on.
1891 * False : Do not create any indexes.
1892 * True : Create indexes on all columns.
1893 * None : Create indexes on all columns.
1894 * listlike : Create indexes on the given columns.
1896 optlevel : int or None, default None
1897 Optimization level, if None, pytables defaults to 6.
1898 kind : str or None, default None
1899 Kind of index, if None, pytables defaults to "medium".
1901 Raises
1902 ------
1903 TypeError
1904 If the node is not a table.
1906 See Also
1907 --------
1908 HDFStore.append : Append data to an existing table; columns are
1909 indexed automatically by default.
1910 HDFStore.select : Filter on indexed columns for fast retrieval.
1912 Examples
1913 --------
1914 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1915 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1916 >>> store.append("data", df, format="table", index=False) # doctest: +SKIP
1917 >>> store.create_table_index("data", columns=["A"]) # doctest: +SKIP
1918 >>> store.close() # doctest: +SKIP
1919 """
1920 # version requirements
1921 _tables()
1922 s = self.get_storer(key)
1923 if s is None:
1924 return
1926 if not isinstance(s, Table):
1927 raise TypeError("cannot create table index on a Fixed format store")
1928 s.create_index(columns=columns, optlevel=optlevel, kind=kind)
1930 def groups(self) -> list:
1931 """
1932 Return a list of all the top-level nodes.
1934 Each node returned is not a pandas storage object.
1936 Returns
1937 -------
1938 list
1939 List of objects.
1941 See Also
1942 --------
1943 HDFStore.get_node : Returns the node with the key.
1945 Examples
1946 --------
1947 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1948 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1949 >>> store.put("data", df) # doctest: +SKIP
1950 >>> print(store.groups()) # doctest: +SKIP
1951 >>> store.close() # doctest: +SKIP
1952 [/data (Group) ''
1953 children := ['axis0' (Array), 'axis1' (Array), 'block0_values' (Array),
1954 'block0_items' (Array)]]
1955 """
1956 _tables()
1957 self._check_if_open()
1958 assert self._handle is not None # for mypy
1959 assert _table_mod is not None # for mypy
1960 return [
1961 g
1962 for g in self._handle.walk_groups()
1963 if (
1964 not isinstance(g, _table_mod.link.Link)
1965 and (
1966 getattr(g._v_attrs, "pandas_type", None)
1967 or getattr(g, "table", None)
1968 or (isinstance(g, _table_mod.table.Table) and g._v_name != "table")
1969 )
1970 )
1971 ]
1973 def walk(self, where: str = "/") -> Iterator[tuple[str, list[str], list[str]]]:
1974 """
1975 Walk the pytables group hierarchy for pandas objects.
1977 This generator will yield the group path, subgroups and pandas object
1978 names for each group.
1980 Any non-pandas PyTables objects that are not a group will be ignored.
1982 The `where` group itself is listed first (preorder), then each of its
1983 child groups (following an alphanumerical order) is also traversed,
1984 following the same procedure.
1986 Parameters
1987 ----------
1988 where : str, default "/"
1989 Group where to start walking.
1991 Yields
1992 ------
1993 path : str
1994 Full path to a group (without trailing '/').
1995 groups : list
1996 Names (strings) of the groups contained in `path`.
1997 leaves : list
1998 Names (strings) of the pandas objects contained in `path`.
2000 See Also
2001 --------
2002 HDFStore.info : Prints detailed information on the store.
2004 Examples
2005 --------
2006 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
2007 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
2008 >>> store.put("data", df1, format="table") # doctest: +SKIP
2009 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["A", "B"])
2010 >>> store.append("data", df2) # doctest: +SKIP
2011 >>> store.close() # doctest: +SKIP
2012 >>> for group in store.walk(): # doctest: +SKIP
2013 ... print(group) # doctest: +SKIP
2014 >>> store.close() # doctest: +SKIP
2015 """
2016 _tables()
2017 self._check_if_open()
2018 assert self._handle is not None # for mypy
2019 assert _table_mod is not None # for mypy
2021 for g in self._handle.walk_groups(where):
2022 if getattr(g._v_attrs, "pandas_type", None) is not None:
2023 continue
2025 groups = []
2026 leaves = []
2027 for child in g._v_children.values():
2028 pandas_type = getattr(child._v_attrs, "pandas_type", None)
2029 if pandas_type is None:
2030 if isinstance(child, _table_mod.group.Group):
2031 groups.append(child._v_name)
2032 else:
2033 leaves.append(child._v_name)
2035 yield (g._v_pathname.rstrip("/"), groups, leaves)
2037 def get_node(self, key: str) -> Node | None:
2038 """return the node with the key or None if it does not exist"""
2039 self._check_if_open()
2040 if not key.startswith("/"):
2041 key = "/" + key
2043 assert self._handle is not None
2044 assert _table_mod is not None # for mypy
2045 try:
2046 node = self._handle.get_node(self.root, key)
2047 except _table_mod.exceptions.NoSuchNodeError:
2048 return None
2050 assert isinstance(node, _table_mod.Node), type(node)
2051 return node
2053 def get_storer(self, key: str) -> GenericFixed | Table:
2054 """
2055 Return the storer object for a key.
2057 The storer is the low-level wrapper around the stored pandas object.
2058 It exposes implementation details such as ``nrows`` (the row count
2059 on disk) and ``table`` (the underlying PyTables ``Table``), which
2060 can be useful for inspecting a store without loading its data.
2062 Parameters
2063 ----------
2064 key : str
2065 Object stored in the file.
2067 Returns
2068 -------
2069 GenericFixed or Table
2070 The storer wrapping the stored object. ``Table`` instances expose
2071 attributes such as ``nrows`` and ``table``.
2073 Raises
2074 ------
2075 KeyError
2076 If ``key`` is not in the file.
2078 See Also
2079 --------
2080 HDFStore.get : Read the stored object back into pandas.
2081 HDFStore.info : Print a summary of the store's contents.
2083 Examples
2084 --------
2085 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
2086 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
2087 >>> store.append("data", df, format="table") # doctest: +SKIP
2088 >>> store.get_storer("data").nrows # doctest: +SKIP
2089 >>> store.close() # doctest: +SKIP
2090 """
2091 group = self.get_node(key)
2092 if group is None:
2093 raise KeyError(f"No object named {key} in the file")
2095 s = self._create_storer(group)
2096 s.infer_axes()
2097 return s
2099 def copy(
2100 self,
2101 file,
2102 mode: str = "w",
2103 propindexes: bool = True,
2104 keys=None,
2105 complib=None,
2106 complevel: int | None = None,
2107 fletcher32: bool = False,
2108 overwrite: bool = True,
2109 ) -> HDFStore:
2110 """
2111 Copy the existing store to a new file, updating in place.
2113 Each object stored in the current ``HDFStore`` is read and re-written
2114 to the destination file, which can be useful for repacking (to
2115 reclaim space after deletes), changing compression options, or
2116 cloning a subset of keys.
2118 Parameters
2119 ----------
2120 file : str or path-like
2121 Destination file to write to.
2122 mode : str, default 'w'
2123 File mode for the destination, see :class:`HDFStore`.
2124 propindexes : bool, default True
2125 Restore indexes in copied file.
2126 keys : list, optional
2127 List of keys to include in the copy (defaults to all).
2128 complib : str, optional
2129 Compression library, see :class:`HDFStore`.
2130 complevel : int, optional
2131 Compression level, see :class:`HDFStore`.
2132 fletcher32 : bool, default False
2133 Whether to use the Fletcher32 checksum, see :class:`HDFStore`.
2134 overwrite : bool, default True
2135 Whether to overwrite (remove and replace) existing nodes in the
2136 new store.
2138 Returns
2139 -------
2140 HDFStore
2141 Open file handle of the new store.
2143 See Also
2144 --------
2145 HDFStore.put : Write an object to the store.
2146 HDFStore.append : Append to an existing table.
2148 Examples
2149 --------
2150 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
2151 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
2152 >>> store.put("data", df) # doctest: +SKIP
2153 >>> new_store = store.copy("store_copy.h5") # doctest: +SKIP
2154 >>> new_store.close() # doctest: +SKIP
2155 >>> store.close() # doctest: +SKIP
2156 """
2157 new_store = HDFStore(
2158 file, mode=mode, complib=complib, complevel=complevel, fletcher32=fletcher32
2159 )
2160 if keys is None:
2161 keys = list(self.keys())
2162 if not isinstance(keys, (tuple, list)):
2163 keys = [keys]
2164 for k in keys:
2165 s = self.get_storer(k)
2166 if s is not None:
2167 if k in new_store:
2168 if overwrite:
2169 new_store.remove(k)
2171 data = self.select(k)
2172 if isinstance(s, Table):
2173 index: bool | list[str] = False
2174 if propindexes:
2175 index = [a.name for a in s.axes if a.is_indexed]
2176 new_store.append(
2177 k,
2178 data,
2179 index=index,
2180 data_columns=getattr(s, "data_columns", None),
2181 encoding=s.encoding,
2182 )
2183 else:
2184 new_store.put(k, data, encoding=s.encoding, track_times=False)
2186 return new_store
2188 def info(self) -> str:
2189 """
2190 Print detailed information on the store.
2192 The information includes the file path, class name, and a listing
2193 of all stored object keys with their types and shapes.
2195 Returns
2196 -------
2197 str
2198 A String containing the python pandas class name, filepath to the HDF5
2199 file and all the object keys along with their respective dataframe shapes.
2201 See Also
2202 --------
2203 HDFStore.get_storer : Returns the storer object for a key.
2205 Examples
2206 --------
2207 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
2208 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["C", "D"])
2209 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
2210 >>> store.put("data1", df1) # doctest: +SKIP
2211 >>> store.put("data2", df2) # doctest: +SKIP
2212 >>> print(store.info()) # doctest: +SKIP
2213 >>> store.close() # doctest: +SKIP
2214 <class 'pandas.io.pytables.HDFStore'>
2215 File path: store.h5
2216 /data1 frame (shape->[2,2])
2217 /data2 frame (shape->[2,2])
2218 """
2219 path = pprint_thing(self._path)
2220 output = f"{type(self)}\nFile path: {path}\n"
2222 if self.is_open:
2223 lkeys = sorted(self.keys())
2224 if lkeys:
2225 keys = []
2226 values = []
2228 for k in lkeys:
2229 try:
2230 s = self.get_storer(k)
2231 if s is not None:
2232 keys.append(pprint_thing(s.pathname or k))
2233 values.append(pprint_thing(s or "invalid_HDFStore node"))
2234 except AssertionError:
2235 # surface any assertion errors for e.g. debugging
2236 raise
2237 except Exception as detail:
2238 keys.append(k)
2239 dstr = pprint_thing(detail)
2240 values.append(f"[invalid_HDFStore node: {dstr}]")
2242 output += adjoin(12, keys, values)
2243 else:
2244 output += "Empty"
2245 else:
2246 output += "File is CLOSED"
2248 return output
2250 # ------------------------------------------------------------------------
2251 # private methods
2253 def _check_if_open(self) -> None:
2254 if not self.is_open:
2255 raise ClosedFileError(f"{self._path} file is not open!")
2257 def _validate_format(self, format: str) -> str:
2258 """validate / deprecate formats"""
2259 # validate
2260 try:
2261 format = _FORMAT_MAP[format.lower()]
2262 except KeyError as err:
2263 raise TypeError(f"invalid HDFStore format specified [{format}]") from err
2265 return format
2267 def _create_storer(
2268 self,
2269 group,
2270 format=None,
2271 value: DataFrame | Series | None = None,
2272 encoding: str = "UTF-8",
2273 errors: str = "strict",
2274 ) -> GenericFixed | Table:
2275 """return a suitable class to operate"""
2276 cls: type[GenericFixed | Table]
2278 if value is not None and not isinstance(value, (Series, DataFrame)):
2279 raise TypeError("value must be None, Series, or DataFrame")
2281 pt = getattr(group._v_attrs, "pandas_type", None)
2282 tt = getattr(group._v_attrs, "table_type", None)
2284 # infer the pt from the passed value
2285 if pt is None:
2286 if value is None:
2287 _tables()
2288 assert _table_mod is not None # for mypy
2289 if getattr(group, "table", None) or isinstance(
2290 group, _table_mod.table.Table
2291 ):
2292 pt = "frame_table"
2293 tt = "generic_table"
2294 else:
2295 raise TypeError(
2296 "cannot create a storer if the object is not existing "
2297 "nor a value are passed"
2298 )
2299 else:
2300 if isinstance(value, Series):
2301 pt = "series"
2302 else:
2303 pt = "frame"
2305 # we are actually a table
2306 if format == "table":
2307 pt += "_table"
2309 # a storer node
2310 if "table" not in pt:
2311 _STORER_MAP = {"series": SeriesFixed, "frame": FrameFixed}
2312 try:
2313 cls = _STORER_MAP[pt]
2314 except KeyError as err:
2315 raise TypeError(
2316 f"cannot properly create the storer for: [_STORER_MAP] [group->"
2317 f"{group},value->{type(value)},format->{format}"
2318 ) from err
2319 return cls(self, group, encoding=encoding, errors=errors)
2321 # existing node (and must be a table)
2322 if tt is None:
2323 # if we are a writer, determine the tt
2324 if value is not None:
2325 if pt == "series_table":
2326 index = getattr(value, "index", None)
2327 if index is not None:
2328 if index.nlevels == 1:
2329 tt = "appendable_series"
2330 elif index.nlevels > 1:
2331 tt = "appendable_multiseries"
2332 elif pt == "frame_table":
2333 index = getattr(value, "index", None)
2334 if index is not None:
2335 if index.nlevels == 1:
2336 tt = "appendable_frame"
2337 elif index.nlevels > 1:
2338 tt = "appendable_multiframe"
2340 _TABLE_MAP = {
2341 "generic_table": GenericTable,
2342 "appendable_series": AppendableSeriesTable,
2343 "appendable_multiseries": AppendableMultiSeriesTable,
2344 "appendable_frame": AppendableFrameTable,
2345 "appendable_multiframe": AppendableMultiFrameTable,
2346 "worm": WORMTable,
2347 }
2348 try:
2349 cls = _TABLE_MAP[tt] # type: ignore[index]
2350 except KeyError as err:
2351 raise TypeError(
2352 f"cannot properly create the storer for: [_TABLE_MAP] [group->"
2353 f"{group},value->{type(value)},format->{format}"
2354 ) from err
2356 return cls(self, group, encoding=encoding, errors=errors)
2358 def _write_to_group(
2359 self,
2360 key: str,
2361 value: DataFrame | Series,
2362 format,
2363 axes=None,
2364 index: bool | list[str] = True,
2365 append: bool = False,
2366 complib=None,
2367 complevel: int | None = None,
2368 fletcher32=None,
2369 min_itemsize: int | dict[str, int] | None = None,
2370 chunksize: int | None = None,
2371 expectedrows=None,
2372 dropna: bool = False,
2373 nan_rep=None,
2374 data_columns=None,
2375 encoding=None,
2376 errors: str = "strict",
2377 track_times: bool = True,
2378 ) -> None:
2379 # we don't want to store a table node at all if our object is 0-len
2380 # as there are no dtypes
2381 if getattr(value, "empty", None) and (format == "table" or append):
2382 warnings.warn(
2383 "Writing an empty DataFrame or Series with format='table' "
2384 "or append=True is a no-op; the HDFStore is not modified.",
2385 UserWarning,
2386 stacklevel=find_stack_level(),
2387 )
2388 return
2390 group = self._identify_group(key, append)
2392 s = self._create_storer(group, format, value, encoding=encoding, errors=errors)
2393 if append:
2394 # raise if we are trying to append to a Fixed format,
2395 # or a table that exists (and we are putting)
2396 if not s.is_table or (s.is_table and format == "fixed" and s.is_exists):
2397 raise ValueError(
2398 f"Can only append to Tables; the write for key {key!r} "
2399 "uses the 'fixed' format. Pass format='table'; if a "
2400 "'fixed'-format object already exists at this key, remove "
2401 "or overwrite it first."
2402 )
2403 if not s.is_exists:
2404 s.set_object_info()
2405 else:
2406 s.set_object_info()
2408 if not s.is_table and complib:
2409 raise ValueError("Compression not supported on Fixed format stores")
2411 # write the object
2412 s.write(
2413 obj=value,
2414 axes=axes,
2415 append=append,
2416 complib=complib,
2417 complevel=complevel,
2418 fletcher32=fletcher32,
2419 min_itemsize=min_itemsize,
2420 chunksize=chunksize,
2421 expectedrows=expectedrows,
2422 dropna=dropna,
2423 nan_rep=nan_rep,
2424 data_columns=data_columns,
2425 track_times=track_times,
2426 )
2428 if isinstance(s, Table) and index:
2429 s.create_index(columns=index)
2431 def _read_group(self, group: Node):
2432 s = self._create_storer(group)
2433 s.infer_axes()
2434 return s.read()
2436 def _identify_group(self, key: str, append: bool) -> Node:
2437 """Identify HDF5 group based on key, delete/create group if needed."""
2438 group = self.get_node(key)
2440 # we make this assertion for mypy; the get_node call will already
2441 # have raised if this is incorrect
2442 assert self._handle is not None
2444 # remove the node if we are not appending
2445 if group is not None and not append:
2446 assert _table_mod is not None # for mypy
2447 children = list(group._v_children.values())
2448 # a "meta" subgroup of a stored object holds its metadata (e.g.
2449 # categories for table-format categoricals), not a nested key
2450 is_stored_object = getattr(group._v_attrs, "pandas_type", None) is not None
2451 nested_keys = [
2452 child
2453 for child in children
2454 if isinstance(child, _table_mod.group.Group)
2455 and not (is_stored_object and child._v_name == "meta")
2456 ]
2457 if nested_keys:
2458 # GH#17267: the group has child keys nested underneath it, so a
2459 # recursive removal would silently delete them. Remove only the
2460 # nodes of the object stored at this key and reset its
2461 # attributes, leaving the nested keys intact.
2462 for child in children:
2463 if child not in nested_keys:
2464 self._handle.remove_node(child, recursive=True)
2465 for attr_name in group._v_attrs._f_list("user"):
2466 delattr(group._v_attrs, attr_name)
2467 else:
2468 self._handle.remove_node(group, recursive=True)
2469 group = None
2471 if group is None:
2472 group = self._create_nodes_and_group(key)
2474 return group
2476 def _create_nodes_and_group(self, key: str) -> Node:
2477 """Create nodes from key and return group name."""
2478 # assertion for mypy
2479 assert self._handle is not None
2481 paths = key.split("/")
2482 # recursively create the groups
2483 path = "/"
2484 for p in paths:
2485 if not len(p):
2486 continue
2487 new_path = path
2488 if not path.endswith("/"):
2489 new_path += "/"
2490 new_path += p
2491 group = self.get_node(new_path)
2492 if group is None:
2493 group = self._handle.create_group(path, p)
2494 path = new_path
2495 return group
2498class TableIterator:
2499 """
2500 Define the iteration interface on a table
2502 Parameters
2503 ----------
2504 store : HDFStore
2505 s : the referred storer
2506 func : the function to execute the query
2507 where : the where of the query
2508 nrows : the rows to iterate on
2509 start : the passed start value (default is None)
2510 stop : the passed stop value (default is None)
2511 iterator : bool, default False
2512 Whether to use the default iterator.
2513 chunksize : the passed chunking value (default is 100000)
2514 auto_close : bool, default False
2515 Whether to automatically close the store at the end of iteration.
2516 """
2518 chunksize: int | None
2519 store: HDFStore
2520 s: GenericFixed | Table
2522 def __init__(
2523 self,
2524 store: HDFStore,
2525 s: GenericFixed | Table,
2526 func,
2527 where,
2528 nrows,
2529 start=None,
2530 stop=None,
2531 iterator: bool = False,
2532 chunksize: int | None = None,
2533 auto_close: bool = False,
2534 ) -> None:
2535 self.store = store
2536 self.s = s
2537 self.func = func
2538 self.where = where
2540 # set start/stop if they are not set if we are a table
2541 if self.s.is_table:
2542 if nrows is None:
2543 nrows = 0
2544 start, stop = _resolve_row_window(start, stop, nrows)
2546 self.nrows = nrows
2547 self.start = start
2548 self.stop = stop
2550 self.coordinates = None
2551 self._called_get_result = False
2552 if iterator or chunksize is not None:
2553 if chunksize is None:
2554 chunksize = 100000
2555 self.chunksize = int(chunksize)
2556 else:
2557 self.chunksize = None
2559 self.auto_close = auto_close
2561 def __iter__(self) -> Iterator:
2562 # iterate
2563 if not self._called_get_result:
2564 raise ValueError("Cannot iterate until get_result is called.")
2565 current = self.start
2566 while current < self.stop:
2567 stop = min(current + self.chunksize, self.stop)
2568 if self.coordinates is None:
2569 # no `where` filter: iterate by (start, stop) directly so we
2570 # don't materialize an np.arange(0, nrows) coordinate array,
2571 # which can exhaust memory on very large tables (GH#15937)
2572 value = self.func(current, stop, None)
2573 else:
2574 value = self.func(None, None, self.coordinates[current:stop])
2575 current = stop
2576 if value is None or not len(value):
2577 continue
2579 yield value
2581 self.close()
2583 def close(self) -> None:
2584 if self.auto_close:
2585 self.store.close()
2587 def get_result(self, coordinates: bool = False):
2588 # return the actual iterator
2589 self._called_get_result = True
2590 if self.chunksize is not None:
2591 if not isinstance(self.s, Table):
2592 raise TypeError("can only use an iterator or chunksize on a table")
2594 if self.where is not None:
2595 # read_coordinates raises a clear NotImplementedError if the
2596 # where is a column projection (e.g. "columns=['A']") rather
2597 # than a row filter (GH#12953).
2598 self.coordinates = self.s.read_coordinates(where=self.where)
2600 return self
2602 # if specified read via coordinates (necessary for multiple selections
2603 # so each table reads the same row set). Skip when self.where is None
2604 # since every row would be selected anyway, and a coordinate-based read
2605 # is much slower than a sequential read (GH#26771).
2606 if coordinates and self.where is not None:
2607 if not isinstance(self.s, Table):
2608 raise TypeError("can only read_coordinates on a table")
2609 where = self.s.read_coordinates(
2610 where=self.where, start=self.start, stop=self.stop
2611 )
2612 else:
2613 where = self.where
2615 # directly return the result
2616 results = self.func(self.start, self.stop, where)
2617 self.close()
2618 return results
2621class IndexCol:
2622 """
2623 an index column description class
2625 Parameters
2626 ----------
2627 axis : axis which I reference
2628 values : the ndarray like converted values
2629 kind : a string description of this type
2630 typ : the pytables type
2631 pos : the position in the pytables
2633 """
2635 is_an_indexable: bool = True
2636 is_data_indexable: bool = True
2637 # GH#9604: set on a DataCol whose missing values use the sentinel in
2638 # ``nan_rep`` rather than a table-wide one (an old file, or a table written
2639 # with an explicit nan_rep).
2640 uses_col_nan_rep: bool = False
2641 _info_fields = ["freq", "tz", "index_name", "ordered"]
2643 def __init__(
2644 self,
2645 name: str,
2646 values=None,
2647 kind=None,
2648 typ=None,
2649 cname: str | None = None,
2650 axis=None,
2651 pos=None,
2652 freq=None,
2653 tz=None,
2654 index_name=None,
2655 ordered=None,
2656 table=None,
2657 meta=None,
2658 metadata=None,
2659 nan_rep=None,
2660 ) -> None:
2661 if not isinstance(name, str):
2662 raise ValueError("`name` must be a str.")
2664 self.values = values
2665 self.kind = kind
2666 self.typ = typ
2667 self.name = name
2668 self.cname = cname or name
2669 self.axis = axis
2670 self.pos = pos
2671 self.freq = freq
2672 self.tz = tz
2673 self.index_name = index_name
2674 self.ordered = ordered
2675 self.table = table
2676 self.meta = meta
2677 self.metadata = metadata
2678 # GH#9604: the sentinel used on write to encode this column's missing
2679 # values (None when it had none).
2680 self.nan_rep = nan_rep
2682 if pos is not None:
2683 self.set_pos(pos)
2685 # These are ensured as long as the passed arguments match the
2686 # constructor annotations.
2687 assert isinstance(self.name, str)
2688 assert isinstance(self.cname, str)
2690 @property
2691 def itemsize(self) -> int:
2692 # Assumes self.typ has already been initialized
2693 return self.typ.itemsize
2695 @property
2696 def kind_attr(self) -> str:
2697 return f"{self.name}_kind"
2699 @property
2700 def meta_attr(self) -> str:
2701 return f"{self.name}_meta"
2703 @property
2704 def nan_rep_attr(self) -> str:
2705 return f"{self.name}_nan_rep"
2707 def set_pos(self, pos: int) -> None:
2708 """set the position of this column in the Table"""
2709 self.pos = pos
2710 if pos is not None and self.typ is not None:
2711 self.typ._v_pos = pos
2713 def __repr__(self) -> str:
2714 temp = tuple(
2715 map(pprint_thing, (self.name, self.cname, self.axis, self.pos, self.kind))
2716 )
2717 return ",".join(
2718 [
2719 f"{key}->{value}"
2720 for key, value in zip(
2721 ["name", "cname", "axis", "pos", "kind"], temp, strict=True
2722 )
2723 ]
2724 )
2726 def __eq__(self, other: object) -> bool:
2727 """compare 2 col items"""
2728 return all(
2729 getattr(self, a, None) == getattr(other, a, None)
2730 for a in ["name", "cname", "axis", "pos"]
2731 )
2733 def __ne__(self, other) -> bool:
2734 return not self.__eq__(other)
2736 @property
2737 def is_indexed(self) -> bool:
2738 """return whether I am an indexed column"""
2739 if not hasattr(self.table, "cols"):
2740 # e.g. if infer hasn't been called yet, self.table will be None.
2741 return False
2742 return getattr(self.table.cols, self.cname).is_indexed
2744 def convert(
2745 self, values: np.ndarray, nan_rep, encoding: str, errors: str
2746 ) -> tuple[np.ndarray, np.ndarray] | tuple[Index, Index]:
2747 """
2748 Convert the data from this selection to the appropriate pandas type.
2749 """
2750 assert isinstance(values, np.ndarray), type(values)
2752 # values is a recarray
2753 if values.dtype.fields is not None:
2754 # Copy, otherwise values will be a view
2755 # preventing the original recarry from being free'ed
2756 values = values[self.cname].copy()
2758 if self.meta == "category":
2759 # GH#33909, GH#16118: reconstruct a CategoricalIndex from the
2760 # stored integer codes and the categories saved as metadata.
2761 categories = self.metadata
2762 codes = values.ravel()
2764 # Mirror DataCol.convert's NaN-category handling so a
2765 # CategoricalIndex round-trips through format="table". GH#65576
2766 if categories is None:
2767 # Zero-category (all-NaN) case: PyTables cannot store an
2768 # empty metadata array, so it round-trips as None.
2769 categories = Index([], dtype=np.float64)
2770 else:
2771 mask = isna(categories)
2772 if mask.any():
2773 # A category decodes to NaN when it equaled the nan_rep
2774 # string on write; drop NaN categories and remap the
2775 # codes so from_codes does not reject a null category.
2776 remap = np.full(len(categories), -1, dtype=codes.dtype)
2777 remap[~mask] = np.arange((~mask).sum(), dtype=codes.dtype)
2778 categories = categories[~mask]
2779 codes = np.where(codes < 0, codes, remap[codes])
2781 cat = Categorical.from_codes(
2782 codes,
2783 categories=categories,
2784 ordered=bool(self.ordered),
2785 validate=False,
2786 )
2787 cat_index = CategoricalIndex._simple_new(cat, name=self.index_name)
2788 return cat_index, cat_index
2790 val_kind = self.kind
2791 # GH#9604: self.nan_rep (set from the persisted per-index sentinel) lets
2792 # a string Index restore missing values while leaving a literal "nan".
2793 values = _maybe_convert(values, val_kind, encoding, errors, self.nan_rep)
2794 kwargs = {}
2795 kwargs["name"] = self.index_name
2797 if val_kind == "string" and using_string_dtype():
2798 # GH#9604: an all-missing string Index -- or an all-missing
2799 # slice/chunk of one -- cannot be inferred back to str, which
2800 # would make the index dtype depend on which rows were read.
2801 kwargs["dtype"] = "str"
2803 if self.freq is not None:
2804 kwargs["freq"] = self.freq
2806 factory: type[Index | DatetimeIndex | TimedeltaIndex] = Index
2807 if lib.is_np_dtype(values.dtype, "M") or isinstance(
2808 values.dtype, DatetimeTZDtype
2809 ):
2810 factory = DatetimeIndex
2811 elif val_kind.startswith("timedelta64"):
2812 # GH#21466 timedelta values are stored as i8; restore the original
2813 # m8[unit] view so we round-trip to TimedeltaIndex (and not to
2814 # PeriodIndex/Index via the i8 branches below).
2815 if val_kind == "timedelta64":
2816 # legacy file: written before we stored timedelta64 resolution
2817 values = values.view("m8[ns]")
2818 else:
2819 values = values.view(val_kind)
2820 factory = TimedeltaIndex
2821 elif values.dtype == "i8" and "freq" in kwargs:
2822 # PeriodIndex data is stored as i8
2823 # error: Incompatible types in assignment (expression has type
2824 # "Callable[[Any, KwArg(Any)], PeriodIndex]", variable has type
2825 # "Union[Type[Index], Type[DatetimeIndex]]")
2826 factory = lambda x, **kwds: PeriodIndex.from_ordinals( # type: ignore[assignment]
2827 x, freq=kwds.get("freq", None)
2828 )._rename(kwds["name"])
2830 # making an Index instance could throw a number of different errors
2831 try:
2832 new_pd_index = factory(values, **kwargs)
2833 except UnicodeEncodeError as err:
2834 if (
2835 errors == "surrogatepass"
2836 and using_string_dtype()
2837 and str(err).endswith("surrogates not allowed")
2838 and HAS_PYARROW
2839 ):
2840 kwargs["dtype"] = StringDtype(storage="python", na_value=np.nan)
2841 new_pd_index = factory(values, **kwargs)
2842 else:
2843 raise
2844 except ValueError:
2845 # if the output freq is different that what we recorded,
2846 # it should be None (see also 'doc example part 2')
2847 if "freq" in kwargs:
2848 kwargs["freq"] = None
2849 new_pd_index = factory(values, **kwargs)
2851 final_pd_index: Index
2852 if self.tz is not None and isinstance(new_pd_index, DatetimeIndex):
2853 final_pd_index = new_pd_index.tz_localize("UTC").tz_convert(self.tz)
2854 else:
2855 final_pd_index = new_pd_index
2856 return final_pd_index, final_pd_index
2858 def take_data(self):
2859 """return the values"""
2860 return self.values
2862 @property
2863 def attrs(self):
2864 return self.table._v_attrs
2866 @property
2867 def description(self):
2868 return self.table.description
2870 @property
2871 def col(self):
2872 """return my current col description"""
2873 return getattr(self.description, self.cname, None)
2875 @property
2876 def cvalues(self):
2877 """return my cython values"""
2878 return self.values
2880 def __iter__(self) -> Iterator:
2881 return iter(self.values)
2883 def maybe_set_size(self, min_itemsize=None) -> None:
2884 """
2885 maybe set a string col itemsize:
2886 min_itemsize can be an integer or a dict with this columns name
2887 with an integer size
2888 """
2889 if self.kind == "string":
2890 if isinstance(min_itemsize, dict):
2891 min_itemsize = min_itemsize.get(self.name)
2893 if min_itemsize is not None and self.typ.itemsize < min_itemsize:
2894 self.typ = _tables().StringCol(itemsize=min_itemsize, pos=self.pos)
2896 def validate_names(self) -> None:
2897 pass
2899 def validate_and_set(self, handler: AppendableTable, append: bool) -> None:
2900 self.table = handler.table
2901 self.validate_col()
2902 self.validate_attr(append)
2903 self.validate_metadata(handler)
2904 self.write_metadata(handler)
2905 self.set_attr()
2907 def validate_col(self, itemsize=None):
2908 """validate this column: return the compared against itemsize"""
2909 # validate this column for string truncation (or reset to the max size)
2910 if self.kind == "string":
2911 c = self.col
2912 if c is not None:
2913 if itemsize is None:
2914 itemsize = self.itemsize
2915 if c.itemsize < itemsize:
2916 raise ValueError(
2917 f"Trying to store a string with len [{itemsize}] in "
2918 f"[{self.cname}] column but\nthis column has a limit of "
2919 f"[{c.itemsize}]!\nConsider using min_itemsize to "
2920 "preset the sizes on these columns"
2921 )
2922 return c.itemsize
2924 return None
2926 def validate_attr(self, append: bool) -> None:
2927 # check for backwards incompatibility
2928 if append:
2929 existing_kind = getattr(self.attrs, self.kind_attr, None)
2930 if existing_kind is None:
2931 # brand new table, nothing to be incompatible with
2932 return
2933 if existing_kind != self.kind:
2934 raise TypeError(
2935 f"incompatible kind in col [{existing_kind} - {self.kind}]"
2936 )
2937 existing_meta = getattr(self.attrs, self.meta_attr, None)
2938 if existing_meta != self.meta:
2939 # A CategoricalIndex stores its codes, so both sides are kind
2940 # "integer" and the check above passes; without this the append
2941 # silently reinterprets codes as labels or vice versa. GH#65576
2942 if self.meta == "category":
2943 raise TypeError(
2944 "cannot append a categorical index to a non-categorical index"
2945 )
2946 raise TypeError(
2947 "cannot append a non-categorical index to a categorical index"
2948 )
2950 def update_info(self, info) -> None:
2951 """
2952 set/update the info for this indexable with the key/value
2953 if there is a conflict raise/warn as needed
2954 """
2955 for key in self._info_fields:
2956 value = getattr(self, key, None)
2957 idx = info.setdefault(self.name, {})
2959 existing_value = idx.get(key)
2960 if key in idx and value is not None and existing_value != value:
2961 # frequency/name just warn
2962 if key in ["freq", "index_name"]:
2963 ws = attribute_conflict_doc % (key, existing_value, value)
2964 warnings.warn(
2965 ws, AttributeConflictWarning, stacklevel=find_stack_level()
2966 )
2968 # reset
2969 idx[key] = None
2970 setattr(self, key, None)
2972 else:
2973 raise ValueError(
2974 f"invalid info for [{self.name}] for [{key}], "
2975 f"existing_value [{existing_value}] conflicts with "
2976 f"new value [{value}]"
2977 )
2978 elif value is not None or existing_value is not None:
2979 idx[key] = value
2981 def set_info(self, info) -> None:
2982 """set my state from the passed info"""
2983 idx = info.get(self.name)
2984 if idx is not None:
2985 self.__dict__.update(idx)
2987 def set_attr(self) -> None:
2988 """set the kind for this column"""
2989 _set_attr_if_changed(self.attrs, self.kind_attr, self.kind)
2990 if self.meta is not None:
2991 # Persist meta="category" so an all-NaN CategoricalIndex, whose
2992 # empty categories cannot be written as metadata, still round-trips
2993 # as categorical rather than as raw integer codes. GH#65576
2994 _set_attr_if_changed(self.attrs, self.meta_attr, self.meta)
2995 if self.kind == "string":
2996 # GH#9604: the NaN sentinel, empty when the index has no missing
2997 # value, so that its absence marks an older file. Table.indexables
2998 # reads it back through _read_index_nan_rep.
2999 _set_attr_if_changed(self.attrs, self.nan_rep_attr, self.nan_rep or "")
3001 def validate_metadata(self, handler: AppendableTable) -> None:
3002 """validate that kind=category does not change the categories"""
3003 if self.meta == "category":
3004 new_metadata = self.metadata
3005 cur_metadata = handler.read_metadata(self.cname)
3006 if (
3007 new_metadata is not None
3008 and cur_metadata is not None
3009 and not array_equivalent(
3010 new_metadata, cur_metadata, strict_nan=True, dtype_equal=True
3011 )
3012 ):
3013 raise ValueError(
3014 "cannot append a categorical with different categories "
3015 "to the existing; align with `cat.set_categories(...)` "
3016 "or pre-merge with `pd.api.types.union_categoricals` "
3017 "before writing"
3018 )
3020 def write_metadata(self, handler: AppendableTable) -> None:
3021 """set the meta data"""
3022 if self.metadata is not None:
3023 handler.write_metadata(self.cname, self.metadata)
3026class GenericIndexCol(IndexCol):
3027 """an index which is not represented in the data of the table"""
3029 @property
3030 def is_indexed(self) -> bool:
3031 return False
3033 def convert(
3034 self, values: np.ndarray, nan_rep, encoding: str, errors: str
3035 ) -> tuple[Index, Index]:
3036 """
3037 Convert the data from this selection to the appropriate pandas type.
3039 Parameters
3040 ----------
3041 values : np.ndarray
3042 nan_rep : str
3043 encoding : str
3044 errors : str
3045 """
3046 assert isinstance(values, np.ndarray), type(values)
3048 index = RangeIndex(len(values))
3049 return index, index
3051 def set_attr(self) -> None:
3052 pass
3055class DataCol(IndexCol):
3056 """
3057 a data holding column, by definition this is not indexable
3059 Parameters
3060 ----------
3061 data : the actual data
3062 cname : the column name in the table to hold the data (typically
3063 values)
3064 meta : a string description of the metadata
3065 metadata : the actual metadata
3066 """
3068 is_an_indexable = False
3069 is_data_indexable = False
3070 _info_fields = ["tz", "ordered"]
3072 def __init__(
3073 self,
3074 name: str,
3075 values=None,
3076 kind=None,
3077 typ=None,
3078 cname: str | None = None,
3079 pos=None,
3080 tz=None,
3081 ordered=None,
3082 table=None,
3083 meta=None,
3084 metadata=None,
3085 dtype: DtypeArg | None = None,
3086 data=None,
3087 ) -> None:
3088 super().__init__(
3089 name=name,
3090 values=values,
3091 kind=kind,
3092 typ=typ,
3093 pos=pos,
3094 cname=cname,
3095 tz=tz,
3096 ordered=ordered,
3097 table=table,
3098 meta=meta,
3099 metadata=metadata,
3100 )
3101 self.dtype = dtype
3102 self.data = data
3104 @property
3105 def dtype_attr(self) -> str:
3106 return f"{self.name}_dtype"
3108 def __repr__(self) -> str:
3109 temp = tuple(
3110 map(
3111 pprint_thing, (self.name, self.cname, self.dtype, self.kind, self.shape)
3112 )
3113 )
3114 return ",".join(
3115 [
3116 f"{key}->{value}"
3117 for key, value in zip(
3118 ["name", "cname", "dtype", "kind", "shape"], temp, strict=True
3119 )
3120 ]
3121 )
3123 def __eq__(self, other: object) -> bool:
3124 """compare 2 col items"""
3125 return all(
3126 getattr(self, a, None) == getattr(other, a, None)
3127 for a in ["name", "cname", "dtype", "pos"]
3128 )
3130 def set_data(self, data: ArrayLike) -> None:
3131 assert data is not None
3132 assert self.dtype is None
3134 data, dtype_name = _get_data_and_dtype_name(data)
3136 self.data = data
3137 self.dtype = dtype_name
3138 self.kind = _dtype_to_kind(dtype_name)
3140 def take_data(self):
3141 """return the data"""
3142 return self.data
3144 @classmethod
3145 def _get_atom(cls, values: ArrayLike) -> Col:
3146 """
3147 Get an appropriately typed and shaped pytables.Col object for values.
3148 """
3149 dtype = values.dtype
3150 # GH#26144, GH#38305, GH#42070. CategoricalDtype/DatetimeTZDtype have
3151 # working code paths below; PeriodDtype is handled in a separate effort.
3152 if isinstance(dtype, ExtensionDtype) and not isinstance(
3153 dtype, (CategoricalDtype, DatetimeTZDtype, PeriodDtype)
3154 ):
3155 raise NotImplementedError(
3156 f"Cannot store a column with dtype {dtype} in an HDF5 file. "
3157 "HDFStore supports NumPy dtypes, datetime64 with timezone, "
3158 "categorical, and string extension types."
3159 )
3160 itemsize = dtype.itemsize
3162 shape = values.shape
3163 if values.ndim == 1:
3164 # EA, use block shape pretending it is 2D
3165 # TODO(EA2D): not necessary with 2D EAs
3166 shape = (1, values.size)
3168 if isinstance(values, Categorical):
3169 codes = values.codes
3170 atom = cls.get_atom_data(shape, kind=codes.dtype.name)
3171 elif lib.is_np_dtype(dtype, "M") or isinstance(dtype, DatetimeTZDtype):
3172 atom = cls.get_atom_datetime64(shape)
3173 elif lib.is_np_dtype(dtype, "m"):
3174 atom = cls.get_atom_timedelta64(shape)
3175 elif is_complex_dtype(dtype):
3176 atom = _tables().ComplexCol(itemsize=itemsize, shape=shape[0])
3177 elif is_string_dtype(dtype):
3178 atom = cls.get_atom_string(shape, itemsize)
3179 else:
3180 atom = cls.get_atom_data(shape, kind=dtype.name)
3182 return atom
3184 @classmethod
3185 def get_atom_string(cls, shape, itemsize):
3186 return _tables().StringCol(itemsize=itemsize, shape=shape[0])
3188 @classmethod
3189 def get_atom_coltype(cls, kind: str) -> type[Col]:
3190 """return the PyTables column class for this column"""
3191 if kind.startswith("uint"):
3192 k4 = kind[4:]
3193 col_name = f"UInt{k4}Col"
3194 elif kind.startswith("period"):
3195 # we store as integer
3196 col_name = "Int64Col"
3197 else:
3198 kcap = kind.capitalize()
3199 col_name = f"{kcap}Col"
3201 return getattr(_tables(), col_name)
3203 @classmethod
3204 def get_atom_data(cls, shape, kind: str) -> Col:
3205 return cls.get_atom_coltype(kind=kind)(shape=shape[0])
3207 @classmethod
3208 def get_atom_datetime64(cls, shape):
3209 return _tables().Int64Col(shape=shape[0])
3211 @classmethod
3212 def get_atom_timedelta64(cls, shape):
3213 return _tables().Int64Col(shape=shape[0])
3215 @property
3216 def shape(self):
3217 return getattr(self.data, "shape", None)
3219 @property
3220 def cvalues(self):
3221 """return my cython values"""
3222 return self.data
3224 def validate_attr(self, append) -> None:
3225 """validate that we have the same order as the existing & same dtype"""
3226 if append:
3227 existing_fields = getattr(self.attrs, self.kind_attr, None)
3228 if existing_fields is not None and existing_fields != list(self.values):
3229 raise ValueError("appended items do not match existing items in table!")
3231 existing_dtype = getattr(self.attrs, self.dtype_attr, None)
3232 if existing_dtype is not None and existing_dtype != self.dtype:
3233 raise ValueError(
3234 "appended items dtype do not match existing items dtype in table!"
3235 )
3237 def convert(self, values: np.ndarray, nan_rep, encoding: str, errors: str):
3238 """
3239 Convert the data from this selection to the appropriate pandas type.
3241 Parameters
3242 ----------
3243 values : np.ndarray
3244 nan_rep :
3245 encoding : str
3246 errors : str
3248 Returns
3249 -------
3250 index : listlike to become an Index
3251 data : ndarraylike to become a column
3252 """
3253 assert isinstance(values, np.ndarray), type(values)
3255 # values is a recarray
3256 if values.dtype.fields is not None:
3257 values = values[self.cname]
3258 if not values.flags["ALIGNED"]:
3259 # GH#54396 copy to realign; unaligned buffers SIGBUS
3260 # on strict-alignment platforms (e.g. 32-bit ARM)
3261 values = values.copy()
3263 assert self.typ is not None
3264 if self.dtype is None:
3265 # Note: in tests we never have timedelta64 or datetime64,
3266 # so the _get_data_and_dtype_name may be unnecessary
3267 converted, dtype_name = _get_data_and_dtype_name(values)
3268 kind = _dtype_to_kind(dtype_name)
3269 else:
3270 converted = values
3271 dtype_name = self.dtype
3272 kind = self.kind
3274 assert isinstance(converted, np.ndarray) # for mypy
3276 # use the meta if needed
3277 meta = self.meta
3278 metadata = self.metadata
3279 ordered = self.ordered
3280 tz = self.tz
3282 assert dtype_name is not None
3283 # convert to the correct dtype
3284 dtype = dtype_name
3286 # reverse converts
3287 if dtype.startswith("datetime64"):
3288 # recreate with tz if indicated
3289 if dtype == "datetime64":
3290 dtype = "datetime64[ns]"
3291 converted = _set_tz(converted, tz, dtype)
3293 elif dtype.startswith("timedelta64"):
3294 if dtype == "timedelta64":
3295 # from before we started storing timedelta64 unit
3296 converted = np.asarray(converted, dtype="m8[ns]")
3297 else:
3298 converted = np.asarray(converted, dtype=dtype)
3299 elif dtype.startswith("period"):
3300 # GH#41978 PeriodArray values are stored as i8 ordinals; the
3301 # PyTables atom has shape (1,) per row, so ravel before wrapping.
3302 pdtype = PeriodDtype.construct_from_string(dtype)
3303 converted = PeriodArray._simple_new(
3304 np.asarray(converted, dtype="i8").ravel(), dtype=pdtype
3305 )
3306 elif dtype == "date":
3307 try:
3308 converted = np.asarray(
3309 [date.fromordinal(v) for v in converted], dtype=object
3310 )
3311 except ValueError:
3312 converted = np.asarray(
3313 [date.fromtimestamp(v) for v in converted], dtype=object
3314 )
3316 elif meta == "category":
3317 # we have a categorical
3318 categories = metadata
3319 codes = converted.ravel()
3321 # if we have stored a NaN in the categories
3322 # then strip it; in theory we could have BOTH
3323 # -1s in the codes and nulls :<
3324 if categories is None:
3325 # Handle case of NaN-only categorical columns in which case
3326 # the categories are an empty array; when this is stored,
3327 # pytables cannot write a zero-len array, so on readback
3328 # the categories would be None and `read_hdf()` would fail.
3329 categories = Index([], dtype=np.float64)
3330 else:
3331 mask = isna(categories)
3332 if mask.any():
3333 # A category can be NaN if the nan_rep string was itself
3334 # a genuine category on write. Drop NaN categories and
3335 # remap the codes. GH#21741
3336 remap = np.full(len(categories), -1, dtype=codes.dtype)
3337 remap[~mask] = np.arange((~mask).sum(), dtype=codes.dtype)
3338 categories = categories[~mask]
3339 codes = np.where(codes < 0, codes, remap[codes])
3341 converted = Categorical.from_codes(
3342 codes, categories=categories, ordered=ordered, validate=False
3343 )
3345 else:
3346 try:
3347 converted = converted.astype(dtype, copy=False)
3348 except TypeError:
3349 converted = converted.astype("O", copy=False)
3351 # convert nans / decode
3352 if kind == "string":
3353 if self.uses_col_nan_rep:
3354 # GH#9604: None when the column had no missing value, in which
3355 # case a literal "nan" is left untouched
3356 col_nan_rep = self.nan_rep
3357 else:
3358 # the table-wide nan_rep, which the writer defaulted to "nan"
3359 col_nan_rep = nan_rep if nan_rep is not None else "nan"
3360 converted = _unconvert_string_array(
3361 converted,
3362 nan_rep=col_nan_rep,
3363 encoding=encoding,
3364 errors=errors,
3365 )
3367 return self.values, converted
3369 def set_attr(self) -> None:
3370 """set the data for this column"""
3371 _set_attr_if_changed(self.attrs, self.kind_attr, self.values)
3372 _set_attr_if_changed(self.attrs, self.meta_attr, self.meta)
3373 assert self.dtype is not None
3374 _set_attr_if_changed(self.attrs, self.dtype_attr, self.dtype)
3375 if self.nan_rep is not None:
3376 # GH#9604: this column's own NaN sentinel (see convert).
3377 _set_attr_if_changed(self.attrs, self.nan_rep_attr, self.nan_rep)
3380class DataIndexableCol(DataCol):
3381 """represent a data column that can be indexed"""
3383 is_data_indexable = True
3385 def validate_names(self) -> None:
3386 if not is_string_dtype(Index(self.values).dtype):
3387 # TODO: should the message here be more specifically non-str?
3388 raise ValueError("cannot have non-object label DataIndexableCol")
3390 @classmethod
3391 def get_atom_string(cls, shape, itemsize):
3392 return _tables().StringCol(itemsize=itemsize)
3394 @classmethod
3395 def get_atom_data(cls, shape, kind: str) -> Col:
3396 return cls.get_atom_coltype(kind=kind)()
3398 @classmethod
3399 def get_atom_datetime64(cls, shape):
3400 return _tables().Int64Col()
3402 @classmethod
3403 def get_atom_timedelta64(cls, shape):
3404 return _tables().Int64Col()
3407class GenericDataIndexableCol(DataIndexableCol):
3408 """represent a generic pytables data column"""
3411class Fixed:
3412 """
3413 represent an object in my store
3414 facilitate read/write of various types of objects
3415 this is an abstract base class
3417 Parameters
3418 ----------
3419 parent : HDFStore
3420 group : Node
3421 The group node where the table resides.
3422 """
3424 pandas_kind: str
3425 format_type: str = "fixed" # GH#30962 needed by dask
3426 obj_type: type[DataFrame | Series]
3427 ndim: int
3428 parent: HDFStore
3429 is_table: bool = False
3431 def __init__(
3432 self,
3433 parent: HDFStore,
3434 group: Node,
3435 encoding: str | None = "UTF-8",
3436 errors: str = "strict",
3437 ) -> None:
3438 assert isinstance(parent, HDFStore), type(parent)
3439 assert _table_mod is not None # needed for mypy
3440 assert isinstance(group, _table_mod.Node), type(group)
3441 self.parent = parent
3442 self.group = group
3443 self.encoding = _ensure_encoding(encoding)
3444 self.errors = errors
3446 @property
3447 def pandas_type(self):
3448 return getattr(self.group._v_attrs, "pandas_type", None)
3450 def __repr__(self) -> str:
3451 """return a pretty representation of myself"""
3452 self.infer_axes()
3453 s = self.shape
3454 if s is not None:
3455 if isinstance(s, (list, tuple)):
3456 jshape = ",".join([pprint_thing(x) for x in s])
3457 s = f"[{jshape}]"
3458 return f"{self.pandas_type:12.12} (shape->{s})"
3459 return self.pandas_type
3461 def set_object_info(self) -> None:
3462 """set my pandas type"""
3463 self.attrs.pandas_type = str(self.pandas_kind)
3465 def copy(self) -> Fixed:
3466 new_self = copy.copy(self)
3467 return new_self
3469 @property
3470 def shape(self):
3471 return self.nrows
3473 @property
3474 def pathname(self):
3475 return self.group._v_pathname
3477 @property
3478 def _handle(self):
3479 return self.parent._handle
3481 @property
3482 def _filters(self):
3483 return self.parent._filters
3485 @property
3486 def _complevel(self) -> int:
3487 return self.parent._complevel
3489 @property
3490 def _fletcher32(self) -> bool:
3491 return self.parent._fletcher32
3493 @property
3494 def attrs(self):
3495 return self.group._v_attrs
3497 def set_attrs(self) -> None:
3498 """set our object attributes"""
3500 def get_attrs(self) -> None:
3501 """get our object attributes"""
3503 @property
3504 def storable(self):
3505 """return my storable"""
3506 return self.group
3508 @property
3509 def is_exists(self) -> bool:
3510 return False
3512 @property
3513 def nrows(self):
3514 return getattr(self.storable, "nrows", None)
3516 def validate(self, other) -> Literal[True] | None:
3517 """validate against an existing storable"""
3518 if other is None:
3519 return None
3520 return True
3522 def infer_axes(self) -> bool:
3523 """
3524 infer the axes of my storer
3525 return a boolean indicating if we have a valid storer or not
3526 """
3527 s = self.storable
3528 if s is None:
3529 return False
3530 self.get_attrs()
3531 return True
3533 def read(
3534 self,
3535 where=None,
3536 columns=None,
3537 start: int | None = None,
3538 stop: int | None = None,
3539 ) -> Series | DataFrame:
3540 raise NotImplementedError(
3541 "cannot read on an abstract storer: subclasses should implement"
3542 )
3544 def write(self, obj, **kwargs) -> None:
3545 raise NotImplementedError(
3546 "cannot write on an abstract storer: subclasses should implement"
3547 )
3549 def delete(
3550 self, where=None, start: int | None = None, stop: int | None = None
3551 ) -> int | None:
3552 """
3553 support fully deleting the node in its entirety (only) - where
3554 specification must be None
3555 """
3556 if com.all_none(where, start, stop):
3557 self._handle.remove_node(self.group, recursive=True)
3558 return None
3560 raise TypeError("cannot delete on an abstract storer")
3563class GenericFixed(Fixed):
3564 """a generified fixed version"""
3566 _index_type_map = {DatetimeIndex: "datetime", PeriodIndex: "period"}
3567 _reverse_index_map = {v: k for k, v in _index_type_map.items()}
3568 attributes: list[str] = []
3570 # indexer helpers
3571 def _class_to_alias(self, cls) -> str:
3572 return self._index_type_map.get(cls, "")
3574 def _alias_to_class(self, alias):
3575 if isinstance(alias, type): # pragma: no cover
3576 # compat: for a short period of time master stored types
3577 return alias
3578 return self._reverse_index_map.get(alias, Index)
3580 def _get_index_factory(self, attrs):
3581 index_class = self._alias_to_class(getattr(attrs, "index_class", ""))
3583 factory: Callable
3585 kwargs = {}
3586 if index_class == DatetimeIndex:
3588 def f(values, freq=None, tz=None): # pyright: ignore[reportRedeclaration]
3589 # data are already in UTC, localize and convert if tz present
3590 dta = DatetimeArray._simple_new(values.values, dtype=values.dtype)
3591 result = DatetimeIndex._simple_new(dta, name=None)
3592 result._freq = freq
3593 if tz is not None:
3594 result = result.tz_localize("UTC").tz_convert(tz)
3595 return result
3597 factory = f
3598 elif index_class == PeriodIndex:
3600 def f(values, freq=None, tz=None):
3601 dtype = PeriodDtype(freq)
3602 parr = PeriodArray._simple_new(values, dtype=dtype)
3603 return PeriodIndex._simple_new(parr, name=None)
3605 factory = f
3606 else:
3607 factory = index_class
3608 kwargs["copy"] = False
3610 if "freq" in attrs and attrs["freq"] is not None:
3611 # GH#33186 old versions of pandas wrote a freq=None attr on every
3612 # index, including non-datetimelike ones; only a non-None freq is
3613 # meaningful (and only a TimedeltaIndex reaches here as plain Index).
3614 freq_attr = attrs["freq"]
3615 if isinstance(freq_attr, bytes):
3616 # GH#35917: HDF5 files written by old (Python 2) pandas
3617 # stored freq as a Python 2 pickle byte-string, which
3618 # pytables can't unpickle in Python 3. The original
3619 # freq is unrecoverable, so drop it. The index data
3620 # itself is unaffected; users can manually reassign
3621 # via `df.index.freq = df.index.inferred_freq`.
3622 warnings.warn(
3623 "Could not decode freq attribute on stored index; "
3624 "the file was likely written by an older pandas "
3625 "version. Setting freq=None.",
3626 stacklevel=find_stack_level(),
3627 )
3628 freq_attr = None
3629 kwargs["freq"] = freq_attr
3630 if index_class is Index:
3631 # DTI/PI would be gotten by _alias_to_class
3632 factory = TimedeltaIndex
3634 if "tz" in attrs:
3635 kwargs["tz"] = attrs["tz"]
3636 assert index_class is DatetimeIndex # just checking
3638 return factory, kwargs
3640 def validate_read(self, columns, where) -> None:
3641 """
3642 raise if any keywords are passed which are not-None
3643 """
3644 if columns is not None:
3645 raise TypeError(
3646 "cannot pass a column specification when reading "
3647 "a Fixed format store. this store must be selected in its entirety"
3648 )
3649 if where is not None:
3650 raise TypeError(
3651 "cannot pass a where specification when reading "
3652 "from a Fixed format store. this store must be selected in its entirety"
3653 )
3655 @property
3656 def is_exists(self) -> bool:
3657 return True
3659 def set_attrs(self) -> None:
3660 """set our object attributes"""
3661 self.attrs.encoding = self.encoding
3662 self.attrs.errors = self.errors
3664 def get_attrs(self) -> None:
3665 """retrieve our attributes"""
3666 self.encoding = _ensure_encoding(getattr(self.attrs, "encoding", None))
3667 self.errors = getattr(self.attrs, "errors", "strict")
3668 for n in self.attributes:
3669 setattr(self, n, getattr(self.attrs, n, None))
3671 def write(self, obj, **kwargs) -> None:
3672 self.set_attrs()
3674 def read_array(self, key: str, start: int | None = None, stop: int | None = None):
3675 """read an array for the specified node (off of group"""
3676 import tables
3678 node = getattr(self.group, key)
3679 attrs = node._v_attrs
3681 transposed = getattr(attrs, "transposed", False)
3683 if isinstance(node, tables.VLArray):
3684 ret = node[0][start:stop]
3685 dtype = getattr(attrs, "value_type", None)
3686 if dtype is not None:
3687 ret = pd_array(ret, dtype=dtype)
3688 else:
3689 dtype = getattr(attrs, "value_type", None)
3690 shape = getattr(attrs, "shape", None)
3692 if shape is not None:
3693 # length 0 axis
3694 ret = np.empty(shape, dtype=dtype)
3695 else:
3696 ret = node[start:stop]
3698 if dtype and dtype.startswith("datetime64"):
3699 # reconstruct a timezone if indicated
3700 if dtype == "datetime64":
3701 dtype = "datetime64[ns]"
3702 tz = getattr(attrs, "tz", None)
3703 ret = _set_tz(ret, tz, dtype)
3705 elif dtype and dtype.startswith("timedelta64"):
3706 if dtype == "timedelta64":
3707 # This was written back before we started writing
3708 # timedelta64 units
3709 ret = np.asarray(ret, dtype="m8[ns]")
3710 else:
3711 ret = np.asarray(ret, dtype=dtype)
3713 elif dtype and dtype.startswith("period"):
3714 pdtype = PeriodDtype.construct_from_string(dtype)
3715 ret = PeriodArray._simple_new(np.asarray(ret, dtype="i8"), dtype=pdtype)
3717 if transposed:
3718 return ret.T
3719 else:
3720 return ret
3722 def read_index(
3723 self, key: str, start: int | None = None, stop: int | None = None
3724 ) -> Index:
3725 variety = getattr(self.attrs, f"{key}_variety")
3727 if variety == "multi":
3728 return self.read_multi_index(key, start=start, stop=stop)
3729 elif variety == "regular":
3730 node = getattr(self.group, key)
3731 index = self.read_index_node(node, start=start, stop=stop)
3732 return index
3733 else: # pragma: no cover
3734 raise TypeError(f"unrecognized index variety: {variety}")
3736 def write_index(self, key: str, index: Index) -> None:
3737 if isinstance(index, MultiIndex):
3738 setattr(self.attrs, f"{key}_variety", "multi")
3739 self.write_multi_index(key, index)
3740 elif isinstance(index, CategoricalIndex):
3741 # GH#33909: round-trip a CategoricalIndex by storing its integer
3742 # codes at this key and the categories at a sibling node. This
3743 # also avoids _convert_index, which has no fixed-format support
3744 # for the "category" dtype.
3745 setattr(self.attrs, f"{key}_variety", "regular")
3746 cat = index._values
3747 self.write_array(key, np.asarray(cat.codes))
3748 self.write_index(f"{key}_categories", Index(cat.categories))
3749 node = getattr(self.group, key)
3750 node._v_attrs.kind = "category"
3751 node._v_attrs.name = index.name
3752 node._v_attrs.ordered = bool(cat.ordered)
3753 else:
3754 setattr(self.attrs, f"{key}_variety", "regular")
3755 converted = _convert_index("index", index, self.encoding, self.errors)
3757 self.write_array(key, converted.values)
3759 node = getattr(self.group, key)
3760 node._v_attrs.kind = converted.kind
3761 node._v_attrs.name = index.name
3763 if converted.kind == "string":
3764 # GH#9604: the NaN sentinel, as in IndexCol.set_attr
3765 node._v_attrs.nan_rep = converted.nan_rep or ""
3767 if isinstance(index, (DatetimeIndex, PeriodIndex)):
3768 node._v_attrs.index_class = self._class_to_alias(type(index))
3770 if isinstance(index, (DatetimeIndex, PeriodIndex, TimedeltaIndex)):
3771 node._v_attrs.freq = index.freq
3773 if isinstance(index, DatetimeIndex) and index.tz is not None:
3774 node._v_attrs.tz = _get_tz(index.tz)
3776 def write_multi_index(self, key: str, index: MultiIndex) -> None:
3777 setattr(self.attrs, f"{key}_nlevels", index.nlevels)
3779 for i, (lev, level_codes, name) in enumerate(
3780 zip(index.levels, index.codes, index.names, strict=True)
3781 ):
3782 # write the level
3783 if isinstance(lev.dtype, ExtensionDtype) and not isinstance(
3784 lev.dtype, StringDtype
3785 ):
3786 raise NotImplementedError(
3787 "Saving a MultiIndex with an extension dtype is not supported."
3788 )
3789 level_key = f"{key}_level{i}"
3790 conv_level = _convert_index(level_key, lev, self.encoding, self.errors)
3791 self.write_array(level_key, conv_level.values)
3792 node = getattr(self.group, level_key)
3793 node._v_attrs.kind = conv_level.kind
3794 node._v_attrs.name = name
3796 if conv_level.kind == "string":
3797 # GH#9604: the NaN sentinel, as in IndexCol.set_attr
3798 node._v_attrs.nan_rep = conv_level.nan_rep or ""
3800 # write the name
3801 setattr(node._v_attrs, f"{key}_name{name}", name)
3803 # write the labels
3804 label_key = f"{key}_label{i}"
3805 self.write_array(label_key, level_codes)
3807 def read_multi_index(
3808 self, key: str, start: int | None = None, stop: int | None = None
3809 ) -> MultiIndex:
3810 nlevels = getattr(self.attrs, f"{key}_nlevels")
3812 levels = []
3813 codes = []
3814 names: list[Hashable] = []
3815 for i in range(nlevels):
3816 level_key = f"{key}_level{i}"
3817 node = getattr(self.group, level_key)
3818 # A level holds the unique values, not one entry per row, so it must
3819 # be read in full; only the codes are sliced by start/stop.
3820 lev = self.read_index_node(node)
3821 levels.append(lev)
3822 names.append(lev.name)
3824 label_key = f"{key}_label{i}"
3825 level_codes = self.read_array(label_key, start=start, stop=stop)
3826 codes.append(level_codes)
3828 return MultiIndex(
3829 levels=levels, codes=codes, names=names, verify_integrity=True
3830 )
3832 def read_index_node(
3833 self, node: Node, start: int | None = None, stop: int | None = None
3834 ) -> Index:
3835 data = node[start:stop]
3836 # If the index was an empty array write_array_empty() will
3837 # have written a sentinel. Here we replace it with the original.
3838 if "shape" in node._v_attrs and np.prod(node._v_attrs.shape) == 0:
3839 data = np.empty(node._v_attrs.shape, dtype=node._v_attrs.value_type)
3840 kind = node._v_attrs.kind
3841 name = None
3843 if "name" in node._v_attrs:
3844 name = _ensure_str(node._v_attrs.name)
3846 if kind == "category":
3847 # GH#33909: reconstruct CategoricalIndex from sliced codes plus
3848 # categories saved at a sibling node by write_index.
3849 categories = self.read_index(f"{node._v_name}_categories")
3850 ordered = bool(getattr(node._v_attrs, "ordered", False))
3851 cat = Categorical.from_codes(
3852 data, categories=categories, ordered=ordered, validate=False
3853 )
3854 return CategoricalIndex._simple_new(cat, name=name)
3856 attrs = node._v_attrs
3857 factory, kwargs = self._get_index_factory(attrs)
3859 nan_rep = _read_index_nan_rep(attrs) if kind == "string" else None
3861 if kind == "string" and using_string_dtype():
3862 # GH#9604: once the sentinel is substituted back to NaN, dtype
3863 # inference can only recover str if some non-missing string
3864 # survives, so an all-missing index would silently degrade to
3865 # object. Pin the dtype the values were written with instead.
3866 kwargs["dtype"] = "str"
3868 if kind in ("date", "object"):
3869 index = factory(
3870 _unconvert_index(
3871 data, kind, encoding=self.encoding, errors=self.errors
3872 ),
3873 dtype=object,
3874 **kwargs,
3875 )
3876 else:
3877 try:
3878 index = factory(
3879 _unconvert_index(
3880 data,
3881 kind,
3882 encoding=self.encoding,
3883 errors=self.errors,
3884 nan_rep=nan_rep,
3885 ),
3886 **kwargs,
3887 )
3888 except UnicodeEncodeError as err:
3889 if (
3890 self.errors == "surrogatepass"
3891 and using_string_dtype()
3892 and str(err).endswith("surrogates not allowed")
3893 and HAS_PYARROW
3894 ):
3895 kwargs["dtype"] = StringDtype(storage="python", na_value=np.nan)
3896 index = factory(
3897 _unconvert_index(
3898 data,
3899 kind,
3900 encoding=self.encoding,
3901 errors=self.errors,
3902 nan_rep=nan_rep,
3903 ),
3904 **kwargs,
3905 )
3906 else:
3907 raise
3909 index.name = name
3911 return index
3913 def write_array_empty(self, key: str, value: ArrayLike) -> None:
3914 """write a 0-len array"""
3915 # ugly hack for length 0 axes
3916 arr = np.empty((1,) * value.ndim)
3917 self._handle.create_array(self.group, key, arr)
3918 node = getattr(self.group, key)
3919 node._v_attrs.value_type = str(value.dtype)
3920 node._v_attrs.shape = value.shape
3922 def write_array(
3923 self, key: str, obj: AnyArrayLike, items: Index | None = None
3924 ) -> None:
3925 value = extract_array(obj, extract_numpy=True)
3927 if key in self.group:
3928 self._handle.remove_node(self.group, key)
3930 # Transform needed to interface with pytables row/col notation
3931 empty_array = value.size == 0
3932 transposed = False
3934 if isinstance(value.dtype, CategoricalDtype):
3935 raise NotImplementedError(
3936 "Cannot store a category dtype in an HDF5 dataset that uses format="
3937 '"fixed". Use format="table".'
3938 )
3939 if not empty_array:
3940 if hasattr(value, "T"):
3941 # ExtensionArrays (1d) may not have transpose.
3942 value = value.T
3943 transposed = True
3945 if isinstance(value, BaseStringArray):
3946 # GH#64180: BaseStringArray must use the VLArray path.
3947 # Atom.from_dtype does not handle ExtensionDtype.
3948 vlarr = self._handle.create_vlarray(
3949 self.group, key, _tables().ObjectAtom(), filters=self._filters
3950 )
3951 vlarr.append(value.to_numpy())
3952 node = getattr(self.group, key)
3953 node._v_attrs.value_type = str(value.dtype)
3955 else:
3956 atom = None
3957 if self._filters is not None:
3958 with suppress(ValueError):
3959 # get the atom for this datatype
3960 atom = _tables().Atom.from_dtype(value.dtype)
3962 if atom is not None:
3963 # We only get here if self._filters is non-None and
3964 # the Atom.from_dtype call succeeded
3966 # create an empty chunked array and fill it from value
3967 if not empty_array:
3968 ca = self._handle.create_carray(
3969 self.group, key, atom, value.shape, filters=self._filters
3970 )
3971 ca[:] = value
3973 else:
3974 self.write_array_empty(key, value)
3976 elif value.dtype.type == np.object_:
3977 # infer the type, warn if we have a non-string type here
3978 # (for performance)
3979 inferred_type = lib.infer_dtype(value, skipna=False)
3980 if empty_array:
3981 pass
3982 elif inferred_type == "string":
3983 pass
3984 elif config["mode"]["performance_warnings"]:
3985 # GH#28460 a single object block may hold several columns;
3986 # only flag the ones that are not plain strings, since a
3987 # string-only column would not warn on its own.
3988 if value.ndim == 2 and items is not None:
3989 block = cast("np.ndarray", value)
3990 mask = [
3991 not lib.is_string_array(block[:, j], skipna=False)
3992 for j in range(block.shape[1])
3993 ]
3994 items = items[mask]
3995 ws = performance_doc % (inferred_type, key, items)
3996 warnings.warn(ws, PerformanceWarning, stacklevel=find_stack_level())
3998 vlarr = self._handle.create_vlarray(
3999 self.group, key, _tables().ObjectAtom(), filters=self._filters
4000 )
4001 vlarr.append(value)
4003 elif lib.is_np_dtype(value.dtype, "M"):
4004 self._handle.create_array(self.group, key, value.view("i8"))
4005 getattr(self.group, key)._v_attrs.value_type = str(value.dtype)
4006 elif isinstance(value.dtype, DatetimeTZDtype):
4007 # store as UTC
4008 # with a zone
4010 # error: "ExtensionArray" has no attribute "asi8"
4011 self._handle.create_array(
4012 self.group,
4013 key,
4014 value.asi8, # type: ignore[attr-defined]
4015 )
4017 node = getattr(self.group, key)
4018 # error: "ExtensionArray" has no attribute "tz"
4019 node._v_attrs.tz = _get_tz(value.tz) # type: ignore[attr-defined]
4020 node._v_attrs.value_type = f"datetime64[{value.dtype.unit}]"
4021 elif lib.is_np_dtype(value.dtype, "m"):
4022 self._handle.create_array(self.group, key, value.view("i8"))
4023 getattr(self.group, key)._v_attrs.value_type = str(value.dtype)
4024 elif isinstance(value.dtype, PeriodDtype):
4025 # GH#41978 store PeriodArray as i8 ordinals + freq attr
4026 # error: "ExtensionArray" has no attribute "asi8"
4027 self._handle.create_array(
4028 self.group,
4029 key,
4030 value.asi8, # type: ignore[attr-defined]
4031 )
4032 node = getattr(self.group, key)
4033 node._v_attrs.value_type = str(value.dtype)
4034 elif empty_array:
4035 self.write_array_empty(key, value)
4036 else:
4037 # GH#26144, GH#38305, GH#42070. PeriodDtype intentionally falls
4038 # through here; it is handled in a separate effort.
4039 if isinstance(value.dtype, ExtensionDtype) and not isinstance(
4040 value.dtype, PeriodDtype
4041 ):
4042 raise NotImplementedError(
4043 f"Cannot store a column with dtype {value.dtype} in "
4044 'an HDF5 file with format="fixed". HDFStore supports '
4045 "NumPy dtypes, datetime64 with timezone, categorical, "
4046 "and string extension types."
4047 )
4048 self._handle.create_array(self.group, key, value)
4050 getattr(self.group, key)._v_attrs.transposed = transposed
4053class SeriesFixed(GenericFixed):
4054 pandas_kind = "series"
4055 attributes = ["name"]
4057 name: Hashable
4059 @property
4060 def shape(self) -> tuple[int] | None:
4061 try:
4062 node = self.group.values
4063 if "shape" in node._v_attrs:
4064 # GH#37235 an empty array is stored as a length-1 sentinel
4065 # (see write_array_empty); the true shape is in this attr.
4066 return tuple(node._v_attrs.shape)
4067 return (len(node),)
4068 except (TypeError, AttributeError):
4069 return None
4071 def read(
4072 self,
4073 where=None,
4074 columns=None,
4075 start: int | None = None,
4076 stop: int | None = None,
4077 ) -> Series:
4078 self.validate_read(columns, where)
4079 index = self.read_index("index", start=start, stop=stop)
4080 values = self.read_array("values", start=start, stop=stop)
4081 try:
4082 result = Series(values, index=index, name=self.name, copy=False)
4083 except UnicodeEncodeError as err:
4084 if (
4085 self.errors == "surrogatepass"
4086 and using_string_dtype()
4087 and str(err).endswith("surrogates not allowed")
4088 and HAS_PYARROW
4089 ):
4090 result = Series(
4091 values,
4092 index=index,
4093 name=self.name,
4094 copy=False,
4095 dtype=StringDtype(storage="python", na_value=np.nan),
4096 )
4097 else:
4098 raise
4099 return result
4101 def write(self, obj, **kwargs) -> None:
4102 super().write(obj, **kwargs)
4103 self.write_index("index", obj.index)
4104 self.write_array("values", obj)
4105 self.attrs.name = obj.name
4108class BlockManagerFixed(GenericFixed):
4109 attributes = ["ndim", "nblocks"]
4111 nblocks: int
4113 @property
4114 def shape(self) -> list[int] | None:
4115 try:
4116 ndim = self.ndim
4118 # items
4119 items = 0
4120 for i in range(self.nblocks):
4121 node = getattr(self.group, f"block{i}_items")
4122 shape = getattr(node, "shape", None)
4123 if shape is not None:
4124 items += shape[0]
4126 # data shape
4127 node = self.group.block0_values
4128 data_shape: tuple[Any, ...] | None
4129 if "shape" in node._v_attrs:
4130 # GH#37235 an empty block is stored un-transposed as a
4131 # (1,)*ndim sentinel (see write_array_empty), with the true
4132 # shape in this attr. Reverse it to match the transposed
4133 # layout used for non-empty blocks.
4134 data_shape = tuple(reversed(node._v_attrs.shape))
4135 else:
4136 data_shape = getattr(node, "shape", None)
4137 if data_shape is not None:
4138 shape = list(data_shape[0 : (ndim - 1)])
4139 else:
4140 shape = []
4142 shape.append(items)
4144 return shape
4145 except AttributeError:
4146 return None
4148 def read(
4149 self,
4150 where=None,
4151 columns=None,
4152 start: int | None = None,
4153 stop: int | None = None,
4154 ) -> DataFrame:
4155 # start, stop applied to rows, so 0th axis only
4156 self.validate_read(columns, where)
4157 select_axis = self.obj_type()._get_block_manager_axis(0)
4159 axes = []
4160 for i in range(self.ndim):
4161 _start, _stop = (start, stop) if i == select_axis else (None, None)
4162 ax = self.read_index(f"axis{i}", start=_start, stop=_stop)
4163 axes.append(ax)
4165 items = axes[0]
4166 dfs = []
4168 for i in range(self.nblocks):
4169 blk_items = self.read_index(f"block{i}_items")
4170 values = self.read_array(f"block{i}_values", start=_start, stop=_stop)
4172 columns = items[items.get_indexer(blk_items)]
4173 arr = values.T
4174 if isinstance(arr, np.ndarray):
4175 # DataFrame stores the block as arr.T, so pass a Fortran-ordered
4176 # arr to get a C-contiguous block (column-major DataFrame), so
4177 # per-column access is contiguous (GH#22073, GH#60469).
4178 arr = np.asfortranarray(arr)
4179 df = DataFrame(arr, columns=columns, index=axes[1], copy=False)
4180 if (
4181 using_string_dtype()
4182 and isinstance(values, np.ndarray)
4183 and is_string_array(values, skipna=True)
4184 ):
4185 df = df.astype(StringDtype(na_value=np.nan))
4186 dfs.append(df)
4188 if len(dfs) > 0:
4189 out = concat(dfs, axis=1)
4190 return out.reindex(columns=items)
4192 return DataFrame(columns=axes[0], index=axes[1])
4194 def write(self, obj, **kwargs) -> None:
4195 super().write(obj, **kwargs)
4197 data = obj._mgr
4198 if not data.is_consolidated():
4199 data = data.consolidate()
4201 self.attrs.ndim = data.ndim
4202 for i, ax in enumerate(data.axes):
4203 if i == 0 and (not ax.is_unique):
4204 raise ValueError("Columns index has to be unique for fixed format")
4205 self.write_index(f"axis{i}", ax)
4207 # Supporting mixed-type DataFrame objects...nontrivial
4208 self.attrs.nblocks = len(data.blocks)
4209 for i, blk in enumerate(data.blocks):
4210 # I have no idea why, but writing values before items fixed #2299
4211 blk_items = data.items.take(blk.mgr_locs)
4212 self.write_array(f"block{i}_values", blk.values, items=blk_items)
4213 self.write_index(f"block{i}_items", blk_items)
4216class FrameFixed(BlockManagerFixed):
4217 pandas_kind = "frame"
4218 obj_type = DataFrame
4221class Table(Fixed):
4222 """
4223 represent a table:
4224 facilitate read/write of various types of tables
4226 Attrs in Table Node
4227 -------------------
4228 These are attributes that are store in the main table node, they are
4229 necessary to recreate these tables when read back in.
4231 index_axes : a list of tuples of the (original indexing axis and
4232 index column)
4233 non_index_axes: a list of tuples of the (original index axis and
4234 columns on a non-indexing axis)
4235 values_axes : a list of the columns which comprise the data of this
4236 table
4237 data_columns : a list of the columns that we are allowing indexing
4238 (these become single columns in values_axes)
4239 nan_rep : the string to use for nan representations for string
4240 objects
4241 levels : the names of levels
4242 metadata : the names of the metadata columns
4243 """
4245 pandas_kind = "wide_table"
4246 format_type: str = "table" # GH#30962 needed by dask
4247 table_type: str
4248 levels: int | list[Hashable] = 1 # pyright: ignore[reportRedeclaration]
4249 is_table = True
4251 metadata: list
4253 def __init__(
4254 self,
4255 parent: HDFStore,
4256 group: Node,
4257 encoding: str | None = None,
4258 errors: str = "strict",
4259 index_axes: list[IndexCol] | None = None,
4260 non_index_axes: list[tuple[AxisInt, Any]] | None = None,
4261 values_axes: list[DataCol] | None = None,
4262 data_columns: list | None = None,
4263 info: dict | None = None,
4264 nan_rep=None,
4265 ) -> None:
4266 super().__init__(parent, group, encoding=encoding, errors=errors)
4267 self.index_axes = index_axes or []
4268 self.non_index_axes = non_index_axes or []
4269 self.values_axes = values_axes or []
4270 self.data_columns = data_columns or []
4271 self.info = info or {}
4272 self.nan_rep = nan_rep
4274 @property
4275 def table_type_short(self) -> str:
4276 return self.table_type.split("_")[0]
4278 def __repr__(self) -> str:
4279 """return a pretty representation of myself"""
4280 self.infer_axes()
4281 jdc = ",".join(self.data_columns) if len(self.data_columns) else ""
4282 dc = f",dc->[{jdc}]"
4284 jindex_axes = ",".join([a.name for a in self.index_axes])
4285 return (
4286 f"{self.pandas_type:12.12} "
4287 f"(typ->{self.table_type_short},nrows->{self.nrows},"
4288 f"ncols->{self.ncols},indexers->[{jindex_axes}]{dc})"
4289 )
4291 def __getitem__(self, c: str):
4292 """return the axis for c"""
4293 for a in self.axes:
4294 if c == a.name:
4295 return a
4296 return None
4298 def validate(self, other) -> None:
4299 """validate against an existing table"""
4300 if other is None:
4301 return
4303 if other.table_type != self.table_type:
4304 raise TypeError(
4305 "incompatible table_type with existing "
4306 f"[{other.table_type} - {self.table_type}]"
4307 )
4309 for c in ["index_axes", "non_index_axes", "values_axes"]:
4310 sv = getattr(self, c, None)
4311 ov = getattr(other, c, None)
4312 if sv != ov:
4313 # show the error for the specific axes
4314 # Argument 1 to "enumerate" has incompatible type
4315 # "Optional[Any]"; expected "Iterable[Any]" [arg-type]
4316 for i, sax in enumerate(sv): # type: ignore[arg-type]
4317 # Value of type "Optional[Any]" is not indexable [index]
4318 oax = ov[i] # type: ignore[index]
4319 if sax != oax:
4320 if c == "values_axes" and sax.kind != oax.kind:
4321 raise ValueError(
4322 f"Cannot serialize the column [{oax.values[0]}] "
4323 f"because its data contents are not [{sax.kind}] "
4324 f"but [{oax.kind}] object dtype"
4325 )
4326 raise ValueError(
4327 f"invalid combination of [{c}] on appending data "
4328 f"[{sax}] vs current table [{oax}]"
4329 )
4331 # should never get here
4332 raise Exception(
4333 f"invalid combination of [{c}] on appending data [{sv}] vs "
4334 f"current table [{ov}]"
4335 )
4337 @property
4338 def is_multi_index(self) -> bool:
4339 """the levels attribute is 1 or a list in the case of a multi-index"""
4340 return isinstance(self.levels, list)
4342 def validate_multiindex(
4343 self, obj: DataFrame | Series
4344 ) -> tuple[DataFrame, list[Hashable]]:
4345 """
4346 validate that we can store the multi-index; reset and return the
4347 new object
4348 """
4349 levels = com.fill_missing_names(obj.index.names)
4350 if "index" in levels:
4351 # GH#6208 'index' is reserved as the implicit row-index name
4352 # in the table format and collides with a level named 'index'.
4353 raise ValueError(
4354 "cannot store a MultiIndex with a level named 'index' as a "
4355 "table; 'index' is reserved for the implicit row index"
4356 )
4357 try:
4358 reset_obj = obj.reset_index()
4359 except ValueError as err:
4360 raise ValueError(
4361 "duplicate names/columns in the multi-index when storing as a table"
4362 ) from err
4363 assert isinstance(reset_obj, DataFrame) # for mypy
4364 return reset_obj, levels
4366 @property
4367 def nrows_expected(self) -> int:
4368 """based on our axes, compute the expected nrows"""
4369 return np.prod([i.cvalues.shape[0] for i in self.index_axes])
4371 @property
4372 def is_exists(self) -> bool:
4373 """has this table been created"""
4374 return "table" in self.group
4376 @property
4377 def storable(self):
4378 return getattr(self.group, "table", None)
4380 @property
4381 def table(self):
4382 """return the table group (this is my storable)"""
4383 return self.storable
4385 @property
4386 def dtype(self):
4387 return self.table.dtype
4389 @property
4390 def description(self):
4391 return self.table.description
4393 @property
4394 def axes(self) -> itertools.chain[IndexCol]:
4395 return itertools.chain(self.index_axes, self.values_axes)
4397 @property
4398 def ncols(self) -> int:
4399 """the number of total columns in the values axes"""
4400 return sum(len(a.values) for a in self.values_axes)
4402 @property
4403 def is_transposed(self) -> bool:
4404 return False
4406 @property
4407 def data_orientation(self) -> tuple[int, ...]:
4408 """return a tuple of my permutated axes, non_indexable at the front"""
4409 return tuple(
4410 itertools.chain(
4411 [int(a[0]) for a in self.non_index_axes],
4412 [int(a.axis) for a in self.index_axes],
4413 )
4414 )
4416 def queryables(self) -> dict[str, Any]:
4417 """return a dict of the kinds allowable columns for this object"""
4418 # mypy doesn't recognize DataFrame._AXIS_NAMES, so we re-write it here
4419 axis_names = {0: "index", 1: "columns"}
4421 # compute the values_axes queryables
4422 d1 = [(a.cname, a) for a in self.index_axes]
4423 d2 = [(axis_names[axis], None) for axis, values in self.non_index_axes]
4424 d3 = [
4425 (v.cname, v) for v in self.values_axes if v.name in set(self.data_columns)
4426 ]
4428 return dict(d1 + d2 + d3)
4430 def index_cols(self) -> list[tuple[Any, Any]]:
4431 """return a list of my index cols"""
4432 # Note: each `i.cname` below is assured to be a str.
4433 return [(i.axis, i.cname) for i in self.index_axes]
4435 def values_cols(self) -> list[str]:
4436 """return a list of my values cols"""
4437 return [i.cname for i in self.values_axes]
4439 def _get_metadata_path(self, key: str) -> str:
4440 """return the metadata pathname for this key"""
4441 group = self.group._v_pathname
4442 return f"{group}/meta/{key}/meta"
4444 def write_metadata(self, key: str, values: np.ndarray) -> None:
4445 """
4446 Write out a metadata array to the key as a fixed-format Series.
4448 Parameters
4449 ----------
4450 key : str
4451 values : ndarray
4452 """
4453 if len(values) == 0:
4454 # PyTables cannot store a zero-len array; the read path already
4455 # treats a missing metadata node as an empty categories Index,
4456 # so skipping the write is the contract we've relied on.
4457 return
4458 self.parent.put(
4459 self._get_metadata_path(key),
4460 Series(values, copy=False),
4461 format="table",
4462 encoding=self.encoding,
4463 errors=self.errors,
4464 nan_rep=self.nan_rep,
4465 track_times=False,
4466 )
4468 def read_metadata(self, key: str):
4469 """return the meta data array for this key"""
4470 if getattr(getattr(self.group, "meta", None), key, None) is not None:
4471 return self.parent.select(self._get_metadata_path(key))
4472 return None
4474 def set_attrs(self) -> None:
4475 """set our table type & indexables"""
4476 self.attrs.table_type = str(self.table_type)
4477 self.attrs.index_cols = self.index_cols()
4478 self.attrs.values_cols = self.values_cols()
4479 self.attrs.non_index_axes = self.non_index_axes
4480 self.attrs.data_columns = self.data_columns
4481 self.attrs.nan_rep = self.nan_rep
4482 self.attrs.encoding = self.encoding
4483 self.attrs.errors = self.errors
4484 self.attrs.levels = self.levels
4485 # GH#9604: marks that string MultiIndex levels are stored with a
4486 # per-level NaN sentinel (older files lack it and read back as before).
4487 # self.levels is the int default 1 for non-MI tables, a list for MI.
4488 self.attrs.mi_level_nan_rep = isinstance(self.levels, list)
4489 # GH#9604: every string data column is stored with its own NaN
4490 # sentinel, superseding mi_level_nan_rep. A caller-supplied nan_rep
4491 # keeps its documented table-wide meaning, so such a table opts out.
4492 self.attrs.data_col_nan_rep = self.nan_rep is None
4493 self.attrs.info = self.info
4495 def get_attrs(self) -> None:
4496 """retrieve our attributes"""
4497 self.non_index_axes = getattr(self.attrs, "non_index_axes", None) or []
4498 self.data_columns = getattr(self.attrs, "data_columns", None) or []
4499 self.info = getattr(self.attrs, "info", None) or {}
4500 self.nan_rep = getattr(self.attrs, "nan_rep", None)
4501 self.encoding = _ensure_encoding(getattr(self.attrs, "encoding", None))
4502 self.errors = getattr(self.attrs, "errors", "strict")
4503 self.levels: list[Hashable] = getattr(self.attrs, "levels", None) or [] # pyright: ignore[reportRedeclaration]
4504 self.index_axes = [a for a in self.indexables if a.is_an_indexable]
4505 self.values_axes = [a for a in self.indexables if not a.is_an_indexable]
4507 def validate_min_itemsize(self, min_itemsize) -> None:
4508 """
4509 validate the min_itemsize doesn't contain items that are not in the
4510 axes this needs data_columns to be defined
4511 """
4512 if min_itemsize is None:
4513 return
4514 if not isinstance(min_itemsize, dict):
4515 return
4517 q = self.queryables()
4518 for k in min_itemsize:
4519 # ok, apply generally
4520 if k == "values":
4521 continue
4522 if k not in q:
4523 raise ValueError(
4524 f"min_itemsize has the key [{k}] which is not an axis or "
4525 "data_column"
4526 )
4528 @cache_readonly
4529 def indexables(self):
4530 """create/cache the indexables if they don't exist"""
4531 _indexables = []
4533 desc = self.description
4534 table_attrs = self.table.attrs
4535 # GH#9604: levels / the MultiIndex-level marker live on the group attrs
4536 # (self.attrs), not the table-node attrs used for per-column metadata.
4537 levels_attr = getattr(self.attrs, "levels", None)
4538 levels = levels_attr if isinstance(levels_attr, list) else []
4539 mi_level_marker = getattr(self.attrs, "mi_level_nan_rep", False)
4540 data_col_marker = getattr(self.attrs, "data_col_nan_rep", False)
4542 # Note: each of the `name` kwargs below are str, ensured
4543 # by the definition in index_cols.
4544 # index columns
4545 for i, (axis, name) in enumerate(self.attrs.index_cols):
4546 atom = getattr(desc, name)
4547 md = self.read_metadata(name)
4548 # Prefer the explicit meta attribute (written since GH#65576); it
4549 # survives even when the categories are empty and no metadata node
4550 # exists. Fall back to inferring from the metadata node for files
4551 # written before the attribute was persisted.
4552 meta = getattr(table_attrs, f"{name}_meta", None)
4553 if meta is None and md is not None:
4554 meta = "category"
4556 kind_attr = f"{name}_kind"
4557 kind = getattr(table_attrs, kind_attr, None)
4558 nan_rep = (
4559 _read_index_nan_rep(table_attrs, f"{name}_nan_rep")
4560 if kind == "string"
4561 else None
4562 )
4564 index_col = IndexCol(
4565 name=name,
4566 axis=axis,
4567 pos=i,
4568 kind=kind,
4569 typ=atom,
4570 table=self.table,
4571 meta=meta,
4572 metadata=md,
4573 nan_rep=nan_rep,
4574 ordered=self.info.get(name, {}).get("ordered"),
4575 )
4576 _indexables.append(index_col)
4578 # values columns
4579 dc = set(self.data_columns)
4580 base_pos = len(_indexables)
4582 def f(i, c: str) -> DataCol:
4583 assert isinstance(c, str)
4584 klass = DataCol
4585 if c in dc:
4586 klass = DataIndexableCol
4588 atom = getattr(desc, c)
4590 # TODO: why kind_attr here?
4591 values = getattr(table_attrs, f"{c}_kind", None)
4592 dtype = getattr(table_attrs, f"{c}_dtype", None)
4593 # Argument 1 to "_dtype_to_kind" has incompatible type
4594 # "Optional[Any]"; expected "str" [arg-type]
4595 kind = _dtype_to_kind(dtype) # type: ignore[arg-type]
4597 md = self.read_metadata(c)
4598 # TODO: figure out why these two versions of `meta` dont always match.
4599 # meta = "category" if md is not None else None
4600 meta = getattr(table_attrs, f"{c}_meta", None)
4602 # GH#9604: mi_level_nan_rep predates data_col_nan_rep and covers
4603 # only the MultiIndex levels, which keep a sentinel even when the
4604 # caller supplied a nan_rep for the data columns.
4605 uses_col_nan_rep = data_col_marker or (c in levels and mi_level_marker)
4606 nan_rep = (
4607 getattr(table_attrs, f"{c}_nan_rep", None) if uses_col_nan_rep else None
4608 )
4610 obj = klass(
4611 name=c,
4612 cname=c,
4613 values=values,
4614 kind=kind,
4615 pos=base_pos + i,
4616 typ=atom,
4617 table=self.table,
4618 meta=meta,
4619 metadata=md,
4620 dtype=dtype,
4621 ordered=self.info.get(c, {}).get("ordered"),
4622 )
4623 obj.uses_col_nan_rep = uses_col_nan_rep
4624 obj.nan_rep = nan_rep
4625 return obj
4627 # Note: the definition of `values_cols` ensures that each
4628 # `c` below is a str.
4629 _indexables.extend([f(i, c) for i, c in enumerate(self.attrs.values_cols)])
4631 return _indexables
4633 def create_index(
4634 self, columns=None, optlevel=None, kind: str | None = None
4635 ) -> None:
4636 """
4637 Create a pytables index on the specified columns.
4639 Parameters
4640 ----------
4641 columns : None, bool, or listlike[str]
4642 Indicate which columns to create an index on.
4644 * False : Do not create any indexes.
4645 * True : Create indexes on all columns.
4646 * None : Create indexes on all columns.
4647 * listlike : Create indexes on the given columns.
4649 optlevel : int or None, default None
4650 Optimization level, if None, pytables defaults to 6.
4651 kind : str or None, default None
4652 Kind of index, if None, pytables defaults to "medium".
4654 Raises
4655 ------
4656 TypeError if trying to create an index on a complex-type column.
4658 Notes
4659 -----
4660 Cannot index Time64Col or ComplexCol.
4661 Pytables must be >= 3.0.
4662 """
4663 if not self.infer_axes():
4664 return
4665 if columns is False:
4666 return
4668 # index all indexables and data_columns
4669 if columns is None or columns is True:
4670 columns = [a.cname for a in self.axes if a.is_data_indexable]
4671 if not isinstance(columns, (tuple, list)):
4672 columns = [columns]
4674 kw = {}
4675 if optlevel is not None:
4676 kw["optlevel"] = optlevel
4677 if kind is not None:
4678 kw["kind"] = kind
4680 table = self.table
4681 for c in columns:
4682 v = getattr(table.cols, c, None)
4683 if v is not None:
4684 # remove the index if the kind/optlevel have changed
4685 if v.is_indexed:
4686 index = v.index
4687 cur_optlevel = index.optlevel
4688 cur_kind = index.kind
4690 if kind is not None and cur_kind != kind:
4691 v.remove_index()
4692 else:
4693 kw["kind"] = cur_kind
4695 if optlevel is not None and cur_optlevel != optlevel:
4696 v.remove_index()
4697 else:
4698 kw["optlevel"] = cur_optlevel
4700 # create the index
4701 if not v.is_indexed:
4702 if v.type.startswith("complex"):
4703 raise TypeError(
4704 "Columns containing complex values can be stored but "
4705 "cannot be indexed when using table format. Either use "
4706 "fixed format, set index=False, or do not include "
4707 "the columns containing complex values to "
4708 "data_columns when initializing the table."
4709 )
4710 v.create_index(**kw)
4711 elif c in self.non_index_axes[0][1]:
4712 # GH 28156
4713 raise AttributeError(
4714 f"column {c} is not a data_column.\n"
4715 f"In order to read column {c} you must reload the dataframe \n"
4716 f"into HDFStore and include {c} with the data_columns argument."
4717 )
4719 def _read_axes(
4720 self, where, start: int | None = None, stop: int | None = None
4721 ) -> list[tuple[np.ndarray, np.ndarray] | tuple[Index, Index]]:
4722 """
4723 Create the axes sniffed from the table.
4725 Parameters
4726 ----------
4727 where : ???
4728 start : int or None, default None
4729 stop : int or None, default None
4731 Returns
4732 -------
4733 List[Tuple[index_values, column_values]]
4734 """
4735 # create the selection
4736 selection = Selection(self, where=where, start=start, stop=stop)
4737 values = selection.select()
4739 results = []
4740 # convert the data
4741 for a in self.axes:
4742 a.set_info(self.info)
4743 res = a.convert(
4744 values,
4745 nan_rep=self.nan_rep,
4746 encoding=self.encoding,
4747 errors=self.errors,
4748 )
4749 results.append(res)
4751 return results
4753 @classmethod
4754 def get_object(cls, obj, transposed: bool):
4755 """return the data for this obj"""
4756 return obj
4758 def validate_data_columns(
4759 self, data_columns, min_itemsize, non_index_axes, index_cnames=()
4760 ) -> list:
4761 """
4762 take the input data_columns and min_itemize and create a data
4763 columns spec
4764 """
4765 if not len(non_index_axes):
4766 return []
4768 axis, axis_labels = non_index_axes[0]
4769 info = self.info.get(axis, {})
4770 if info.get("type") == "MultiIndex":
4771 if data_columns:
4772 raise ValueError(
4773 f"cannot use a multi-index on axis [{axis}] with "
4774 f"data_columns {data_columns}"
4775 )
4776 if isinstance(min_itemsize, dict):
4777 # GH#12154 'values' sizes every string column and the index
4778 # cname(s) size the row index; both are legal here. Only
4779 # per-column keys are unsupported for MultiIndex columns.
4780 allowed = {"values", *index_cnames}
4781 mi_keys = [k for k in min_itemsize if k not in allowed]
4782 if mi_keys:
4783 raise ValueError(
4784 f"cannot use min_itemsize keys {mi_keys} on axis "
4785 f"[{axis}] with a MultiIndex; per-column "
4786 "min_itemsize requires data_columns, which are not "
4787 "supported with MultiIndex columns. Use "
4788 "min_itemsize={'values': N} to apply a single "
4789 "min_itemsize across all string columns."
4790 )
4792 # evaluate the passed data_columns, True == use all columns
4793 # take only valid axis labels
4794 if data_columns is True:
4795 data_columns = list(axis_labels)
4796 elif data_columns is None:
4797 data_columns = []
4799 # if min_itemsize is a dict, add the keys (exclude 'values')
4800 if isinstance(min_itemsize, dict):
4801 existing_data_columns = set(data_columns)
4802 data_columns = list(data_columns) # ensure we do not modify
4803 data_columns.extend(
4804 [
4805 k
4806 for k in min_itemsize.keys()
4807 if k != "values" and k not in existing_data_columns
4808 ]
4809 )
4811 # return valid columns in the order of our axis
4812 return [c for c in data_columns if c in axis_labels]
4814 def _create_axes(
4815 self,
4816 axes,
4817 obj: DataFrame,
4818 validate: bool = True,
4819 nan_rep=None,
4820 data_columns=None,
4821 min_itemsize=None,
4822 ):
4823 """
4824 Create and return the axes.
4826 Parameters
4827 ----------
4828 axes: list or None
4829 The names or numbers of the axes to create.
4830 obj : DataFrame
4831 The object to create axes on.
4832 validate: bool, default True
4833 Whether to validate the obj against an existing object already written.
4834 nan_rep :
4835 A value to use for string column nan_rep.
4836 data_columns : List[str], True, or None, default None
4837 Specify the columns that we want to create to allow indexing on.
4839 * True : Use all available columns.
4840 * None : Use no columns.
4841 * List[str] : Use the specified columns.
4843 min_itemsize: Dict[str, int] or None, default None
4844 The min itemsize for a column in bytes.
4845 """
4846 if not isinstance(obj, DataFrame):
4847 group = self.group._v_name
4848 raise TypeError(
4849 f"cannot properly create the storer for: [group->{group},"
4850 f"value->{type(obj)}]"
4851 )
4853 # set the default axes if needed
4854 if axes is None:
4855 axes = [0]
4857 # map axes to numbers
4858 axes = [obj._get_axis_number(a) for a in axes]
4860 # GH#9604: None means no nan_rep was requested, so each string column
4861 # mints a collision-free sentinel instead. A caller-supplied one keeps
4862 # its documented table-wide meaning and is persisted, so a later append
4863 # to the same table keeps it.
4864 user_nan_rep = nan_rep
4866 # do we have an existing table (if so, use its axes & data_columns)
4867 if self.infer_axes():
4868 table_exists = True
4869 axes = [a.axis for a in self.index_axes]
4870 data_columns = list(self.data_columns)
4871 # an existing table's nan_rep supersedes whatever this append passed
4872 nan_rep = user_nan_rep = self.nan_rep
4873 # TODO: do we always have validate=True here?
4874 else:
4875 table_exists = False
4877 new_info = self.info
4879 assert self.ndim == 2 # with next check, we must have len(axes) == 1
4880 # currently support on ndim-1 axes
4881 if len(axes) != self.ndim - 1:
4882 raise ValueError(
4883 "currently only support ndim-1 indexers in an AppendableTable"
4884 )
4886 # create according to the new data
4887 new_non_index_axes: list = []
4889 # nan_representation
4890 if nan_rep is None:
4891 nan_rep = "nan"
4893 # We construct the non-index-axis first, since that alters new_info
4894 idx = next(x for x in [0, 1] if x not in axes)
4896 a = obj.axes[idx]
4897 # we might be able to change the axes on the appending data if necessary
4898 append_axis = list(a)
4899 if table_exists:
4900 indexer = len(new_non_index_axes) # i.e. 0
4901 exist_axis = self.non_index_axes[indexer][1]
4902 if not array_equivalent(
4903 np.array(append_axis),
4904 np.array(exist_axis),
4905 strict_nan=True,
4906 dtype_equal=True,
4907 ):
4908 # ahah! -> reindex
4909 if array_equivalent(
4910 np.array(sorted(append_axis)),
4911 np.array(sorted(exist_axis)),
4912 strict_nan=True,
4913 dtype_equal=True,
4914 ):
4915 append_axis = exist_axis
4917 # the non_index_axes info
4918 info = new_info.setdefault(idx, {})
4919 info["names"] = list(a.names)
4920 info["type"] = type(a).__name__
4922 new_non_index_axes.append((idx, append_axis))
4924 # Now we can construct our new index axis
4925 idx = axes[0]
4926 a = obj.axes[idx]
4927 axis_name = obj._get_axis_name(idx)
4928 # GH#9604: reuse the string-Index NaN sentinel persisted by an earlier
4929 # append so it is stable across chunks; if none has been persisted yet
4930 # but this chunk introduces a missing value, choose the sentinel against
4931 # the values already stored so it cannot collide with them.
4932 existing_nan_rep = None
4933 existing_values = None
4934 if table_exists:
4935 existing_index_col = self.index_axes[0]
4936 existing_nan_rep = existing_index_col.nan_rep
4937 if existing_index_col.kind == "string" and not hasattr(
4938 self.table.attrs, existing_index_col.nan_rep_attr
4939 ):
4940 # _read_index_nan_rep assumes "nan" is the sentinel of a file
4941 # written before one was persisted, which only holds if such a
4942 # value is stored. Without one the column has no sentinel to
4943 # protect, so a literal "nan" may still be appended to it. Test
4944 # the raw bytes rather than decoding a possibly large column.
4945 stored = getattr(self.table.cols, existing_index_col.cname)[:]
4946 if not (stored == existing_nan_rep.encode(self.encoding)).any():
4947 existing_nan_rep = None
4948 if (
4949 existing_nan_rep is None
4950 and existing_index_col.kind == "string"
4951 and np.asarray(a.isna()).any()
4952 ):
4953 existing_values = _read_stored_column_values(
4954 self.table, existing_index_col.cname, self.encoding, self.errors
4955 )
4956 new_index = _convert_index(
4957 axis_name,
4958 a,
4959 self.encoding,
4960 self.errors,
4961 existing_nan_rep=existing_nan_rep,
4962 existing_values=existing_values,
4963 )
4964 new_index.axis = idx
4966 # Because we are always 2D, there is only one new_index, so
4967 # we know it will have pos=0
4968 new_index.set_pos(0)
4970 if table_exists and new_index.kind == existing_index_col.kind == "integer":
4971 # A PeriodIndex is stored as its i8 ordinals with kind "integer", so
4972 # freq is the only thing separating it from a plain integer index.
4973 # Check it before update_info silently overwrites the stored freq,
4974 # which would reinterpret the rows already stored. GH#68523
4975 existing_freq = new_info.get(new_index.name, {}).get("freq")
4976 if existing_freq != new_index.freq:
4977 if existing_freq is None:
4978 raise TypeError(
4979 "cannot append a period index to a non-period index"
4980 )
4981 if new_index.freq is None:
4982 raise TypeError(
4983 "cannot append a non-period index to a period index"
4984 )
4985 # PeriodDtype's alias ("M"), not the offset's datetime alias
4986 # ("ME"), which Period itself rejects. Same alias and warning
4987 # filter as raise_on_incompatible.
4988 with warnings.catch_warnings():
4989 warnings.filterwarnings(
4990 "ignore",
4991 r"PeriodDtype\[B\] is deprecated",
4992 category=FutureWarning,
4993 )
4994 existing_str = PeriodDtype(existing_freq)._freqstr
4995 new_str = PeriodDtype(new_index.freq)._freqstr
4996 raise TypeError(
4997 f"incompatible freq in col [{existing_str} - {new_str}]"
4998 )
5000 # tz can only differ within a matching kind; a differing one already
5001 # gets validate_attr's more specific message.
5002 if table_exists and new_index.kind == existing_index_col.kind:
5003 _check_tz_conflict(
5004 new_index.name,
5005 new_info.get(new_index.name, {}).get("tz"),
5006 new_index.tz,
5007 )
5009 new_index.update_info(new_info)
5010 new_index.maybe_set_size(min_itemsize) # check for column conflicts
5012 new_index_axes = [new_index]
5013 j = len(new_index_axes) # i.e. 1
5014 assert j == 1
5016 # reindex by our non_index_axes & compute data_columns
5017 assert len(new_non_index_axes) == 1
5018 for a in new_non_index_axes:
5019 obj = _reindex_axis(obj, a[0], a[1])
5021 transposed = new_index.axis == 1
5023 # figure out data_columns and get out blocks
5024 data_columns = self.validate_data_columns(
5025 data_columns, min_itemsize, new_non_index_axes, (new_index.cname,)
5026 )
5028 if new_index.cname in data_columns:
5029 # GH#41437 the implicit row index is stored under the reserved
5030 # cname (typically 'index') and a data column of the same name
5031 # would collide with it in the table description.
5032 raise ValueError(
5033 f"cannot use a column named {new_index.cname!r} as a "
5034 f"data_column; {new_index.cname!r} is reserved for the "
5035 "implicit row index"
5036 )
5038 frame = self.get_object(obj, transposed)._consolidate()
5040 blocks, blk_items = self._get_blocks_and_items(
5041 frame, table_exists, new_non_index_axes, self.values_axes, data_columns
5042 )
5044 # add my values
5045 vaxes = []
5046 for i, (blk, b_items) in enumerate(zip(blocks, blk_items, strict=True)):
5047 # shape of the data column are the indexable axes
5048 klass = DataCol
5049 name = None
5051 # we have a data_column
5052 if data_columns and len(b_items) == 1 and b_items[0] in data_columns:
5053 klass = DataIndexableCol
5054 name = b_items[0]
5055 if not (name is None or isinstance(name, str)):
5056 # TODO: should the message here be more specifically non-str?
5057 raise ValueError("cannot have non-object label DataIndexableCol")
5059 # make sure that we match up the existing columns
5060 # if we have an existing table
5061 existing_col: DataCol | None
5063 if table_exists and validate:
5064 try:
5065 existing_col = self.values_axes[i]
5066 except (IndexError, KeyError) as err:
5067 raise ValueError(
5068 f"Incompatible appended table [{blocks}]"
5069 f"with existing table [{self.values_axes}]"
5070 ) from err
5071 else:
5072 existing_col = None
5074 new_name = name or f"values_block_{i}"
5076 # GH#9604: the read path honors a per-column sentinel only when
5077 # the table carries the marker attribute, which set_attrs writes at
5078 # creation and an append cannot add retroactively. So when appending
5079 # to a table written before the marker existed, keep encoding
5080 # missing values with the table-wide nan_rep, otherwise the sentinel
5081 # would be read back as a literal string.
5082 level_names = self.levels if isinstance(self.levels, list) else []
5083 is_level = name is not None and name in level_names
5084 if table_exists:
5085 uses_col_nan_rep = bool(
5086 getattr(self.attrs, "data_col_nan_rep", False)
5087 ) or (is_level and bool(getattr(self.attrs, "mi_level_nan_rep", False)))
5088 else:
5089 # a MultiIndex level is part of the index rather than of the
5090 # data, so a caller-supplied nan_rep does not govern it
5091 uses_col_nan_rep = user_nan_rep is None or is_level
5092 col_nan_rep = None
5093 if uses_col_nan_rep:
5094 col_nan_rep = _make_data_col_nan_rep(
5095 blk.values,
5096 b_items,
5097 existing_col,
5098 self.table,
5099 self.encoding,
5100 self.errors,
5101 )
5102 block_nan_rep = col_nan_rep if col_nan_rep is not None else nan_rep
5104 data_converted = _maybe_convert_for_string_atom(
5105 new_name,
5106 blk.values,
5107 existing_col=existing_col,
5108 min_itemsize=min_itemsize,
5109 nan_rep=block_nan_rep,
5110 encoding=self.encoding,
5111 errors=self.errors,
5112 columns=b_items,
5113 )
5115 typ = klass._get_atom(data_converted)
5116 kind = _dtype_to_kind(data_converted.dtype.name)
5117 tz = None
5118 if getattr(data_converted, "tz", None) is not None:
5119 tz = _get_tz(data_converted.tz)
5121 meta = metadata = ordered = None
5122 if isinstance(data_converted.dtype, CategoricalDtype):
5123 ordered = data_converted.ordered
5124 meta = "category"
5125 metadata = np.asarray(data_converted.categories).ravel()
5126 elif isinstance(blk.dtype, StringDtype):
5127 meta = str(blk.dtype)
5129 data, dtype_name = _get_data_and_dtype_name(data_converted)
5131 # A new tz-aware data column's kind carries the tz while the
5132 # stored one's does not, so gate on the dtype actually persisted.
5133 if existing_col is not None and existing_col.dtype == dtype_name:
5134 _check_tz_conflict(new_name, new_info.get(new_name, {}).get("tz"), tz)
5136 col = klass(
5137 name=new_name,
5138 cname=new_name,
5139 values=list(b_items),
5140 typ=typ,
5141 pos=j,
5142 kind=kind,
5143 tz=tz,
5144 ordered=ordered,
5145 meta=meta,
5146 metadata=metadata,
5147 dtype=dtype_name,
5148 data=data,
5149 )
5150 col.uses_col_nan_rep = uses_col_nan_rep
5151 col.nan_rep = col_nan_rep
5152 col.update_info(new_info)
5154 vaxes.append(col)
5156 j += 1
5158 dcs = [col.name for col in vaxes if col.is_data_indexable]
5160 new_table = type(self)(
5161 parent=self.parent,
5162 group=self.group,
5163 encoding=self.encoding,
5164 errors=self.errors,
5165 index_axes=new_index_axes,
5166 non_index_axes=new_non_index_axes,
5167 values_axes=vaxes,
5168 data_columns=dcs,
5169 info=new_info,
5170 nan_rep=user_nan_rep,
5171 )
5172 if hasattr(self, "levels"):
5173 # TODO: get this into constructor, only for appropriate subclass
5174 new_table.levels = self.levels
5176 new_table.validate_min_itemsize(min_itemsize)
5178 if validate and table_exists:
5179 new_table.validate(self)
5181 return new_table
5183 @staticmethod
5184 def _get_blocks_and_items(
5185 frame: DataFrame,
5186 table_exists: bool,
5187 new_non_index_axes,
5188 values_axes,
5189 data_columns,
5190 ):
5191 # Helper to clarify non-state-altering parts of _create_axes
5192 def get_blk_items(mgr):
5193 return [mgr.items.take(blk.mgr_locs) for blk in mgr.blocks]
5195 mgr = frame._mgr
5196 blocks: list[Block] = list(mgr.blocks)
5197 blk_items: list[Index] = get_blk_items(mgr)
5199 if len(data_columns):
5200 # TODO: prove that we only get here with axis == 1?
5201 # It is the case in all extant tests, but NOT the case
5202 # outside this `if len(data_columns)` check.
5204 axis, axis_labels = new_non_index_axes[0]
5205 new_labels = Index(axis_labels).difference(Index(data_columns))
5206 mgr = frame.reindex(new_labels, axis=axis)._mgr
5208 blocks = list(mgr.blocks)
5209 blk_items = get_blk_items(mgr)
5210 for c in data_columns:
5211 # This reindex would raise ValueError if we had a duplicate
5212 # index, so we can infer that (as long as axis==1) we
5213 # get a single column back, so a single block.
5214 mgr = frame.reindex([c], axis=axis)._mgr
5215 blocks.extend(mgr.blocks)
5216 blk_items.extend(get_blk_items(mgr))
5218 # reorder the blocks in the same order as the existing table if we can
5219 if table_exists:
5220 by_items = {
5221 tuple(b_items.tolist()): (b, b_items)
5222 for b, b_items in zip(blocks, blk_items, strict=True)
5223 }
5224 new_blocks: list[Block] = []
5225 new_blk_items = []
5226 for ea in values_axes:
5227 items = tuple(ea.values)
5228 try:
5229 b, b_items = by_items.pop(items)
5230 new_blocks.append(b)
5231 new_blk_items.append(b_items)
5232 except (IndexError, KeyError) as err:
5233 jitems = ",".join([pprint_thing(item) for item in items])
5234 raise ValueError(
5235 f"cannot match existing table structure for [{jitems}] "
5236 "on appending data"
5237 ) from err
5238 blocks = new_blocks
5239 blk_items = new_blk_items
5241 return blocks, blk_items
5243 def process_axes(self, obj, selection: Selection, columns=None) -> DataFrame:
5244 """process axes filters"""
5245 # make a copy to avoid side effects
5246 if columns is not None:
5247 columns = list(columns)
5249 # make sure to include levels if we have them
5250 if columns is not None and self.is_multi_index:
5251 assert isinstance(self.levels, list) # assured by is_multi_index
5252 for n in self.levels:
5253 if n not in columns:
5254 columns.insert(0, n)
5256 # reorder by any non_index_axes & limit to the select columns
5257 for axis, labels in self.non_index_axes:
5258 obj = _reindex_axis(obj, axis, labels, columns)
5260 def process_filter(field, filt, op):
5261 for axis_name in obj._AXIS_ORDERS:
5262 axis_number = obj._get_axis_number(axis_name)
5263 axis_values = obj._get_axis(axis_name)
5264 assert axis_number is not None
5266 # see if the field is the name of an axis
5267 if field == axis_name:
5268 # if we have a multi-index, then need to include
5269 # the levels
5270 if self.is_multi_index:
5271 filt = filt.union(Index(self.levels))
5273 takers = op(axis_values, filt)
5274 return obj.loc(axis=axis_number)[takers]
5276 # this might be the name of a file IN an axis
5277 elif field in axis_values:
5278 # we need to filter on this dimension
5279 # use the column Series rather than .values so the
5280 # filter respects the column dtype -- .values strips
5281 # tz from a datetimetz column and the isin below then
5282 # matches nothing (GH#12953). This mirrors the
5283 # coordinate path in Selection.select_coords.
5284 values = obj[field]
5285 filt = ensure_index(filt)
5287 # hack until we support reversed dim flags
5288 if isinstance(obj, DataFrame):
5289 axis_number = 1 - axis_number
5291 takers = op(values, filt)
5292 return obj.loc(axis=axis_number)[takers]
5294 raise ValueError(f"cannot find the field [{field}] for filtering!")
5296 # apply the selection filters (but keep in the same order)
5297 if selection.filter is not None:
5298 for field, op, filt in selection.filter.format():
5299 obj = process_filter(field, filt, op)
5301 return obj
5303 def create_description(
5304 self,
5305 complib,
5306 complevel: int | None,
5307 fletcher32: bool,
5308 expectedrows: int | None,
5309 ) -> dict[str, Any]:
5310 """create the description of the table from the axes & values"""
5311 # provided expected rows if its passed
5312 if expectedrows is None:
5313 expectedrows = max(self.nrows_expected, 10000)
5315 d = {"name": "table", "expectedrows": expectedrows}
5317 # description from the axes & values
5318 d["description"] = {a.cname: a.typ for a in self.axes}
5320 if complib:
5321 if complevel is None:
5322 complevel = self._complevel or 9
5323 filters = _tables().Filters(
5324 complevel=complevel,
5325 complib=complib,
5326 fletcher32=fletcher32 or self._fletcher32,
5327 )
5328 d["filters"] = filters
5329 elif self._filters is not None:
5330 d["filters"] = self._filters
5332 return d
5334 def read_coordinates(
5335 self, where=None, start: int | None = None, stop: int | None = None
5336 ):
5337 """
5338 select coordinates (row numbers) from a table; return the
5339 coordinates object
5340 """
5341 # infer the data kind
5342 if not self.infer_axes():
5343 return False
5345 if where is not None and self._where_selects_columns(where):
5346 raise NotImplementedError(
5347 "selecting columns through the 'where' expression "
5348 "(e.g. \"columns=['A']\") is a column projection, not a row "
5349 "filter, so it cannot be read as row coordinates (e.g. with "
5350 "select_as_coordinates, select_as_multiple, iterator, or "
5351 "chunksize); use the 'columns' argument of 'select' instead"
5352 )
5354 # create the selection
5355 selection = Selection(self, where=where, start=start, stop=stop)
5356 coords = selection.select_coords()
5358 return Index(coords, copy=False)
5360 def _where_selects_columns(self, where) -> bool:
5361 """
5362 Whether ``where`` contains a column selection such as "columns=['A']".
5364 Such a selection is a column projection rather than a row filter, which
5365 cannot be applied via row coordinates and so is unsupported on any
5366 coordinate-based read -- ``iterator``/``chunksize``,
5367 :meth:`select_as_coordinates`, and :meth:`select_as_multiple`
5368 (GH#12953); ``read_coordinates`` raises a clear error instead of a
5369 cryptic ``KeyError``.
5370 """
5371 if not self.non_index_axes:
5372 return False
5374 selection = Selection(self, where=where)
5375 if selection.filter is None:
5376 return False
5378 # a data-column row filter has a field matching a readable column; a
5379 # column selection's field is the column-axis name (e.g. "columns")
5380 data_column_names = {a.name for a in self.axes}
5381 return any(
5382 field not in data_column_names
5383 for field, _op, _filt in selection.filter.format()
5384 )
5386 def read_column(
5387 self,
5388 column: str,
5389 where=None,
5390 start: int | None = None,
5391 stop: int | None = None,
5392 ):
5393 """
5394 return a single column from the table, generally only indexables
5395 are interesting
5396 """
5397 # infer the data kind
5398 if not self.infer_axes():
5399 return False
5401 if where is not None:
5402 raise TypeError("read_column does not currently accept a where clause")
5404 # find the axes
5405 for a in self.axes:
5406 if column == a.name:
5407 if not a.is_data_indexable:
5408 raise ValueError(
5409 f"column [{column}] can not be extracted individually; "
5410 "it is not data indexable"
5411 )
5413 # column must be an indexable or a data column
5414 c = getattr(self.table.cols, column)
5415 a.set_info(self.info)
5416 col_values = a.convert(
5417 c[start:stop],
5418 nan_rep=self.nan_rep,
5419 encoding=self.encoding,
5420 errors=self.errors,
5421 )
5422 cvs = col_values[1]
5423 dtype = getattr(self.table.attrs, f"{column}_meta", None)
5424 return Series(cvs, name=column, copy=False, dtype=dtype)
5426 raise KeyError(f"column [{column}] not found in the table")
5429class WORMTable(Table):
5430 """
5431 a write-once read-many table: this format DOES NOT ALLOW appending to a
5432 table. writing is a one-time operation the data are stored in a format
5433 that allows for searching the data on disk
5434 """
5436 table_type = "worm"
5438 def read(
5439 self,
5440 where=None,
5441 columns=None,
5442 start: int | None = None,
5443 stop: int | None = None,
5444 ):
5445 """
5446 read the indices and the indexing array, calculate offset rows and return
5447 """
5448 raise NotImplementedError("WORMTable needs to implement read")
5450 def write(self, obj, **kwargs) -> None:
5451 """
5452 write in a format that we can search later on (but cannot append
5453 to): write out the indices and the values using _write_array
5454 (e.g. a CArray) create an indexing table so that we can search
5455 """
5456 raise NotImplementedError("WORMTable needs to implement write")
5459class AppendableTable(Table):
5460 """support the new appendable table formats"""
5462 table_type = "appendable"
5464 # error: Signature of "write" incompatible with supertype "Fixed"
5465 def write( # type: ignore[override]
5466 self,
5467 obj,
5468 axes=None,
5469 append: bool = False,
5470 complib=None,
5471 complevel=None,
5472 fletcher32=None,
5473 min_itemsize=None,
5474 chunksize: int | None = None,
5475 expectedrows=None,
5476 dropna: bool = False,
5477 nan_rep=None,
5478 data_columns=None,
5479 track_times: bool = True,
5480 ) -> None:
5481 if not append and self.is_exists:
5482 self._handle.remove_node(self.group, "table")
5484 # create the axes
5485 table = self._create_axes(
5486 axes=axes,
5487 obj=obj,
5488 validate=append,
5489 min_itemsize=min_itemsize,
5490 nan_rep=nan_rep,
5491 data_columns=data_columns,
5492 )
5494 for a in table.axes:
5495 a.validate_names()
5497 if not table.is_exists:
5498 # create the table
5499 options = table.create_description(
5500 complib=complib,
5501 complevel=complevel,
5502 fletcher32=fletcher32,
5503 expectedrows=expectedrows,
5504 )
5506 # set the table attributes
5507 table.set_attrs()
5509 options["track_times"] = track_times
5511 # create the table
5512 table._handle.create_table(table.group, **options)
5514 # update my info
5515 table.attrs.info = table.info
5517 # validate the axes and set the kinds
5518 for a in table.axes:
5519 a.validate_and_set(table, append)
5521 # add the rows
5522 table.write_data(chunksize, dropna=dropna)
5524 def write_data(self, chunksize: int | None, dropna: bool = False) -> None:
5525 """
5526 we form the data into a 2-d including indexes,values,mask write chunk-by-chunk
5527 """
5528 names = self.dtype.names
5529 nrows = self.nrows_expected
5531 # if dropna==True, then drop ALL nan rows
5532 masks = []
5533 if dropna:
5534 for a in self.values_axes:
5535 # figure the mask: only do if we can successfully process this
5536 # column, otherwise ignore the mask
5537 mask = isna(a.data).all(axis=0)
5538 if isinstance(mask, np.ndarray):
5539 masks.append(mask.astype("u1", copy=False))
5541 # consolidate masks
5542 if masks:
5543 mask = masks[0]
5544 for m in masks[1:]:
5545 mask = mask & m
5546 mask = mask.ravel()
5547 else:
5548 mask = None
5550 # broadcast the indexes if needed
5551 indexes = [a.cvalues for a in self.index_axes]
5552 nindexes = len(indexes)
5553 assert nindexes == 1, nindexes # ensures we dont need to broadcast
5555 # transpose the values so first dimension is last
5556 # reshape the values if needed
5557 values = [a.take_data() for a in self.values_axes]
5558 values = [v.transpose(np.roll(np.arange(v.ndim), v.ndim - 1)) for v in values]
5559 bvalues = []
5560 for i, v in enumerate(values):
5561 new_shape = (nrows, *self.dtype[names[nindexes + i]].shape)
5562 bvalues.append(v.reshape(new_shape))
5564 # write the chunks
5565 if chunksize is None:
5566 chunksize = 100000
5568 rows = np.empty(min(chunksize, nrows), dtype=self.dtype)
5569 chunks = nrows // chunksize + 1
5570 for i in range(chunks):
5571 start_i = i * chunksize
5572 end_i = min((i + 1) * chunksize, nrows)
5573 if start_i >= end_i:
5574 break
5576 self.write_data_chunk(
5577 rows,
5578 indexes=[a[start_i:end_i] for a in indexes], # pyright: ignore[reportOptionalSubscript]
5579 mask=mask[start_i:end_i] if mask is not None else None,
5580 values=[v[start_i:end_i] for v in bvalues],
5581 )
5583 def write_data_chunk(
5584 self,
5585 rows: np.ndarray,
5586 indexes: list[np.ndarray],
5587 mask: npt.NDArray[np.bool_] | None,
5588 values: list[np.ndarray],
5589 ) -> None:
5590 """
5591 Parameters
5592 ----------
5593 rows : an empty memory space where we are putting the chunk
5594 indexes : an array of the indexes
5595 mask : an array of the masks
5596 values : an array of the values
5597 """
5598 # 0 len
5599 for v in values:
5600 if not np.prod(v.shape):
5601 return
5603 nrows = indexes[0].shape[0]
5604 if nrows != len(rows):
5605 rows = np.empty(nrows, dtype=self.dtype)
5606 names = self.dtype.names
5607 nindexes = len(indexes)
5609 # indexes
5610 for i, idx in enumerate(indexes):
5611 rows[names[i]] = idx
5613 # values
5614 for i, v in enumerate(values):
5615 rows[names[i + nindexes]] = v
5617 # mask
5618 if mask is not None:
5619 m = ~mask.ravel().astype(bool, copy=False)
5620 if not m.all():
5621 rows = rows[m]
5623 if len(rows):
5624 self.table.append(rows)
5625 self.table.flush()
5627 def delete(
5628 self, where=None, start: int | None = None, stop: int | None = None
5629 ) -> int | None:
5630 # delete all rows (and return the nrows)
5631 if where is None or not len(where):
5632 if start is None and stop is None:
5633 nrows = self.nrows
5634 self._handle.remove_node(self.group, recursive=True)
5635 else:
5636 # pytables<3.0 would remove a single row with stop=None
5637 if stop is None:
5638 stop = self.nrows
5639 nrows = self.table.remove_rows(start=start, stop=stop)
5640 self.table.flush()
5641 return nrows
5643 # infer the data kind
5644 if not self.infer_axes():
5645 return None
5647 # create the selection
5648 table = self.table
5649 selection = Selection(self, where, start=start, stop=stop)
5650 values = selection.select_coords()
5652 # delete the rows in reverse order
5653 sorted_series = Series(values, copy=False).sort_values()
5654 ln = len(sorted_series)
5656 if ln:
5657 # construct groups of consecutive rows
5658 diff = sorted_series.diff()
5659 groups = list(diff[diff > 1].index)
5661 # 1 group
5662 if not groups:
5663 groups = [0]
5665 # final element
5666 if groups[-1] != ln:
5667 groups.append(ln)
5669 # initial element
5670 if groups[0] != 0:
5671 groups.insert(0, 0)
5673 # we must remove in reverse order!
5674 pg = groups.pop()
5675 for g in reversed(groups):
5676 rows = sorted_series.take(range(g, pg))
5677 table.remove_rows(
5678 start=rows[rows.index[0]], stop=rows[rows.index[-1]] + 1
5679 )
5680 pg = g
5682 self.table.flush()
5684 # return the number of rows removed
5685 return ln
5688class AppendableFrameTable(AppendableTable):
5689 """support the new appendable table formats"""
5691 pandas_kind = "frame_table"
5692 table_type = "appendable_frame"
5693 ndim = 2
5694 obj_type: type[DataFrame | Series] = DataFrame
5696 @property
5697 def is_transposed(self) -> bool:
5698 return self.index_axes[0].axis == 1
5700 @classmethod
5701 def get_object(cls, obj, transposed: bool):
5702 """these are written transposed"""
5703 if transposed:
5704 obj = obj.T
5705 return obj
5707 def read(
5708 self,
5709 where=None,
5710 columns=None,
5711 start: int | None = None,
5712 stop: int | None = None,
5713 ):
5714 # infer the data kind
5715 if not self.infer_axes():
5716 return None
5718 result = self._read_axes(where=where, start=start, stop=stop)
5720 info = (
5721 self.info.get(self.non_index_axes[0][0], {})
5722 if len(self.non_index_axes)
5723 else {}
5724 )
5726 inds = [i for i, ax in enumerate(self.axes) if ax is self.index_axes[0]]
5727 assert len(inds) == 1
5728 ind = inds[0]
5730 index = result[ind][0]
5732 frames = []
5733 for i, a in enumerate(self.axes):
5734 if a not in self.values_axes:
5735 continue
5736 index_vals, cvalues = result[i]
5738 # we could have a multi-index constructor here
5739 # ensure_index doesn't recognized our list-of-tuples here
5740 if info.get("type") != "MultiIndex":
5741 cols = Index(index_vals)
5742 else:
5743 cols = MultiIndex.from_tuples(index_vals)
5745 names = info.get("names")
5746 if names is not None:
5747 cols.set_names(names, inplace=True)
5749 if self.is_transposed:
5750 values = cvalues
5751 index_ = cols
5752 cols_ = Index(index, name=getattr(index, "name", None))
5753 else:
5754 values = cvalues.T
5755 index_ = Index(index, name=getattr(index, "name", None))
5756 cols_ = cols
5758 # if we have a DataIndexableCol, its shape will only be 1 dim
5759 if values.ndim == 1 and isinstance(values, np.ndarray):
5760 values = values.reshape((1, values.shape[0]))
5762 if isinstance(values, (np.ndarray, DatetimeArray)):
5763 try:
5764 df = DataFrame(values.T, columns=cols_, index=index_, copy=False)
5765 except UnicodeEncodeError as err:
5766 if (
5767 self.errors == "surrogatepass"
5768 and using_string_dtype()
5769 and str(err).endswith("surrogates not allowed")
5770 and HAS_PYARROW
5771 ):
5772 df = DataFrame(
5773 values.T,
5774 columns=cols_,
5775 index=index_,
5776 copy=False,
5777 dtype=StringDtype(storage="python", na_value=np.nan),
5778 )
5779 else:
5780 raise
5781 elif isinstance(values, Index):
5782 df = DataFrame(values, columns=cols_, index=index_)
5783 else:
5784 # Categorical
5785 df = DataFrame._from_arrays([values], columns=cols_, index=index_)
5786 if not (using_string_dtype() and values.dtype.kind == "O"):
5787 assert (df.dtypes == values.dtype).all(), (df.dtypes, values.dtype)
5789 # If str / string dtype is stored in meta, use that.
5790 for column in cols_:
5791 dtype = getattr(self.table.attrs, f"{column}_meta", None)
5792 if dtype in ["str", "string"]:
5793 df[column] = df[column].astype(dtype)
5794 frames.append(df)
5796 if len(frames) == 1:
5797 df = frames[0]
5798 else:
5799 df = concat(frames, axis=1)
5801 selection = Selection(self, where=where, start=start, stop=stop)
5802 # apply the selection filters & axis orderings
5803 df = self.process_axes(df, selection=selection, columns=columns)
5804 return df
5807class AppendableSeriesTable(AppendableFrameTable):
5808 """support the new appendable table formats"""
5810 pandas_kind = "series_table"
5811 table_type = "appendable_series"
5812 ndim = 2
5813 obj_type = Series
5815 @property
5816 def is_transposed(self) -> bool:
5817 return False
5819 @classmethod
5820 def get_object(cls, obj, transposed: bool):
5821 return obj
5823 # error: Signature of "write" incompatible with supertype "Fixed"
5824 def write(self, obj, data_columns=None, **kwargs) -> None: # type: ignore[override]
5825 """we are going to write this as a frame table"""
5826 if not isinstance(obj, DataFrame):
5827 name = obj.name or "values"
5828 obj = obj.to_frame(name)
5829 super().write(obj=obj, data_columns=obj.columns.tolist(), **kwargs)
5831 def read(
5832 self,
5833 where=None,
5834 columns=None,
5835 start: int | None = None,
5836 stop: int | None = None,
5837 ) -> Series:
5838 is_multi_index = self.is_multi_index
5839 if columns is not None and is_multi_index:
5840 assert isinstance(self.levels, list) # needed for mypy
5841 for n in self.levels:
5842 if n not in columns:
5843 columns.insert(0, n)
5844 s = super().read(where=where, columns=columns, start=start, stop=stop)
5845 if is_multi_index:
5846 s = s.set_index(self.levels)
5848 s = s.iloc[:, 0]
5850 # remove the default name
5851 if s.name == "values":
5852 s.name = None
5853 return s
5856class AppendableMultiSeriesTable(AppendableSeriesTable):
5857 """support the new appendable table formats"""
5859 pandas_kind = "series_table"
5860 table_type = "appendable_multiseries"
5862 # error: Signature of "write" incompatible with supertype "Fixed"
5863 def write(self, obj, **kwargs) -> None: # type: ignore[override]
5864 """we are going to write this as a frame table"""
5865 name = obj.name or "values"
5866 newobj, self.levels = self.validate_multiindex(obj)
5867 assert isinstance(self.levels, list) # for mypy
5868 cols = list(self.levels)
5869 cols.append(name)
5870 newobj.columns = Index(cols)
5871 super().write(obj=newobj, **kwargs)
5874class GenericTable(AppendableFrameTable):
5875 """a table that read/writes the generic pytables table format"""
5877 pandas_kind = "frame_table"
5878 table_type = "generic_table"
5879 ndim = 2
5880 obj_type = DataFrame
5881 levels: list[Hashable]
5883 @property
5884 def pandas_type(self) -> str:
5885 return self.pandas_kind
5887 @property
5888 def storable(self):
5889 return getattr(self.group, "table", None) or self.group
5891 def get_attrs(self) -> None:
5892 """retrieve our attributes"""
5893 self.non_index_axes = []
5894 self.nan_rep = None
5895 self.levels = []
5897 self.index_axes = [a for a in self.indexables if a.is_an_indexable]
5898 self.values_axes = [a for a in self.indexables if not a.is_an_indexable]
5899 self.data_columns = [a.name for a in self.values_axes]
5901 @cache_readonly
5902 def indexables(self):
5903 """create the indexables from the table description"""
5904 d = self.description
5906 # TODO: can we get a typ for this? AFAICT it is the only place
5907 # where we aren't passing one
5908 # the index columns is just a simple index
5909 md = self.read_metadata("index")
5910 meta = "category" if md is not None else None
5911 index_col = GenericIndexCol(
5912 name="index", axis=0, table=self.table, meta=meta, metadata=md
5913 )
5915 _indexables: list[GenericIndexCol | GenericDataIndexableCol] = [index_col]
5917 for i, n in enumerate(d._v_names):
5918 assert isinstance(n, str)
5920 atom = getattr(d, n)
5921 md = self.read_metadata(n)
5922 meta = "category" if md is not None else None
5923 dc = GenericDataIndexableCol(
5924 name=n,
5925 pos=i,
5926 values=[n],
5927 typ=atom,
5928 table=self.table,
5929 meta=meta,
5930 metadata=md,
5931 )
5932 _indexables.append(dc)
5934 return _indexables
5936 # error: Signature of "write" incompatible with supertype "AppendableTable"
5937 def write(self, **kwargs) -> None: # type: ignore[override]
5938 raise NotImplementedError("cannot write on a generic table")
5941class AppendableMultiFrameTable(AppendableFrameTable):
5942 """a frame with a multi-index"""
5944 table_type = "appendable_multiframe"
5945 obj_type = DataFrame
5946 ndim = 2
5947 _re_levels = re.compile(r"^level_\d+$")
5949 @property
5950 def table_type_short(self) -> str:
5951 return "appendable_multi"
5953 # error: Signature of "write" incompatible with supertype "Fixed"
5954 def write(self, obj, data_columns=None, **kwargs) -> None: # type: ignore[override]
5955 if data_columns is None:
5956 data_columns = []
5957 elif data_columns is True:
5958 data_columns = obj.columns.tolist()
5959 obj, self.levels = self.validate_multiindex(obj)
5960 assert isinstance(self.levels, list) # for mypy
5961 for n in self.levels:
5962 if n not in data_columns:
5963 data_columns.insert(0, n)
5964 super().write(obj=obj, data_columns=data_columns, **kwargs)
5966 def read(
5967 self,
5968 where=None,
5969 columns=None,
5970 start: int | None = None,
5971 stop: int | None = None,
5972 ) -> DataFrame:
5973 df = super().read(where=where, columns=columns, start=start, stop=stop)
5974 df = df.set_index(self.levels)
5976 # remove names for 'level_%d'
5977 df.index = df.index.set_names(
5978 [None if self._re_levels.search(name) else name for name in df.index.names]
5979 )
5981 return df
5984def _reindex_axis(
5985 obj: DataFrame, axis: AxisInt, labels: Index, other=None
5986) -> DataFrame:
5987 ax = obj._get_axis(axis)
5988 labels = ensure_index(labels)
5990 # try not to reindex even if other is provided
5991 # if it equals our current index
5992 if other is not None:
5993 other = ensure_index(other)
5994 if (other is None or labels.equals(other)) and labels.equals(ax):
5995 return obj
5997 labels = ensure_index(labels.unique())
5998 if other is not None:
5999 labels = ensure_index(other.unique()).intersection(labels, sort=False)
6000 if not labels.equals(ax):
6001 slicer: list[slice | Index] = [slice(None, None)] * obj.ndim
6002 slicer[axis] = labels
6003 obj = obj.loc[tuple(slicer)]
6004 return obj
6007# tz to/from coercion
6010def _get_tz(tz: tzinfo) -> str | tzinfo:
6011 """for a tz-aware type, return an encoded zone"""
6012 zone = timezones.get_timezone(tz)
6013 return zone
6016def _check_tz_conflict(
6017 name: str, existing_tz: str | tzinfo | None, tz: str | tzinfo | None
6018) -> None:
6019 """
6020 Raise if an append would add or drop the timezone of a stored column.
6022 The read path localizes the whole stored column with this single tz, so
6023 changing it reinterprets the rows already written. update_info raises only
6024 when both sides are set, leaving the None cases silent. GH#68583
6025 """
6026 if existing_tz is None and tz is not None:
6027 raise TypeError(f"cannot append tz-aware data to tz-naive col [{name}]")
6028 if existing_tz is not None and tz is None:
6029 raise TypeError(f"cannot append tz-naive data to tz-aware col [{name}]")
6032def _set_tz(
6033 values: npt.NDArray[np.int64], tz: str | tzinfo | None, datetime64_dtype: str
6034) -> DatetimeArray:
6035 """
6036 Coerce the values to a DatetimeArray with appropriate tz.
6038 Parameters
6039 ----------
6040 values : ndarray[int64]
6041 tz : str, tzinfo, or None
6042 datetime64_dtype : str, e.g. "datetime64[ns]", "datetime64[25s]"
6043 """
6044 assert values.dtype == "i8", values.dtype
6045 # Argument "tz" to "tz_to_dtype" has incompatible type "str | tzinfo | None";
6046 # expected "tzinfo"
6047 unit, _ = np.datetime_data(datetime64_dtype) # parsing dtype: unit, count
6048 unit = cast("TimeUnit", unit)
6049 # error: Argument "tz" to "tz_to_dtype" has incompatible type
6050 # "str | tzinfo | None"; expected "tzinfo"
6051 dtype = tz_to_dtype(tz=tz, unit=unit) # type: ignore[arg-type]
6052 dta = DatetimeArray._from_sequence(values, dtype=dtype)
6053 return dta
6056def _read_index_nan_rep(attrs, name: str = "nan_rep") -> str | None:
6057 """
6058 Read back the NaN sentinel persisted for a string Index column (GH#9604).
6060 The writer records the attribute for every string Index, empty when the
6061 index had no missing value, so its absence means the file predates the
6062 sentinel. Such a file stored a NaN as the bare string "nan", which is how
6063 pandas read it back before the sentinel existed.
6064 """
6065 nan_rep = getattr(attrs, name, "nan")
6066 return nan_rep or None
6069def _make_nan_rep(
6070 values: np.ndarray, mask: np.ndarray, existing: set | None = None
6071) -> str:
6072 """
6073 Choose a string NaN sentinel for a string column that does not collide with
6074 any non-missing value, so genuine missing values round-trip without being
6075 confused with a literal string such as ``"nan"`` (GH#9604).
6077 ``existing`` holds values already stored for this column in a prior
6078 ``table``-format append; the sentinel must avoid those too, since a single
6079 sentinel is persisted for the whole column.
6080 """
6081 non_missing = values[~mask]
6082 if lib.infer_dtype(non_missing, skipna=True) != "string":
6083 # Only a str can equal the sentinel, and an elementwise == against a
6084 # list or ndarray element is not a bool. Dropping the rest also keeps an
6085 # unhashable element from raising here, ahead of the "Cannot serialize
6086 # the column" message it deserves.
6087 non_missing = np.array(
6088 [val for val in non_missing if isinstance(val, str)], dtype=object
6089 )
6090 nan_rep = "nan"
6091 while (non_missing == nan_rep).any() or (
6092 existing is not None and nan_rep in existing
6093 ):
6094 nan_rep = f"_{nan_rep}_"
6095 return nan_rep
6098def _read_stored_column_values(table, cname: str, encoding: str, errors: str) -> set:
6099 """
6100 Read the values already stored for a string column (GH#9604), so a NaN
6101 sentinel chosen for a later append cannot collide with them.
6102 """
6103 raw = getattr(table.cols, cname)[:]
6104 decoded = _unconvert_string_array(
6105 raw, nan_rep=None, encoding=encoding, errors=errors
6106 )
6107 # a multi-column data block stores a 2-D array; one sentinel covers it all
6108 return set(decoded.ravel())
6111def _make_data_col_nan_rep(
6112 blk_values,
6113 columns: list,
6114 existing_col,
6115 table,
6116 encoding: str,
6117 errors: str,
6118) -> str | None:
6119 """
6120 Choose the NaN sentinel for a string data column, mirroring the string-Index
6121 handling (GH#9604): reuse the sentinel already persisted for the column,
6122 otherwise mint a collision-free one, and only when the column actually has a
6123 missing value. Returns None when no sentinel is needed (a literal "nan" then
6124 round-trips because the read path skips substitution when the sentinel is
6125 absent).
6126 """
6127 if isinstance(blk_values.dtype, StringDtype):
6128 blk_values = blk_values.to_numpy()
6129 if blk_values.dtype != object:
6130 return None
6132 values = np.asarray(blk_values)
6133 flat = values.reshape(-1)
6134 mask = isna(flat)
6135 nan_rep = existing_col.nan_rep if existing_col is not None else None
6136 if mask.any() and nan_rep is None:
6137 existing_values = None
6138 if existing_col is not None and existing_col.kind == "string":
6139 existing_values = _read_stored_column_values(
6140 table, existing_col.cname, encoding, errors
6141 )
6142 nan_rep = _make_nan_rep(flat, mask, existing_values)
6143 if (
6144 nan_rep is not None
6145 # an elementwise == against a list/ndarray element is not a bool, and a
6146 # non-string column has _maybe_convert_for_string_atom's much better
6147 # "Cannot serialize the column" error waiting for it anyway
6148 and lib.infer_dtype(flat, skipna=True) == "string"
6149 and (flat[~mask] == nan_rep).any()
6150 ):
6151 # A real value equal to the sentinel is indistinguishable from a
6152 # missing value on read; refuse rather than silently corrupt it.
6153 collides = (values == nan_rep) & ~isna(values)
6154 # a block is (n_columns, n_rows), so the first axis picks the label
6155 pos = int(np.argmax(collides.any(axis=1))) if collides.ndim == 2 else 0
6156 label = columns[pos]
6157 raise ValueError(
6158 f"Cannot store the string {str(nan_rep)!r} in column [{label}]: it "
6159 "collides with the sentinel used to encode missing values on disk"
6160 )
6161 return nan_rep
6164def _convert_index(
6165 name: str,
6166 index: Index,
6167 encoding: str,
6168 errors: str,
6169 existing_nan_rep: str | None = None,
6170 existing_values: set | None = None,
6171) -> IndexCol:
6172 assert isinstance(name, str)
6174 index_name = index.name
6175 # GH#26144, GH#38305, GH#42070: most extension dtypes cannot be stored as
6176 # an Index in HDF5. Allow the ones that have working code paths below
6177 # (DatetimeTZ/Period via i8 conversion, BooleanDtype via is_bool_dtype,
6178 # StringDtype, CategoricalDtype) and reject everything else early before
6179 # _dtype_to_kind / _get_atom choke with a low-level error.
6180 # BooleanDtype round-trips lossily but does not error on main, so it
6181 # remains carved out here.
6182 if (
6183 isinstance(index.dtype, ExtensionDtype)
6184 and not needs_i8_conversion(index.dtype)
6185 and not is_bool_dtype(index.dtype)
6186 and not isinstance(index.dtype, (StringDtype, CategoricalDtype))
6187 ):
6188 raise NotImplementedError(
6189 f"Cannot store an Index with dtype {index.dtype} in an HDF5 file. "
6190 "HDFStore supports NumPy dtypes, datetime64 with timezone, "
6191 "categorical, and string extension types."
6192 )
6194 if isinstance(index.dtype, CategoricalDtype):
6195 # GH#33909, GH#16118: round-trip a CategoricalIndex by storing its
6196 # integer codes as the column data and the categories as metadata,
6197 # mirroring how a CategoricalDtype data column is persisted.
6198 cat = cast("Categorical", index._values)
6199 converted, dtype_name = _get_data_and_dtype_name(cat)
6200 kind = _dtype_to_kind(dtype_name)
6201 atom = DataIndexableCol._get_atom(converted)
6202 return IndexCol(
6203 name,
6204 values=converted,
6205 kind=kind,
6206 typ=atom,
6207 index_name=index_name,
6208 meta="category",
6209 metadata=np.asarray(cat.categories).ravel(),
6210 ordered=cat.ordered,
6211 )
6213 # error: Argument 1 to "_get_data_and_dtype_name" has incompatible type "Index";
6214 # expected "Union[ExtensionArray, ndarray]"
6215 converted, dtype_name = _get_data_and_dtype_name(index) # type: ignore[arg-type]
6216 kind = _dtype_to_kind(dtype_name)
6217 atom = DataIndexableCol._get_atom(converted)
6219 if (
6220 lib.is_np_dtype(index.dtype, "iu")
6221 or needs_i8_conversion(index.dtype)
6222 or is_bool_dtype(index.dtype)
6223 ):
6224 # Includes Index, RangeIndex, DatetimeIndex, TimedeltaIndex, PeriodIndex,
6225 # in which case "kind" is "integer", "integer", "datetime64",
6226 # "timedelta64", and "integer", respectively.
6227 return IndexCol(
6228 name,
6229 values=converted,
6230 kind=kind,
6231 typ=atom,
6232 freq=getattr(index, "freq", None),
6233 tz=getattr(index, "tz", None),
6234 index_name=index_name,
6235 )
6237 if isinstance(index, MultiIndex):
6238 raise TypeError("MultiIndex not supported here!")
6240 inferred_type = lib.infer_dtype(index, skipna=False)
6241 # we won't get inferred_type of "datetime64" or "timedelta64" as these
6242 # would go through the DatetimeIndex/TimedeltaIndex paths above
6244 values = np.asarray(index)
6246 if inferred_type == "date":
6247 converted = np.asarray([v.toordinal() for v in values], dtype=np.int32)
6248 return IndexCol(
6249 name, converted, "date", _tables().Time32Col(), index_name=index_name
6250 )
6251 elif inferred_type == "string":
6252 # GH#9604: a genuine missing value and the literal string "nan" both
6253 # serialize to b"nan" via the fixed-width string cast, so they cannot
6254 # be told apart on read. Substitute the missing values with a sentinel
6255 # that does not collide with any real value and persist it (see
6256 # write_index / IndexCol.set_attr) so the reader can restore the NAs
6257 # while leaving a literal "nan" untouched.
6258 mask = np.asarray(index.isna())
6259 # Reuse the sentinel already persisted for this column (a ``table``
6260 # append) so it stays stable across chunks; only mint a fresh one when
6261 # none exists yet, avoiding every value already stored (existing_values)
6262 # as well as the ones in this chunk.
6263 nan_rep = existing_nan_rep
6264 if mask.any() and nan_rep is None:
6265 nan_rep = _make_nan_rep(values, mask, existing_values)
6266 if nan_rep is not None:
6267 if (values[~mask] == nan_rep).any():
6268 # A real value equal to the sentinel is indistinguishable from a
6269 # missing value on read; refuse rather than silently corrupt it.
6270 raise ValueError(
6271 f"Cannot store the string {str(nan_rep)!r} in Index column "
6272 f"[{name}]: it collides with the sentinel used to encode "
6273 "missing values in this column"
6274 )
6275 if mask.any():
6276 values = values.copy()
6277 values[mask] = nan_rep
6278 converted = _convert_string_array(values, encoding, errors)
6279 itemsize = converted.dtype.itemsize
6280 return IndexCol(
6281 name,
6282 converted,
6283 "string",
6284 _tables().StringCol(itemsize),
6285 index_name=index_name,
6286 nan_rep=nan_rep,
6287 )
6289 elif inferred_type in ["integer", "floating"]:
6290 return IndexCol(
6291 name, values=converted, kind=kind, typ=atom, index_name=index_name
6292 )
6293 else:
6294 assert isinstance(converted, np.ndarray) and converted.dtype == object
6295 assert kind == "object", kind
6296 atom = _tables().ObjectAtom()
6297 return IndexCol(name, converted, kind, atom, index_name=index_name)
6300def _unconvert_index(
6301 data, kind: str, encoding: str, errors: str, nan_rep=None
6302) -> np.ndarray | Index:
6303 index: Index | np.ndarray
6305 if kind.startswith("datetime64"):
6306 if kind == "datetime64":
6307 # created before we stored resolution information
6308 index = DatetimeIndex(data, copy=False)
6309 else:
6310 index = DatetimeIndex(data.view(kind), copy=False)
6311 elif kind.startswith("timedelta64"):
6312 if kind == "timedelta64":
6313 # created before we stored resolution information
6314 index = TimedeltaIndex(data, copy=False)
6315 else:
6316 index = TimedeltaIndex(data.view(kind), copy=False)
6317 elif kind == "date":
6318 try:
6319 index = np.asarray([date.fromordinal(v) for v in data], dtype=object)
6320 except ValueError:
6321 index = np.asarray([date.fromtimestamp(v) for v in data], dtype=object)
6322 elif kind in ("integer", "float", "bool"):
6323 index = np.asarray(data)
6324 elif kind in ("string"):
6325 index = _unconvert_string_array(
6326 data, nan_rep=nan_rep, encoding=encoding, errors=errors
6327 )
6328 elif kind == "object":
6329 index = np.asarray(data[0])
6330 else: # pragma: no cover
6331 raise ValueError(f"unrecognized index type {kind}")
6332 return index
6335def _maybe_convert_for_string_atom(
6336 name: str,
6337 bvalues: ArrayLike,
6338 existing_col,
6339 min_itemsize,
6340 nan_rep,
6341 encoding,
6342 errors,
6343 columns: list[str],
6344):
6345 if isinstance(bvalues.dtype, StringDtype):
6346 bvalues = bvalues.to_numpy()
6347 if bvalues.dtype != object:
6348 return bvalues
6350 bvalues = cast("np.ndarray", bvalues)
6352 dtype_name = bvalues.dtype.name
6353 inferred_type = lib.infer_dtype(bvalues, skipna=False)
6355 if inferred_type == "date":
6356 raise TypeError("[date] is not implemented as a table column")
6357 if inferred_type == "datetime":
6358 # after GH#8260
6359 # this only would be hit for a multi-timezone dtype which is an error
6360 raise TypeError(
6361 "too many timezones in this block, create separate data columns"
6362 )
6364 if not (inferred_type == "string" or dtype_name == "object"):
6365 return bvalues
6367 mask = isna(bvalues)
6368 data = bvalues.copy()
6369 data[mask] = nan_rep
6371 if existing_col and mask.any() and len(nan_rep) > existing_col.itemsize:
6372 raise ValueError(
6373 "NaN representation is too large for existing column size: "
6374 f"{str(nan_rep)!r} has length {len(nan_rep)} but this column has a "
6375 f"limit of {existing_col.itemsize}!\n"
6376 "It can be longer than the values themselves. Recreate the table "
6377 "passing min_itemsize to reserve room for it."
6378 )
6380 # see if we have a valid string type
6381 inferred_type = lib.infer_dtype(data, skipna=False)
6382 if inferred_type != "string":
6383 # we cannot serialize this data, so report an exception on a column
6384 # by column basis
6386 # expected behaviour:
6387 # search block for a non-string object column by column
6388 for i in range(data.shape[0]):
6389 col = data[i]
6390 inferred_type = lib.infer_dtype(col, skipna=False)
6391 if inferred_type != "string":
6392 error_column_label = columns[i] if len(columns) > i else f"No.{i}"
6393 raise TypeError(
6394 f"Cannot serialize the column [{error_column_label}]\n"
6395 f"because its data contents are not [string] but "
6396 f"[{inferred_type}] object dtype"
6397 )
6399 # itemsize is the maximum length of a string (along any dimension)
6401 data_converted = _convert_string_array(data, encoding, errors).reshape(data.shape)
6402 itemsize = data_converted.itemsize
6404 # specified min_itemsize?
6405 if isinstance(min_itemsize, dict):
6406 min_itemsize = int(min_itemsize.get(name) or min_itemsize.get("values") or 0)
6407 itemsize = max(min_itemsize or 0, itemsize)
6409 # check for column in the values conflicts
6410 if existing_col is not None:
6411 eci = existing_col.validate_col(itemsize)
6412 if eci is not None and eci > itemsize:
6413 itemsize = eci
6415 data_converted = data_converted.astype(f"|S{itemsize}", copy=False)
6416 return data_converted
6419def _convert_string_array(data: np.ndarray, encoding: str, errors: str) -> np.ndarray:
6420 """
6421 Take a string-like that is object dtype and coerce to a fixed size string type.
6423 Parameters
6424 ----------
6425 data : np.ndarray[object]
6426 encoding : str
6427 errors : str
6428 Handler for encoding errors.
6430 Returns
6431 -------
6432 np.ndarray[fixed-length-string]
6433 """
6434 # encode if needed
6435 if len(data):
6436 data = (
6437 Series(data.ravel(), copy=False, dtype="object")
6438 .str.encode(encoding, errors)
6439 ._values.reshape(data.shape)
6440 )
6442 # create the sized dtype
6443 ensured = ensure_object(data.ravel())
6444 itemsize = max(1, libwriters.max_len_string_array(ensured))
6446 data = np.asarray(data, dtype=f"S{itemsize}")
6447 return data
6450def _unconvert_string_array(
6451 data: np.ndarray, nan_rep, encoding: str, errors: str
6452) -> np.ndarray:
6453 """
6454 Inverse of _convert_string_array.
6456 Parameters
6457 ----------
6458 data : np.ndarray[fixed-length-string]
6459 nan_rep : the storage repr of NaN, or None to skip substitution.
6460 Pass None when the writer did not encode NaN as a sentinel string;
6461 otherwise legitimate occurrences of the sentinel value would be
6462 incorrectly replaced with NaN on read.
6463 encoding : str
6464 errors : str
6465 Handler for encoding errors.
6467 Returns
6468 -------
6469 np.ndarray[object]
6470 Decoded data.
6471 """
6472 shape = data.shape
6473 data = np.asarray(data.ravel(), dtype=object)
6475 if len(data):
6476 itemsize = libwriters.max_len_string_array(ensure_object(data))
6477 dtype = f"U{itemsize}"
6479 if isinstance(data[0], bytes):
6480 ser = Series(data, copy=False).str.decode(
6481 encoding, errors=errors, dtype="object"
6482 )
6483 data = ser.to_numpy()
6484 data.flags.writeable = True
6485 else:
6486 data = data.astype(dtype, copy=False).astype(object, copy=False)
6488 if nan_rep is not None:
6489 libwriters.string_array_replace_from_nan_rep(data, nan_rep)
6490 return data.reshape(shape)
6493def _maybe_convert(
6494 values: np.ndarray, val_kind: str, encoding: str, errors: str, nan_rep=None
6495):
6496 assert isinstance(val_kind, str), type(val_kind)
6497 if _need_convert(val_kind):
6498 conv = _get_converter(val_kind, encoding, errors, nan_rep)
6499 values = conv(values)
6500 return values
6503def _get_converter(kind: str, encoding: str, errors: str, nan_rep=None):
6504 if kind == "datetime64":
6505 return lambda x: np.asarray(x, dtype="M8[ns]")
6506 elif "datetime64" in kind:
6507 return lambda x: np.asarray(x, dtype=kind)
6508 elif kind == "string":
6509 return lambda x: _unconvert_string_array(
6510 x, nan_rep=nan_rep, encoding=encoding, errors=errors
6511 )
6512 else: # pragma: no cover
6513 raise ValueError(f"invalid kind {kind}")
6516def _need_convert(kind: str) -> bool:
6517 if kind in ("datetime64", "string") or "datetime64" in kind:
6518 return True
6519 return False
6522def _dtype_to_kind(dtype_str: str) -> str:
6523 """
6524 Find the "kind" string describing the given dtype name.
6525 """
6526 if dtype_str.startswith(("string", "bytes")):
6527 kind = "string"
6528 elif dtype_str.startswith("float"):
6529 kind = "float"
6530 elif dtype_str.startswith("complex"):
6531 kind = "complex"
6532 elif dtype_str.startswith(("int", "uint")):
6533 kind = "integer"
6534 elif dtype_str.startswith("datetime64"):
6535 kind = dtype_str
6536 elif dtype_str.startswith("timedelta"):
6537 kind = dtype_str
6538 elif dtype_str.startswith("bool"):
6539 kind = "bool"
6540 elif dtype_str.startswith("category"):
6541 kind = "category"
6542 elif dtype_str.startswith("period"):
6543 # We store the `freq` attr so we can restore from integers
6544 kind = "integer"
6545 elif dtype_str == "object":
6546 kind = "object"
6547 elif dtype_str == "str":
6548 kind = "str"
6549 else:
6550 raise ValueError(f"cannot interpret dtype of [{dtype_str}]")
6552 return kind
6555def _get_data_and_dtype_name(data: ArrayLike):
6556 """
6557 Convert the passed data into a storable form and a dtype string.
6558 """
6559 if isinstance(data, Categorical):
6560 data = data.codes
6562 if isinstance(data.dtype, DatetimeTZDtype):
6563 # For datetime64tz we need to drop the TZ in tests TODO: why?
6564 dtype_name = f"datetime64[{data.dtype.unit}]"
6565 else:
6566 dtype_name = data.dtype.name
6568 if data.dtype.kind in "mM":
6569 data = np.asarray(data.view("i8"))
6570 # TODO: we used to reshape for the dt64tz case, but no longer
6571 # doing that doesn't seem to break anything. why?
6573 elif isinstance(data, (PeriodIndex, PeriodArray)):
6574 data = data.asi8
6576 data = np.asarray(data)
6577 return data, dtype_name
6580def _or_of_ands_columns(condition) -> set[str]:
6581 """
6582 Look for an OR with at least one AND operand, e.g. ``(A & B) | (C & D)``
6583 or ``(A & B) | C``, in a pruned PyTables condition tree. The AND operand
6584 need not span two columns -- ``(A > 1 & A < 5) | B`` counts too.
6586 This is the query shape that can trigger an upstream PyTables bug where
6587 index-accelerated reads silently drop matching rows (GH#50598). Return the
6588 set of column names referenced by the AND-ed operands of such an OR, or an
6589 empty set if the pattern is not present.
6591 Only the AND-ed operands matter: the bug needs one of *their* columns to be
6592 indexed. Indexing a column that appears solely as a bare OR operand (the
6593 ``C`` of ``(A & B) | C``) does not trigger it.
6594 """
6595 cols: set[str] = set()
6597 def collect(node) -> None:
6598 """Add every column name referenced under ``node``."""
6599 if not isinstance(node, JointConditionBinOp):
6600 # leaf comparison: lhs.value is the column name
6601 cols.add(node.lhs.value)
6602 return
6603 collect(node.lhs)
6604 collect(node.rhs)
6606 def visit(node) -> None:
6607 if not isinstance(node, JointConditionBinOp):
6608 return
6609 if node.op == "|":
6610 for operand in (node.lhs, node.rhs):
6611 if isinstance(operand, JointConditionBinOp) and operand.op == "&":
6612 collect(operand)
6613 visit(node.lhs)
6614 visit(node.rhs)
6616 visit(condition)
6617 return cols
6620class Selection:
6621 """
6622 Carries out a selection operation on a tables.Table object.
6624 Parameters
6625 ----------
6626 table : a Table object
6627 where : list of Terms (or convertible to)
6628 start, stop: indices to start and/or stop selection
6630 """
6632 def __init__(
6633 self,
6634 table: Table,
6635 where=None,
6636 start: int | None = None,
6637 stop: int | None = None,
6638 ) -> None:
6639 self.table = table
6640 self.where = where
6641 self.start = start
6642 self.stop = stop
6643 self.condition = None
6644 self.filter = None
6645 self.terms = None
6646 self.coordinates = None
6648 if is_list_like(where):
6649 # see if we have a passed coordinate like
6650 inferred = lib.infer_dtype(where, skipna=False)
6651 if inferred in ("integer", "boolean"):
6652 where = np.asarray(where)
6653 # start/stop are None on the read_coordinates paths, which is how
6654 # an out-of-range coordinate used to reach PyTables unchecked
6655 start, stop = _resolve_row_window(
6656 self.start, self.stop, self.table.nrows
6657 )
6658 if where.dtype == np.bool_:
6659 self.coordinates = np.arange(start, stop)[where]
6660 elif issubclass(where.dtype.type, np.integer):
6661 invalid = where[(where < start) | (where >= stop)]
6662 if len(invalid):
6663 raise ValueError(
6664 "where must have index locations >= start and < stop, "
6665 f"but got {invalid[:5].tolist()}"
6666 f"{' ...' if len(invalid) > 5 else ''} "
6667 f"with start={start} and stop={stop}"
6668 )
6669 self.coordinates = where
6671 if self.coordinates is None:
6672 self.terms = self.generate(where)
6674 # create the numexpr & the filter
6675 if self.terms is not None:
6676 self.condition, self.filter = self.terms.evaluate()
6678 def _warn_if_unreliable_or(self) -> None:
6679 """
6680 Warn for nested-OR queries with AND-ed operands over indexed columns
6681 (e.g. ``(A & B) | (C & D)``) that can return incorrect results due to
6682 an upstream PyTables bug (GH#50598).
6684 Called from the query methods rather than ``__init__``: one read builds
6685 several ``Selection`` objects for the same ``where``, and only the one
6686 that runs the condition should warn.
6687 """
6688 table = getattr(self.table, "table", None)
6689 if table is None:
6690 return
6691 cols = _or_of_ands_columns(self.condition)
6692 if not any(
6693 table.colinstances[name].is_indexed
6694 for name in cols
6695 if name in table.colnames
6696 ):
6697 return
6698 warnings.warn(
6699 "Querying with a nested 'where' that ORs AND-ed conditions over "
6700 "indexed columns (e.g. '(A & B) | (C & D)') can silently match the "
6701 "wrong rows due to an upstream PyTables bug (GH#50598). To get "
6702 "correct results, write the table with 'index=False', or split the "
6703 "'where' into one operation per OR branch.",
6704 UserWarning,
6705 stacklevel=find_stack_level(),
6706 )
6708 @overload
6709 def generate(self, where: dict | list | tuple | str) -> PyTablesExpr: ...
6711 @overload
6712 def generate(self, where: None) -> None: ...
6714 def generate(self, where: dict | list | tuple | str | None) -> PyTablesExpr | None:
6715 """where can be a : dict,list,tuple,string"""
6716 if where is None:
6717 return None
6719 q = self.table.queryables()
6720 try:
6721 return PyTablesExpr(where, queryables=q, encoding=self.table.encoding)
6722 except NameError as err:
6723 # raise a nice message, suggesting that the user should use
6724 # data_columns
6725 qkeys = ",".join(q.keys())
6726 msg = dedent(
6727 f"""\
6728 The passed where expression: {where}
6729 contains an invalid variable reference
6730 all of the variable references must be a reference to
6731 an axis (e.g. 'index' or 'columns'), or a data_column
6732 The currently defined references are: {qkeys}
6733 """
6734 )
6735 raise ValueError(msg) from err
6737 def select(self):
6738 """
6739 generate the selection
6740 """
6741 if self.condition is not None:
6742 self._warn_if_unreliable_or()
6743 try:
6744 return self.table.table.read_where(
6745 self.condition.format(), start=self.start, stop=self.stop
6746 )
6747 except ValueError as err:
6748 # GH#39752 PyTables queries indexed columns by combining one
6749 # boolean array per comparison term in a single numexpr call,
6750 # capped at NPY_MAXARGS-1 inputs (a numexpr build-time
6751 # constant, 32 or 64 depending on the targeted numpy).
6752 # Translate the opaque numexpr error into actionable guidance.
6753 if "too many inputs" not in str(err):
6754 raise
6755 raise ValueError(
6756 "The passed where expression has too many comparisons "
6757 "for a query against indexed columns (a numexpr "
6758 "limitation). Reduce the number of comparisons, or store "
6759 "the table with 'index=False' (e.g. "
6760 "DataFrame.to_hdf(..., index=False)) so the query does "
6761 "not use the column index."
6762 ) from err
6763 elif self.coordinates is not None:
6764 return self.table.table.read_coordinates(self.coordinates)
6765 return self.table.table.read(start=self.start, stop=self.stop)
6767 def select_coords(self):
6768 """
6769 generate the selection
6770 """
6771 start, stop = _resolve_row_window(self.start, self.stop, self.table.nrows)
6773 if self.condition is not None:
6774 self._warn_if_unreliable_or()
6775 coords = self.table.table.get_where_list(
6776 self.condition.format(), start=start, stop=stop, sort=True
6777 )
6778 elif self.coordinates is not None:
6779 return self.coordinates
6780 else:
6781 coords = np.arange(start, stop)
6783 if self.filter is not None and len(coords):
6784 # a term the numexpr condition can't carry -- e.g. an "index in
6785 # [...]" clause with more selectors than numexpr can handle -- is
6786 # realized as a post-read filter (GH#17567). A where clause may
6787 # combine such a filter with a numexpr condition (e.g. "index in
6788 # [...] and A>=1"), so it must be applied on top of get_where_list's
6789 # coordinates here -- this also keeps the row-coordinate path
6790 # (iterator / chunksize) dropping the same rows as a plain read
6791 # (GH#12953).
6792 for field, op, filt in self.filter.format():
6793 data = self.table.read_column(
6794 field, start=coords.min(), stop=coords.max() + 1
6795 )
6796 coords = coords[op(data.iloc[coords - coords.min()], filt).values]
6797 return coords