Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.11/site-packages/pandas/io/pytables.py: 19%
Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2High level interface to PyTables for reading and writing pandas data structures
3to disk
4"""
6from __future__ import annotations
8from contextlib import suppress
9import copy
10from datetime import (
11 date,
12 tzinfo,
13)
14import itertools
15import os
16import re
17from textwrap import dedent
18from typing import (
19 TYPE_CHECKING,
20 Any,
21 Final,
22 Literal,
23 Self,
24 TypeAlias,
25 cast,
26 overload,
27)
28import warnings
30import numpy as np
32from pandas._config import (
33 config,
34 get_option,
35 using_string_dtype,
36)
38from pandas._libs import (
39 lib,
40 writers as libwriters,
41)
42from pandas._libs.lib import is_string_array
43from pandas._libs.tslibs import timezones
44from pandas.compat import HAS_PYARROW
45from pandas.compat._optional import import_optional_dependency
46from pandas.compat.pickle_compat import patch_pickle
47from pandas.errors import (
48 AttributeConflictWarning,
49 ClosedFileError,
50 IncompatibilityWarning,
51 PerformanceWarning,
52 PossibleDataLossError,
53)
54from pandas.util._decorators import (
55 cache_readonly,
56 set_module,
57)
58from pandas.util._exceptions import find_stack_level
60from pandas.core.dtypes.common import (
61 ensure_object,
62 is_bool_dtype,
63 is_complex_dtype,
64 is_list_like,
65 is_string_dtype,
66 needs_i8_conversion,
67)
68from pandas.core.dtypes.dtypes import (
69 CategoricalDtype,
70 DatetimeTZDtype,
71 ExtensionDtype,
72 PeriodDtype,
73)
74from pandas.core.dtypes.missing import array_equivalent
76from pandas import (
77 DataFrame,
78 DatetimeIndex,
79 Index,
80 MultiIndex,
81 PeriodIndex,
82 RangeIndex,
83 Series,
84 StringDtype,
85 TimedeltaIndex,
86 concat,
87 isna,
88)
89from pandas.core.arrays import (
90 Categorical,
91 DatetimeArray,
92 PeriodArray,
93)
94from pandas.core.arrays.datetimes import tz_to_dtype
95from pandas.core.arrays.string_ import BaseStringArray
96import pandas.core.common as com
97from pandas.core.computation.pytables import (
98 PyTablesExpr,
99 maybe_expression,
100)
101from pandas.core.construction import (
102 array as pd_array,
103 extract_array,
104)
105from pandas.core.indexes.api import ensure_index
107from pandas.io.common import stringify_path
108from pandas.io.formats.printing import (
109 adjoin,
110 pprint_thing,
111)
113if TYPE_CHECKING:
114 from collections.abc import (
115 Callable,
116 Hashable,
117 Iterator,
118 Sequence,
119 )
120 from types import (
121 ModuleType,
122 TracebackType,
123 )
125 from tables import (
126 Col,
127 File,
128 Node,
129 )
131 from pandas._typing import (
132 AnyArrayLike,
133 ArrayLike,
134 AxisInt,
135 DtypeArg,
136 FilePath,
137 TimeUnit,
138 npt,
139 )
141 from pandas.core.internals import Block
143# versioning attribute
144_version = "0.15.2"
146# encoding
147_default_encoding = "UTF-8"
150def _ensure_encoding(encoding: str | None) -> str:
151 # set the encoding if we need
152 if encoding is None:
153 encoding = _default_encoding
155 return encoding
158def _ensure_str(name):
159 """
160 Ensure that an index / column name is a str (python 3); otherwise they
161 may be np.string dtype. Non-string dtypes are passed through unchanged.
163 https://github.com/pandas-dev/pandas/issues/13492
164 """
165 if isinstance(name, str):
166 name = str(name)
167 return name
170Term: TypeAlias = PyTablesExpr
173def _ensure_term(where, scope_level: int):
174 """
175 Ensure that the where is a Term or a list of Term.
177 This makes sure that we are capturing the scope of variables that are
178 passed create the terms here with a frame_level=2 (we are 2 levels down)
179 """
180 # only consider list/tuple here as an ndarray is automatically a coordinate
181 # list
182 level = scope_level + 1
183 if isinstance(where, (list, tuple)):
184 where = [
185 Term(term, scope_level=level + 1) if maybe_expression(term) else term
186 for term in where
187 if term is not None
188 ]
189 elif maybe_expression(where):
190 where = Term(where, scope_level=level)
191 return where if where is None or len(where) else None
194incompatibility_doc: Final = """
195where criteria is being ignored as this version [%s] is too old (or
196not-defined), read the file in and write it out to a new file to upgrade (with
197the copy_to method)
198"""
200attribute_conflict_doc: Final = """
201the [%s] attribute of the existing index is [%s] which conflicts with the new
202[%s], resetting the attribute to None
203"""
205performance_doc: Final = """
206your performance may suffer as PyTables will pickle object types that it cannot
207map directly to c-types [inferred_type->%s,key->%s] [items->%s]
208"""
210# formats
211_FORMAT_MAP = {"f": "fixed", "fixed": "fixed", "t": "table", "table": "table"}
213# axes map
214_AXES_MAP = {DataFrame: [0]}
216# register our configuration options
217dropna_doc: Final = """
218: boolean
219 drop ALL nan rows when appending to a table
220"""
221format_doc: Final = """
222: format
223 default format writing format, if None, then
224 put will default to 'fixed' and append will default to 'table'
225"""
227with config.config_prefix("io.hdf"):
228 config.register_option("dropna_table", False, dropna_doc, validator=config.is_bool)
229 config.register_option(
230 "default_format",
231 None,
232 format_doc,
233 validator=config.is_one_of_factory(["fixed", "table", None]),
234 )
236# oh the troubles to reduce import time
237_table_mod: ModuleType | None = None
238_table_file_open_policy_is_strict = False
241def _tables():
242 global _table_mod
243 global _table_file_open_policy_is_strict
244 if _table_mod is None:
245 import tables
247 _table_mod = tables
249 # set the file open policy
250 # return the file open policy; this changes as of pytables 3.1
251 # depending on the HDF5 version
252 with suppress(AttributeError):
253 _table_file_open_policy_is_strict = (
254 tables.file._FILE_OPEN_POLICY == "strict"
255 )
257 return _table_mod
260# interface to/from ###
263def to_hdf(
264 path_or_buf: FilePath | HDFStore,
265 key: str,
266 value: DataFrame | Series,
267 mode: str = "a",
268 complevel: int | None = None,
269 complib: str | None = None,
270 append: bool = False,
271 format: str | None = None,
272 index: bool = True,
273 min_itemsize: int | dict[str, int] | None = None,
274 nan_rep=None,
275 dropna: bool | None = None,
276 data_columns: Literal[True] | list[str] | None = None,
277 errors: str = "strict",
278 encoding: str = "UTF-8",
279) -> None:
280 """store this object, close it if we opened it"""
281 if append:
282 f = lambda store: store.append(
283 key,
284 value,
285 format=format,
286 index=index,
287 min_itemsize=min_itemsize,
288 nan_rep=nan_rep,
289 dropna=dropna,
290 data_columns=data_columns,
291 errors=errors,
292 encoding=encoding,
293 )
294 else:
295 # NB: dropna is not passed to `put`
296 f = lambda store: store.put(
297 key,
298 value,
299 format=format,
300 index=index,
301 min_itemsize=min_itemsize,
302 nan_rep=nan_rep,
303 data_columns=data_columns,
304 errors=errors,
305 encoding=encoding,
306 dropna=dropna,
307 )
309 if isinstance(path_or_buf, HDFStore):
310 f(path_or_buf)
311 else:
312 path_or_buf = stringify_path(path_or_buf)
313 with HDFStore(
314 path_or_buf, mode=mode, complevel=complevel, complib=complib
315 ) as store:
316 f(store)
319@set_module("pandas")
320def read_hdf(
321 path_or_buf: FilePath | HDFStore,
322 key=None,
323 mode: str = "r",
324 errors: str = "strict",
325 where: str | list | None = None,
326 start: int | None = None,
327 stop: int | None = None,
328 columns: list[str] | None = None,
329 iterator: bool = False,
330 chunksize: int | None = None,
331 **kwargs,
332):
333 """
334 Read from the store, close it if we opened it.
336 Retrieve pandas object stored in file, optionally based on where
337 criteria.
339 .. warning::
341 Pandas uses PyTables for reading and writing HDF5 files, which allows
342 serializing object-dtype data with pickle when using the "fixed" format.
343 Loading pickled data received from untrusted sources can be unsafe.
345 See: https://docs.python.org/3/library/pickle.html for more.
347 Parameters
348 ----------
349 path_or_buf : str, path object, pandas.HDFStore
350 Any valid string path is acceptable. Only supports the local file system,
351 remote URLs and file-like objects are not supported.
353 If you want to pass in a path object, pandas accepts any
354 ``os.PathLike``.
356 Alternatively, pandas accepts an open :class:`pandas.HDFStore` object.
358 key : object, optional
359 The group identifier in the store. Can be omitted if the HDF file
360 contains a single pandas object.
361 mode : {'r', 'r+', 'a'}, default 'r'
362 Mode to use when opening the file. Ignored if path_or_buf is a
363 :class:`pandas.HDFStore`. Default is 'r'.
364 errors : str, default 'strict'
365 Specifies how encoding and decoding errors are to be handled.
366 See the errors argument for :func:`open` for a full list
367 of options.
368 where : list, optional
369 A list of Term (or convertible) objects.
370 start : int, optional
371 Row number to start selection.
372 stop : int, optional
373 Row number to stop selection.
374 columns : list, optional
375 A list of columns names to return.
376 iterator : bool, optional
377 Return an iterator object.
378 chunksize : int, optional
379 Number of rows to include in an iteration when using an iterator.
380 **kwargs
381 Additional keyword arguments passed to HDFStore.
383 Returns
384 -------
385 object
386 The selected object. Return type depends on the object stored.
388 See Also
389 --------
390 DataFrame.to_hdf : Write an HDF file from a DataFrame.
391 HDFStore : Low-level access to HDF files.
393 Notes
394 -----
395 When ``errors="surrogatepass"``, ``pd.options.future.infer_string`` is true,
396 and PyArrow is installed, if a UTF-16 surrogate is encountered when decoding
397 to UTF-8, the resulting dtype will be
398 ``pd.StringDtype(storage="python", na_value=np.nan)``.
400 Examples
401 --------
402 >>> df = pd.DataFrame([[1, 1.0, "a"]], columns=["x", "y", "z"]) # doctest: +SKIP
403 >>> df.to_hdf("./store.h5", "data") # doctest: +SKIP
404 >>> reread = pd.read_hdf("./store.h5") # doctest: +SKIP
405 """
406 if mode not in ["r", "r+", "a"]:
407 raise ValueError(
408 f"mode {mode} is not allowed while performing a read. "
409 f"Allowed modes are r, r+ and a."
410 )
411 # grab the scope
412 if where is not None:
413 where = _ensure_term(where, scope_level=1)
415 if isinstance(path_or_buf, HDFStore):
416 if not path_or_buf.is_open:
417 raise OSError("The HDFStore must be open for reading.")
419 store = path_or_buf
420 auto_close = False
421 else:
422 path_or_buf = stringify_path(path_or_buf)
423 if not isinstance(path_or_buf, str):
424 raise NotImplementedError(
425 "Support for generic buffers has not been implemented."
426 )
427 try:
428 exists = os.path.exists(path_or_buf)
430 # if filepath is too long
431 except (TypeError, ValueError):
432 exists = False
434 if not exists:
435 raise FileNotFoundError(f"File {path_or_buf} does not exist")
437 store = HDFStore(path_or_buf, mode=mode, errors=errors, **kwargs)
438 # can't auto open/close if we are using an iterator
439 # so delegate to the iterator
440 auto_close = True
442 try:
443 if key is None:
444 groups = store.groups()
445 if len(groups) == 0:
446 raise ValueError(
447 "Dataset(s) incompatible with Pandas data types, "
448 "not table, or no datasets found in HDF5 file."
449 )
450 candidate_only_group = groups[0]
452 # For the HDF file to have only one dataset, all other groups
453 # should then be metadata groups for that candidate group. (This
454 # assumes that the groups() method enumerates parent groups
455 # before their children.)
456 for group_to_check in groups[1:]:
457 if not _is_metadata_of(group_to_check, candidate_only_group):
458 raise ValueError(
459 "key must be provided when HDF5 "
460 "file contains multiple datasets."
461 )
462 key = candidate_only_group._v_pathname
463 return store.select(
464 key,
465 where=where,
466 start=start,
467 stop=stop,
468 columns=columns,
469 iterator=iterator,
470 chunksize=chunksize,
471 auto_close=auto_close,
472 )
473 except (ValueError, TypeError, LookupError):
474 if not isinstance(path_or_buf, HDFStore):
475 # if there is an error, close the store if we opened it.
476 with suppress(AttributeError):
477 store.close()
479 raise
482def _is_metadata_of(group: Node, parent_group: Node) -> bool:
483 """Check if a given group is a metadata group for a given parent_group."""
484 if group._v_depth <= parent_group._v_depth:
485 return False
487 current = group
488 while current._v_depth > 1:
489 parent = current._v_parent
490 if parent == parent_group and current._v_name == "meta":
491 return True
492 current = current._v_parent
493 return False
496@set_module("pandas")
497class HDFStore:
498 """
499 Dict-like IO interface for storing pandas objects in PyTables.
501 Either Fixed or Table format.
503 .. warning::
505 Pandas uses PyTables for reading and writing HDF5 files, which allows
506 serializing object-dtype data with pickle when using the "fixed" format.
507 Loading pickled data received from untrusted sources can be unsafe.
509 See: https://docs.python.org/3/library/pickle.html for more.
511 Parameters
512 ----------
513 path : str
514 File path to HDF5 file.
515 mode : {'a', 'w', 'r', 'r+'}, default 'a'
517 ``'r'``
518 Read-only; no data can be modified.
519 ``'w'``
520 Write; a new file is created (an existing file with the same
521 name would be deleted).
522 ``'a'``
523 Append; an existing file is opened for reading and writing,
524 and if the file does not exist it is created.
525 ``'r+'``
526 It is similar to ``'a'``, but the file must already exist.
527 complevel : int, 0-9, default None
528 Specifies a compression level for data.
529 A value of 0 or None disables compression.
530 complib : {'zlib', 'lzo', 'bzip2', 'blosc'}, default 'zlib'
531 Specifies the compression library to be used.
532 These additional compressors for Blosc are supported
533 (default if no compressor specified: 'blosc:blosclz'):
534 {'blosc:blosclz', 'blosc:lz4', 'blosc:lz4hc', 'blosc:snappy',
535 'blosc:zlib', 'blosc:zstd'}.
536 Specifying a compression library which is not available issues
537 a ValueError.
538 fletcher32 : bool, default False
539 If applying compression use the fletcher32 checksum.
540 **kwargs
541 These parameters will be passed to the PyTables open_file method.
543 Examples
544 --------
545 >>> bar = pd.DataFrame(np.random.randn(10, 4))
546 >>> store = pd.HDFStore("test.h5")
547 >>> store["foo"] = bar # write to HDF5
548 >>> bar = store["foo"] # retrieve
549 >>> store.close()
551 **Create or load HDF5 file in-memory**
553 When passing the `driver` option to the PyTables open_file method through
554 **kwargs, the HDF5 file is loaded or created in-memory and will only be
555 written when closed:
557 >>> bar = pd.DataFrame(np.random.randn(10, 4))
558 >>> store = pd.HDFStore("test.h5", driver="H5FD_CORE")
559 >>> store["foo"] = bar
560 >>> store.close() # only now, data is written to disk
561 """
563 _handle: File | None
564 _mode: str
566 def __init__(
567 self,
568 path,
569 mode: str = "a",
570 complevel: int | None = None,
571 complib=None,
572 fletcher32: bool = False,
573 **kwargs,
574 ) -> None:
575 if "format" in kwargs:
576 raise ValueError("format is not a defined argument for HDFStore")
578 tables = import_optional_dependency("tables")
580 if complib is not None and complib not in tables.filters.all_complibs:
581 raise ValueError(
582 f"complib only supports {tables.filters.all_complibs} compression."
583 )
585 if complib is None and complevel is not None:
586 complib = tables.filters.default_complib
588 self._path = stringify_path(path)
589 if mode is None:
590 mode = "a"
591 self._mode = mode
592 self._handle = None
593 self._complevel = complevel if complevel else 0
594 self._complib = complib
595 self._fletcher32 = fletcher32
596 self._filters = None
597 self.open(mode=mode, **kwargs)
599 def __fspath__(self) -> str:
600 return self._path
602 @property
603 def root(self):
604 """return the root node"""
605 self._check_if_open()
606 assert self._handle is not None # for mypy
607 return self._handle.root
609 @property
610 def filename(self) -> str:
611 return self._path
613 def __getitem__(self, key: str):
614 return self.get(key)
616 def __setitem__(self, key: str, value) -> None:
617 self.put(key, value)
619 def __delitem__(self, key: str) -> int | None:
620 return self.remove(key)
622 def __getattr__(self, name: str):
623 """allow attribute access to get stores"""
624 try:
625 return self.get(name)
626 except (KeyError, ClosedFileError):
627 pass
628 raise AttributeError(
629 f"'{type(self).__name__}' object has no attribute '{name}'"
630 )
632 def __contains__(self, key: str) -> bool:
633 """
634 check for existence of this key
635 can match the exact pathname or the pathnm w/o the leading '/'
636 """
637 node = self.get_node(key)
638 if node is not None:
639 name = node._v_pathname
640 if key in (name, name[1:]):
641 return True
642 return False
644 def __len__(self) -> int:
645 return len(self.groups())
647 def __repr__(self) -> str:
648 pstr = pprint_thing(self._path)
649 return f"{type(self)}\nFile path: {pstr}\n"
651 def __enter__(self) -> Self:
652 return self
654 def __exit__(
655 self,
656 exc_type: type[BaseException] | None,
657 exc_value: BaseException | None,
658 traceback: TracebackType | None,
659 ) -> None:
660 self.close()
662 def keys(self, include: str = "pandas") -> list[str]:
663 """
664 Return a list of keys corresponding to objects stored in HDFStore.
666 Parameters
667 ----------
669 include : str, default 'pandas'
670 When kind equals 'pandas' return pandas objects.
671 When kind equals 'native' return native HDF5 Table objects.
673 Returns
674 -------
675 list
676 List of ABSOLUTE path-names (e.g. have the leading '/').
678 Raises
679 ------
680 raises ValueError if kind has an illegal value
682 See Also
683 --------
684 HDFStore.info : Prints detailed information on the store.
685 HDFStore.get_node : Returns the node with the key.
686 HDFStore.get_storer : Returns the storer object for a key.
688 Examples
689 --------
690 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
691 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
692 >>> store.put("data", df) # doctest: +SKIP
693 >>> store.get("data") # doctest: +SKIP
694 >>> print(store.keys()) # doctest: +SKIP
695 ['/data1', '/data2']
696 >>> store.close() # doctest: +SKIP
697 """
698 if include == "pandas":
699 return [n._v_pathname for n in self.groups()]
701 elif include == "native":
702 assert self._handle is not None # mypy
703 return [
704 n._v_pathname for n in self._handle.walk_nodes("/", classname="Table")
705 ]
706 raise ValueError(
707 f"`include` should be either 'pandas' or 'native' but is '{include}'"
708 )
710 def __iter__(self) -> Iterator[str]:
711 return iter(self.keys())
713 def items(self) -> Iterator[tuple[str, list]]:
714 """
715 iterate on key->group
716 """
717 for g in self.groups():
718 yield g._v_pathname, g
720 def open(self, mode: str = "a", **kwargs) -> None:
721 """
722 Open the file in the specified mode
724 Parameters
725 ----------
726 mode : {'a', 'w', 'r', 'r+'}, default 'a'
727 See HDFStore docstring or tables.open_file for info about modes
728 **kwargs
729 These parameters will be passed to the PyTables open_file method.
730 """
731 tables = _tables()
733 if self._mode != mode:
734 # if we are changing a write mode to read, ok
735 if self._mode in ["a", "w"] and mode in ["r", "r+"]:
736 pass
737 elif mode in ["w"]:
738 # this would truncate, raise here
739 if self.is_open:
740 raise PossibleDataLossError(
741 f"Re-opening the file [{self._path}] with mode [{self._mode}] "
742 "will delete the current file!"
743 )
745 self._mode = mode
747 # close and reopen the handle
748 if self.is_open:
749 self.close()
751 if self._complevel and self._complevel > 0:
752 self._filters = _tables().Filters(
753 self._complevel, self._complib, fletcher32=self._fletcher32
754 )
756 if _table_file_open_policy_is_strict and self.is_open:
757 msg = (
758 "Cannot open HDF5 file, which is already opened, "
759 "even in read-only mode."
760 )
761 raise ValueError(msg)
763 self._handle = tables.open_file(self._path, self._mode, **kwargs)
765 def close(self) -> None:
766 """
767 Close the PyTables file handle
768 """
769 if self._handle is not None:
770 self._handle.close()
771 self._handle = None
773 @property
774 def is_open(self) -> bool:
775 """
776 return a boolean indicating whether the file is open
777 """
778 if self._handle is None:
779 return False
780 return bool(self._handle.isopen)
782 def flush(self, fsync: bool = False) -> None:
783 """
784 Force all buffered modifications to be written to disk.
786 Parameters
787 ----------
788 fsync : bool (default False)
789 call ``os.fsync()`` on the file handle to force writing to disk.
791 Notes
792 -----
793 Without ``fsync=True``, flushing may not guarantee that the OS writes
794 to disk. With fsync, the operation will block until the OS claims the
795 file has been written; however, other caching layers may still
796 interfere.
797 """
798 if self._handle is not None:
799 self._handle.flush()
800 if fsync:
801 with suppress(OSError):
802 os.fsync(self._handle.fileno())
804 def get(self, key: str):
805 """
806 Retrieve pandas object stored in file.
808 Parameters
809 ----------
810 key : str
811 Object to retrieve from file. Raises KeyError if not found.
813 Returns
814 -------
815 object
816 Same type as object stored in file.
818 See Also
819 --------
820 HDFStore.get_node : Returns the node with the key.
821 HDFStore.get_storer : Returns the storer object for a key.
823 Examples
824 --------
825 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
826 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
827 >>> store.put("data", df) # doctest: +SKIP
828 >>> store.get("data") # doctest: +SKIP
829 >>> store.close() # doctest: +SKIP
830 """
831 with patch_pickle():
832 # GH#31167 Without this patch, pickle doesn't know how to unpickle
833 # old DateOffset objects now that they are cdef classes.
834 group = self.get_node(key)
835 if group is None:
836 raise KeyError(f"No object named {key} in the file")
837 return self._read_group(group)
839 def select(
840 self,
841 key: str,
842 where=None,
843 start=None,
844 stop=None,
845 columns=None,
846 iterator: bool = False,
847 chunksize: int | None = None,
848 auto_close: bool = False,
849 ):
850 """
851 Retrieve pandas object stored in file, optionally based on where criteria.
853 .. warning::
855 Pandas uses PyTables for reading and writing HDF5 files, which allows
856 serializing object-dtype data with pickle when using the "fixed" format.
857 Loading pickled data received from untrusted sources can be unsafe.
859 See: https://docs.python.org/3/library/pickle.html for more.
861 Parameters
862 ----------
863 key : str
864 Object being retrieved from file.
865 where : list or None
866 List of Term (or convertible) objects, optional.
867 start : int or None
868 Row number to start selection.
869 stop : int, default None
870 Row number to stop selection.
871 columns : list or None
872 A list of columns that if not None, will limit the return columns.
873 iterator : bool or False
874 Returns an iterator.
875 chunksize : int or None
876 Number or rows to include in iteration, return an iterator.
877 auto_close : bool or False
878 Should automatically close the store when finished.
880 Returns
881 -------
882 object
883 Retrieved object from file.
885 See Also
886 --------
887 HDFStore.select_as_coordinates : Returns the selection as an index.
888 HDFStore.select_column : Returns a single column from the table.
889 HDFStore.select_as_multiple : Retrieves pandas objects from multiple tables.
891 Examples
892 --------
893 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
894 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
895 >>> store.put("data", df) # doctest: +SKIP
896 >>> store.get("data") # doctest: +SKIP
897 >>> print(store.keys()) # doctest: +SKIP
898 ['/data1', '/data2']
899 >>> store.select("/data1") # doctest: +SKIP
900 A B
901 0 1 2
902 1 3 4
903 >>> store.select("/data1", where="columns == A") # doctest: +SKIP
904 A
905 0 1
906 1 3
907 >>> store.close() # doctest: +SKIP
908 """
909 group = self.get_node(key)
910 if group is None:
911 raise KeyError(f"No object named {key} in the file")
913 # create the storer and axes
914 where = _ensure_term(where, scope_level=1)
915 s = self._create_storer(group)
916 s.infer_axes()
918 # function to call on iteration
919 def func(_start, _stop, _where):
920 return s.read(start=_start, stop=_stop, where=_where, columns=columns)
922 # create the iterator
923 it = TableIterator(
924 self,
925 s,
926 func,
927 where=where,
928 nrows=s.nrows,
929 start=start,
930 stop=stop,
931 iterator=iterator,
932 chunksize=chunksize,
933 auto_close=auto_close,
934 )
936 return it.get_result()
938 def select_as_coordinates(
939 self,
940 key: str,
941 where=None,
942 start: int | None = None,
943 stop: int | None = None,
944 ):
945 """
946 return the selection as an Index
948 .. warning::
950 Pandas uses PyTables for reading and writing HDF5 files, which allows
951 serializing object-dtype data with pickle when using the "fixed" format.
952 Loading pickled data received from untrusted sources can be unsafe.
954 See: https://docs.python.org/3/library/pickle.html for more.
957 Parameters
958 ----------
959 key : str
960 where : list of Term (or convertible) objects, optional
961 start : integer (defaults to None), row number to start selection
962 stop : integer (defaults to None), row number to stop selection
963 """
964 where = _ensure_term(where, scope_level=1)
965 tbl = self.get_storer(key)
966 if not isinstance(tbl, Table):
967 raise TypeError("can only read_coordinates with a table")
968 return tbl.read_coordinates(where=where, start=start, stop=stop)
970 def select_column(
971 self,
972 key: str,
973 column: str,
974 start: int | None = None,
975 stop: int | None = None,
976 ):
977 """
978 return a single column from the table. This is generally only useful to
979 select an indexable
981 .. warning::
983 Pandas uses PyTables for reading and writing HDF5 files, which allows
984 serializing object-dtype data with pickle when using the "fixed" format.
985 Loading pickled data received from untrusted sources can be unsafe.
987 See: https://docs.python.org/3/library/pickle.html for more.
989 Parameters
990 ----------
991 key : str
992 column : str
993 The column of interest.
994 start : int or None, default None
995 stop : int or None, default None
997 Raises
998 ------
999 raises KeyError if the column is not found (or key is not a valid
1000 store)
1001 raises ValueError if the column can not be extracted individually (it
1002 is part of a data block)
1004 """
1005 tbl = self.get_storer(key)
1006 if not isinstance(tbl, Table):
1007 raise TypeError("can only read_column with a table")
1008 return tbl.read_column(column=column, start=start, stop=stop)
1010 def select_as_multiple(
1011 self,
1012 keys,
1013 where=None,
1014 selector=None,
1015 columns=None,
1016 start=None,
1017 stop=None,
1018 iterator: bool = False,
1019 chunksize: int | None = None,
1020 auto_close: bool = False,
1021 ):
1022 """
1023 Retrieve pandas objects from multiple tables.
1025 .. warning::
1027 Pandas uses PyTables for reading and writing HDF5 files, which allows
1028 serializing object-dtype data with pickle when using the "fixed" format.
1029 Loading pickled data received from untrusted sources can be unsafe.
1031 See: https://docs.python.org/3/library/pickle.html for more.
1033 Parameters
1034 ----------
1035 keys : a list of the tables
1036 selector : the table to apply the where criteria (defaults to keys[0]
1037 if not supplied)
1038 columns : the columns I want back
1039 start : integer (defaults to None), row number to start selection
1040 stop : integer (defaults to None), row number to stop selection
1041 iterator : bool, return an iterator, default False
1042 chunksize : nrows to include in iteration, return an iterator
1043 auto_close : bool, default False
1044 Should automatically close the store when finished.
1046 Raises
1047 ------
1048 raises KeyError if keys or selector is not found or keys is empty
1049 raises TypeError if keys is not a list or tuple
1050 raises ValueError if the tables are not ALL THE SAME DIMENSIONS
1051 """
1052 # default to single select
1053 where = _ensure_term(where, scope_level=1)
1054 if isinstance(keys, (list, tuple)) and len(keys) == 1:
1055 keys = keys[0]
1056 if isinstance(keys, str):
1057 return self.select(
1058 key=keys,
1059 where=where,
1060 columns=columns,
1061 start=start,
1062 stop=stop,
1063 iterator=iterator,
1064 chunksize=chunksize,
1065 auto_close=auto_close,
1066 )
1068 if not isinstance(keys, (list, tuple)):
1069 raise TypeError("keys must be a list/tuple")
1071 if not len(keys):
1072 raise ValueError("keys must have a non-zero length")
1074 if selector is None:
1075 selector = keys[0]
1077 # collect the tables
1078 tbls = [self.get_storer(k) for k in keys]
1079 s = self.get_storer(selector)
1081 # validate rows
1082 nrows = None
1083 for t, k in itertools.chain([(s, selector)], zip(tbls, keys, strict=True)):
1084 if t is None:
1085 raise KeyError(f"Invalid table [{k}]")
1086 if not t.is_table:
1087 raise TypeError(
1088 f"object [{t.pathname}] is not a table, and cannot be used in all "
1089 "select as multiple"
1090 )
1092 if nrows is None:
1093 nrows = t.nrows
1094 elif t.nrows != nrows:
1095 raise ValueError("all tables must have exactly the same nrows!")
1097 # The isinstance checks here are redundant with the check above,
1098 # but necessary for mypy; see GH#29757
1099 _tbls = [x for x in tbls if isinstance(x, Table)]
1101 # axis is the concentration axes
1102 axis = {t.non_index_axes[0][0] for t in _tbls}.pop()
1104 def func(_start, _stop, _where):
1105 # retrieve the objs, _where is always passed as a set of
1106 # coordinates here
1107 objs = [
1108 t.read(where=_where, columns=columns, start=_start, stop=_stop)
1109 for t in tbls
1110 ]
1112 # concat and return
1113 return concat(objs, axis=axis, verify_integrity=False)._consolidate()
1115 # create the iterator
1116 it = TableIterator(
1117 self,
1118 s,
1119 func,
1120 where=where,
1121 nrows=nrows,
1122 start=start,
1123 stop=stop,
1124 iterator=iterator,
1125 chunksize=chunksize,
1126 auto_close=auto_close,
1127 )
1129 return it.get_result(coordinates=True)
1131 def put(
1132 self,
1133 key: str,
1134 value: DataFrame | Series,
1135 format=None,
1136 index: bool = True,
1137 append: bool = False,
1138 complib=None,
1139 complevel: int | None = None,
1140 min_itemsize: int | dict[str, int] | None = None,
1141 nan_rep=None,
1142 data_columns: Literal[True] | list[str] | None = None,
1143 encoding=None,
1144 errors: str = "strict",
1145 track_times: bool = True,
1146 dropna: bool = False,
1147 ) -> None:
1148 """
1149 Store object in HDFStore.
1151 This method writes a pandas DataFrame or Series into an HDF5 file using
1152 either the fixed or table format. The `table` format allows additional
1153 operations like incremental appends and queries but may have performance
1154 trade-offs. The `fixed` format provides faster read/write operations but
1155 does not support appends or queries.
1157 Parameters
1158 ----------
1159 key : str
1160 Key of object to store in file.
1161 value : {Series, DataFrame}
1162 Value of object to store in file.
1163 format : 'fixed(f)|table(t)', default is 'fixed'
1164 Format to use when storing object in HDFStore. Value can be one of:
1166 ``'fixed'``
1167 Fixed format. Fast writing/reading. Not-appendable, nor searchable.
1168 ``'table'``
1169 Table format. Write as a PyTables Table structure which may perform
1170 worse but allow more flexible operations like searching / selecting
1171 subsets of the data.
1172 index : bool, default True
1173 Write DataFrame index as a column.
1174 append : bool, default False
1175 This will force Table format, append the input data to the existing.
1176 complib : default None
1177 This parameter is currently not accepted.
1178 complevel : int, 0-9, default None
1179 Specifies a compression level for data.
1180 A value of 0 or None disables compression.
1181 min_itemsize : int, dict, or None
1182 Dict of columns that specify minimum str sizes.
1183 nan_rep : str
1184 Str to use as str nan representation.
1185 data_columns : list of columns or True, default None
1186 List of columns to create as data columns, or True to use all columns.
1187 See `here
1188 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
1189 encoding : str, default None
1190 Provide an encoding for strings.
1191 errors : str, default 'strict'
1192 The error handling scheme to use for encoding errors.
1193 The default is 'strict' meaning that encoding errors raise a
1194 UnicodeEncodeError. Other possible values are 'ignore', 'replace' and
1195 'xmlcharrefreplace' as well as any other name registered with
1196 codecs.register_error that can handle UnicodeEncodeErrors.
1197 track_times : bool, default True
1198 Parameter is propagated to 'create_table' method of 'PyTables'.
1199 If set to False it enables to have the same h5 files (same hashes)
1200 independent on creation time.
1201 dropna : bool, default False, optional
1202 Remove missing values.
1204 See Also
1205 --------
1206 HDFStore.info : Prints detailed information on the store.
1207 HDFStore.get_storer : Returns the storer object for a key.
1209 Examples
1210 --------
1211 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1212 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1213 >>> store.put("data", df) # doctest: +SKIP
1214 """
1215 if format is None:
1216 format = get_option("io.hdf.default_format") or "fixed"
1217 format = self._validate_format(format)
1218 self._write_to_group(
1219 key,
1220 value,
1221 format=format,
1222 index=index,
1223 append=append,
1224 complib=complib,
1225 complevel=complevel,
1226 min_itemsize=min_itemsize,
1227 nan_rep=nan_rep,
1228 data_columns=data_columns,
1229 encoding=encoding,
1230 errors=errors,
1231 track_times=track_times,
1232 dropna=dropna,
1233 )
1235 def remove(self, key: str, where=None, start=None, stop=None) -> int | None:
1236 """
1237 Remove pandas object partially by specifying the where condition
1239 Parameters
1240 ----------
1241 key : str
1242 Node to remove or delete rows from
1243 where : list of Term (or convertible) objects, optional
1244 start : integer (defaults to None), row number to start selection
1245 stop : integer (defaults to None), row number to stop selection
1247 Returns
1248 -------
1249 number of rows removed (or None if not a Table)
1251 Raises
1252 ------
1253 raises KeyError if key is not a valid store
1255 """
1256 where = _ensure_term(where, scope_level=1)
1257 try:
1258 s = self.get_storer(key)
1259 except KeyError:
1260 # the key is not a valid store, re-raising KeyError
1261 raise
1262 except AssertionError:
1263 # surface any assertion errors for e.g. debugging
1264 raise
1265 except Exception as err:
1266 # In tests we get here with ClosedFileError, TypeError, and
1267 # _table_mod.NoSuchNodeError. TODO: Catch only these?
1269 if where is not None:
1270 raise ValueError(
1271 "trying to remove a node with a non-None where clause!"
1272 ) from err
1274 # we are actually trying to remove a node (with children)
1275 node = self.get_node(key)
1276 if node is not None:
1277 node._f_remove(recursive=True)
1278 return None
1280 # remove the node
1281 if com.all_none(where, start, stop):
1282 s.group._f_remove(recursive=True)
1283 return None
1285 # delete from the table
1286 if not s.is_table:
1287 raise ValueError("can only remove with where on objects written as tables")
1288 return s.delete(where=where, start=start, stop=stop)
1290 def append(
1291 self,
1292 key: str,
1293 value: DataFrame | Series,
1294 format=None,
1295 axes=None,
1296 index: bool | list[str] = True,
1297 append: bool = True,
1298 complib=None,
1299 complevel: int | None = None,
1300 columns=None,
1301 min_itemsize: int | dict[str, int] | None = None,
1302 nan_rep=None,
1303 chunksize: int | None = None,
1304 expectedrows=None,
1305 dropna: bool | None = None,
1306 data_columns: Literal[True] | list[str] | None = None,
1307 encoding=None,
1308 errors: str = "strict",
1309 ) -> None:
1310 """
1311 Append to Table in file.
1313 Node must already exist and be Table format.
1315 Parameters
1316 ----------
1317 key : str
1318 Key of object to append.
1319 value : {Series, DataFrame}
1320 Value of object to append.
1321 format : 'table' is the default
1322 Format to use when storing object in HDFStore. Value can be one of:
1324 ``'table'``
1325 Table format. Write as a PyTables Table structure which may perform
1326 worse but allow more flexible operations like searching / selecting
1327 subsets of the data.
1328 axes : default None
1329 This parameter is currently not accepted.
1330 index : bool, default True
1331 Write DataFrame index as a column.
1332 append : bool, default True
1333 Append the input data to the existing.
1334 complib : default None
1335 This parameter is currently not accepted.
1336 complevel : int, 0-9, default None
1337 Specifies a compression level for data.
1338 A value of 0 or None disables compression.
1339 columns : default None
1340 This parameter is currently not accepted, try data_columns.
1341 min_itemsize : int, dict, or None
1342 Dict of columns that specify minimum str sizes.
1343 nan_rep : str
1344 Str to use as str nan representation.
1345 chunksize : int or None
1346 Size to chunk the writing.
1347 expectedrows : int
1348 Expected TOTAL row size of this table.
1349 dropna : bool, default False, optional
1350 Do not write an ALL nan row to the store settable
1351 by the option 'io.hdf.dropna_table'.
1352 data_columns : list of columns, or True, default None
1353 List of columns to create as indexed data columns for on-disk
1354 queries, or True to use all columns. By default only the axes
1355 of the object are indexed. See `here
1356 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html#query-via-data-columns>`__.
1357 encoding : default None
1358 Provide an encoding for str.
1359 errors : str, default 'strict'
1360 The error handling scheme to use for encoding errors.
1361 The default is 'strict' meaning that encoding errors raise a
1362 UnicodeEncodeError. Other possible values are 'ignore', 'replace' and
1363 'xmlcharrefreplace' as well as any other name registered with
1364 codecs.register_error that can handle UnicodeEncodeErrors.
1366 See Also
1367 --------
1368 HDFStore.append_to_multiple : Append to multiple tables.
1370 Notes
1371 -----
1372 Does *not* check if data being appended overlaps with existing
1373 data in the table, so be careful
1375 Examples
1376 --------
1377 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1378 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1379 >>> store.put("data", df1, format="table") # doctest: +SKIP
1380 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["A", "B"])
1381 >>> store.append("data", df2) # doctest: +SKIP
1382 >>> store.close() # doctest: +SKIP
1383 A B
1384 0 1 2
1385 1 3 4
1386 0 5 6
1387 1 7 8
1388 """
1389 if columns is not None:
1390 raise TypeError(
1391 "columns is not a supported keyword in append, try data_columns"
1392 )
1394 if dropna is None:
1395 dropna = get_option("io.hdf.dropna_table")
1396 if format is None:
1397 format = get_option("io.hdf.default_format") or "table"
1398 format = self._validate_format(format)
1399 self._write_to_group(
1400 key,
1401 value,
1402 format=format,
1403 axes=axes,
1404 index=index,
1405 append=append,
1406 complib=complib,
1407 complevel=complevel,
1408 min_itemsize=min_itemsize,
1409 nan_rep=nan_rep,
1410 chunksize=chunksize,
1411 expectedrows=expectedrows,
1412 dropna=dropna,
1413 data_columns=data_columns,
1414 encoding=encoding,
1415 errors=errors,
1416 )
1418 def append_to_multiple(
1419 self,
1420 d: dict,
1421 value,
1422 selector,
1423 data_columns=None,
1424 axes=None,
1425 dropna: bool = False,
1426 **kwargs,
1427 ) -> None:
1428 """
1429 Append to multiple tables
1431 Parameters
1432 ----------
1433 d : a dict of table_name to table_columns, None is acceptable as the
1434 values of one node (this will get all the remaining columns)
1435 value : a pandas object
1436 selector : a string that designates the indexable table; all of its
1437 columns will be designed as data_columns, unless data_columns is
1438 passed, in which case these are used
1439 data_columns : list of columns to create as data columns, or True to
1440 use all columns
1441 dropna : if evaluates to True, drop rows from all tables if any single
1442 row in each table has all NaN. Default False.
1444 Notes
1445 -----
1446 axes parameter is currently not accepted
1448 """
1449 if axes is not None:
1450 raise TypeError(
1451 "axes is currently not accepted as a parameter to append_to_multiple; "
1452 "you can create the tables independently instead"
1453 )
1455 if not isinstance(d, dict):
1456 raise ValueError(
1457 "append_to_multiple must have a dictionary specified as the "
1458 "way to split the value"
1459 )
1461 if selector not in d:
1462 raise ValueError(
1463 "append_to_multiple requires a selector that is in passed dict"
1464 )
1466 # figure out the splitting axis (the non_index_axis)
1467 axis = next(iter(set(range(value.ndim)) - set(_AXES_MAP[type(value)])))
1469 # figure out how to split the value
1470 remain_key = None
1471 remain_values: list = []
1472 for k, v in d.items():
1473 if v is None:
1474 if remain_key is not None:
1475 raise ValueError(
1476 "append_to_multiple can only have one value in d that is None"
1477 )
1478 remain_key = k
1479 else:
1480 remain_values.extend(v)
1481 if remain_key is not None:
1482 ordered = value.axes[axis]
1483 ordd = ordered.difference(Index(remain_values))
1484 ordd = sorted(ordered.get_indexer(ordd))
1485 d[remain_key] = ordered.take(ordd)
1487 # data_columns
1488 if data_columns is None:
1489 data_columns = d[selector]
1491 # ensure rows are synchronized across the tables
1492 if dropna:
1493 idxs = (value[cols].dropna(how="all").index for cols in d.values())
1494 valid_index = next(idxs)
1495 for index in idxs:
1496 valid_index = valid_index.intersection(index)
1497 value = value.loc[valid_index]
1499 min_itemsize = kwargs.pop("min_itemsize", None)
1501 # append
1502 for k, v in d.items():
1503 dc = data_columns if k == selector else None
1505 # compute the val
1506 val = value.reindex(v, axis=axis)
1508 filtered = (
1509 {key: value for (key, value) in min_itemsize.items() if key in v}
1510 if min_itemsize is not None
1511 else None
1512 )
1513 self.append(k, val, data_columns=dc, min_itemsize=filtered, **kwargs)
1515 def create_table_index(
1516 self,
1517 key: str,
1518 columns=None,
1519 optlevel: int | None = None,
1520 kind: str | None = None,
1521 ) -> None:
1522 """
1523 Create a pytables index on the table.
1525 Parameters
1526 ----------
1527 key : str
1528 columns : None, bool, or listlike[str]
1529 Indicate which columns to create an index on.
1531 * False : Do not create any indexes.
1532 * True : Create indexes on all columns.
1533 * None : Create indexes on all columns.
1534 * listlike : Create indexes on the given columns.
1536 optlevel : int or None, default None
1537 Optimization level, if None, pytables defaults to 6.
1538 kind : str or None, default None
1539 Kind of index, if None, pytables defaults to "medium".
1541 Raises
1542 ------
1543 TypeError: raises if the node is not a table
1544 """
1545 # version requirements
1546 _tables()
1547 s = self.get_storer(key)
1548 if s is None:
1549 return
1551 if not isinstance(s, Table):
1552 raise TypeError("cannot create table index on a Fixed format store")
1553 s.create_index(columns=columns, optlevel=optlevel, kind=kind)
1555 def groups(self) -> list:
1556 """
1557 Return a list of all the top-level nodes.
1559 Each node returned is not a pandas storage object.
1561 Returns
1562 -------
1563 list
1564 List of objects.
1566 See Also
1567 --------
1568 HDFStore.get_node : Returns the node with the key.
1570 Examples
1571 --------
1572 >>> df = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1573 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1574 >>> store.put("data", df) # doctest: +SKIP
1575 >>> print(store.groups()) # doctest: +SKIP
1576 >>> store.close() # doctest: +SKIP
1577 [/data (Group) ''
1578 children := ['axis0' (Array), 'axis1' (Array), 'block0_values' (Array),
1579 'block0_items' (Array)]]
1580 """
1581 _tables()
1582 self._check_if_open()
1583 assert self._handle is not None # for mypy
1584 assert _table_mod is not None # for mypy
1585 return [
1586 g
1587 for g in self._handle.walk_groups()
1588 if (
1589 not isinstance(g, _table_mod.link.Link)
1590 and (
1591 getattr(g._v_attrs, "pandas_type", None)
1592 or getattr(g, "table", None)
1593 or (isinstance(g, _table_mod.table.Table) and g._v_name != "table")
1594 )
1595 )
1596 ]
1598 def walk(self, where: str = "/") -> Iterator[tuple[str, list[str], list[str]]]:
1599 """
1600 Walk the pytables group hierarchy for pandas objects.
1602 This generator will yield the group path, subgroups and pandas object
1603 names for each group.
1605 Any non-pandas PyTables objects that are not a group will be ignored.
1607 The `where` group itself is listed first (preorder), then each of its
1608 child groups (following an alphanumerical order) is also traversed,
1609 following the same procedure.
1611 Parameters
1612 ----------
1613 where : str, default "/"
1614 Group where to start walking.
1616 Yields
1617 ------
1618 path : str
1619 Full path to a group (without trailing '/').
1620 groups : list
1621 Names (strings) of the groups contained in `path`.
1622 leaves : list
1623 Names (strings) of the pandas objects contained in `path`.
1625 See Also
1626 --------
1627 HDFStore.info : Prints detailed information on the store.
1629 Examples
1630 --------
1631 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1632 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1633 >>> store.put("data", df1, format="table") # doctest: +SKIP
1634 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["A", "B"])
1635 >>> store.append("data", df2) # doctest: +SKIP
1636 >>> store.close() # doctest: +SKIP
1637 >>> for group in store.walk(): # doctest: +SKIP
1638 ... print(group) # doctest: +SKIP
1639 >>> store.close() # doctest: +SKIP
1640 """
1641 _tables()
1642 self._check_if_open()
1643 assert self._handle is not None # for mypy
1644 assert _table_mod is not None # for mypy
1646 for g in self._handle.walk_groups(where):
1647 if getattr(g._v_attrs, "pandas_type", None) is not None:
1648 continue
1650 groups = []
1651 leaves = []
1652 for child in g._v_children.values():
1653 pandas_type = getattr(child._v_attrs, "pandas_type", None)
1654 if pandas_type is None:
1655 if isinstance(child, _table_mod.group.Group):
1656 groups.append(child._v_name)
1657 else:
1658 leaves.append(child._v_name)
1660 yield (g._v_pathname.rstrip("/"), groups, leaves)
1662 def get_node(self, key: str) -> Node | None:
1663 """return the node with the key or None if it does not exist"""
1664 self._check_if_open()
1665 if not key.startswith("/"):
1666 key = "/" + key
1668 assert self._handle is not None
1669 assert _table_mod is not None # for mypy
1670 try:
1671 node = self._handle.get_node(self.root, key)
1672 except _table_mod.exceptions.NoSuchNodeError:
1673 return None
1675 assert isinstance(node, _table_mod.Node), type(node)
1676 return node
1678 def get_storer(self, key: str) -> GenericFixed | Table:
1679 """return the storer object for a key, raise if not in the file"""
1680 group = self.get_node(key)
1681 if group is None:
1682 raise KeyError(f"No object named {key} in the file")
1684 s = self._create_storer(group)
1685 s.infer_axes()
1686 return s
1688 def copy(
1689 self,
1690 file,
1691 mode: str = "w",
1692 propindexes: bool = True,
1693 keys=None,
1694 complib=None,
1695 complevel: int | None = None,
1696 fletcher32: bool = False,
1697 overwrite: bool = True,
1698 ) -> HDFStore:
1699 """
1700 Copy the existing store to a new file, updating in place.
1702 Parameters
1703 ----------
1704 propindexes : bool, default True
1705 Restore indexes in copied file.
1706 keys : list, optional
1707 List of keys to include in the copy (defaults to all).
1708 overwrite : bool, default True
1709 Whether to overwrite (remove and replace) existing nodes in the new store.
1710 mode, complib, complevel, fletcher32 same as in HDFStore.__init__
1712 Returns
1713 -------
1714 open file handle of the new store
1715 """
1716 new_store = HDFStore(
1717 file, mode=mode, complib=complib, complevel=complevel, fletcher32=fletcher32
1718 )
1719 if keys is None:
1720 keys = list(self.keys())
1721 if not isinstance(keys, (tuple, list)):
1722 keys = [keys]
1723 for k in keys:
1724 s = self.get_storer(k)
1725 if s is not None:
1726 if k in new_store:
1727 if overwrite:
1728 new_store.remove(k)
1730 data = self.select(k)
1731 if isinstance(s, Table):
1732 index: bool | list[str] = False
1733 if propindexes:
1734 index = [a.name for a in s.axes if a.is_indexed]
1735 new_store.append(
1736 k,
1737 data,
1738 index=index,
1739 data_columns=getattr(s, "data_columns", None),
1740 encoding=s.encoding,
1741 )
1742 else:
1743 new_store.put(k, data, encoding=s.encoding)
1745 return new_store
1747 def info(self) -> str:
1748 """
1749 Print detailed information on the store.
1751 Returns
1752 -------
1753 str
1754 A String containing the python pandas class name, filepath to the HDF5
1755 file and all the object keys along with their respective dataframe shapes.
1757 See Also
1758 --------
1759 HDFStore.get_storer : Returns the storer object for a key.
1761 Examples
1762 --------
1763 >>> df1 = pd.DataFrame([[1, 2], [3, 4]], columns=["A", "B"])
1764 >>> df2 = pd.DataFrame([[5, 6], [7, 8]], columns=["C", "D"])
1765 >>> store = pd.HDFStore("store.h5", "w") # doctest: +SKIP
1766 >>> store.put("data1", df1) # doctest: +SKIP
1767 >>> store.put("data2", df2) # doctest: +SKIP
1768 >>> print(store.info()) # doctest: +SKIP
1769 >>> store.close() # doctest: +SKIP
1770 <class 'pandas.io.pytables.HDFStore'>
1771 File path: store.h5
1772 /data1 frame (shape->[2,2])
1773 /data2 frame (shape->[2,2])
1774 """
1775 path = pprint_thing(self._path)
1776 output = f"{type(self)}\nFile path: {path}\n"
1778 if self.is_open:
1779 lkeys = sorted(self.keys())
1780 if lkeys:
1781 keys = []
1782 values = []
1784 for k in lkeys:
1785 try:
1786 s = self.get_storer(k)
1787 if s is not None:
1788 keys.append(pprint_thing(s.pathname or k))
1789 values.append(pprint_thing(s or "invalid_HDFStore node"))
1790 except AssertionError:
1791 # surface any assertion errors for e.g. debugging
1792 raise
1793 except Exception as detail:
1794 keys.append(k)
1795 dstr = pprint_thing(detail)
1796 values.append(f"[invalid_HDFStore node: {dstr}]")
1798 output += adjoin(12, keys, values)
1799 else:
1800 output += "Empty"
1801 else:
1802 output += "File is CLOSED"
1804 return output
1806 # ------------------------------------------------------------------------
1807 # private methods
1809 def _check_if_open(self) -> None:
1810 if not self.is_open:
1811 raise ClosedFileError(f"{self._path} file is not open!")
1813 def _validate_format(self, format: str) -> str:
1814 """validate / deprecate formats"""
1815 # validate
1816 try:
1817 format = _FORMAT_MAP[format.lower()]
1818 except KeyError as err:
1819 raise TypeError(f"invalid HDFStore format specified [{format}]") from err
1821 return format
1823 def _create_storer(
1824 self,
1825 group,
1826 format=None,
1827 value: DataFrame | Series | None = None,
1828 encoding: str = "UTF-8",
1829 errors: str = "strict",
1830 ) -> GenericFixed | Table:
1831 """return a suitable class to operate"""
1832 cls: type[GenericFixed | Table]
1834 if value is not None and not isinstance(value, (Series, DataFrame)):
1835 raise TypeError("value must be None, Series, or DataFrame")
1837 pt = getattr(group._v_attrs, "pandas_type", None)
1838 tt = getattr(group._v_attrs, "table_type", None)
1840 # infer the pt from the passed value
1841 if pt is None:
1842 if value is None:
1843 _tables()
1844 assert _table_mod is not None # for mypy
1845 if getattr(group, "table", None) or isinstance(
1846 group, _table_mod.table.Table
1847 ):
1848 pt = "frame_table"
1849 tt = "generic_table"
1850 else:
1851 raise TypeError(
1852 "cannot create a storer if the object is not existing "
1853 "nor a value are passed"
1854 )
1855 else:
1856 if isinstance(value, Series):
1857 pt = "series"
1858 else:
1859 pt = "frame"
1861 # we are actually a table
1862 if format == "table":
1863 pt += "_table"
1865 # a storer node
1866 if "table" not in pt:
1867 _STORER_MAP = {"series": SeriesFixed, "frame": FrameFixed}
1868 try:
1869 cls = _STORER_MAP[pt]
1870 except KeyError as err:
1871 raise TypeError(
1872 f"cannot properly create the storer for: [_STORER_MAP] [group->"
1873 f"{group},value->{type(value)},format->{format}"
1874 ) from err
1875 return cls(self, group, encoding=encoding, errors=errors)
1877 # existing node (and must be a table)
1878 if tt is None:
1879 # if we are a writer, determine the tt
1880 if value is not None:
1881 if pt == "series_table":
1882 index = getattr(value, "index", None)
1883 if index is not None:
1884 if index.nlevels == 1:
1885 tt = "appendable_series"
1886 elif index.nlevels > 1:
1887 tt = "appendable_multiseries"
1888 elif pt == "frame_table":
1889 index = getattr(value, "index", None)
1890 if index is not None:
1891 if index.nlevels == 1:
1892 tt = "appendable_frame"
1893 elif index.nlevels > 1:
1894 tt = "appendable_multiframe"
1896 _TABLE_MAP = {
1897 "generic_table": GenericTable,
1898 "appendable_series": AppendableSeriesTable,
1899 "appendable_multiseries": AppendableMultiSeriesTable,
1900 "appendable_frame": AppendableFrameTable,
1901 "appendable_multiframe": AppendableMultiFrameTable,
1902 "worm": WORMTable,
1903 }
1904 try:
1905 cls = _TABLE_MAP[tt] # type: ignore[index]
1906 except KeyError as err:
1907 raise TypeError(
1908 f"cannot properly create the storer for: [_TABLE_MAP] [group->"
1909 f"{group},value->{type(value)},format->{format}"
1910 ) from err
1912 return cls(self, group, encoding=encoding, errors=errors)
1914 def _write_to_group(
1915 self,
1916 key: str,
1917 value: DataFrame | Series,
1918 format,
1919 axes=None,
1920 index: bool | list[str] = True,
1921 append: bool = False,
1922 complib=None,
1923 complevel: int | None = None,
1924 fletcher32=None,
1925 min_itemsize: int | dict[str, int] | None = None,
1926 chunksize: int | None = None,
1927 expectedrows=None,
1928 dropna: bool = False,
1929 nan_rep=None,
1930 data_columns=None,
1931 encoding=None,
1932 errors: str = "strict",
1933 track_times: bool = True,
1934 ) -> None:
1935 # we don't want to store a table node at all if our object is 0-len
1936 # as there are not dtypes
1937 if getattr(value, "empty", None) and (format == "table" or append):
1938 return
1940 group = self._identify_group(key, append)
1942 s = self._create_storer(group, format, value, encoding=encoding, errors=errors)
1943 if append:
1944 # raise if we are trying to append to a Fixed format,
1945 # or a table that exists (and we are putting)
1946 if not s.is_table or (s.is_table and format == "fixed" and s.is_exists):
1947 raise ValueError("Can only append to Tables")
1948 if not s.is_exists:
1949 s.set_object_info()
1950 else:
1951 s.set_object_info()
1953 if not s.is_table and complib:
1954 raise ValueError("Compression not supported on Fixed format stores")
1956 # write the object
1957 s.write(
1958 obj=value,
1959 axes=axes,
1960 append=append,
1961 complib=complib,
1962 complevel=complevel,
1963 fletcher32=fletcher32,
1964 min_itemsize=min_itemsize,
1965 chunksize=chunksize,
1966 expectedrows=expectedrows,
1967 dropna=dropna,
1968 nan_rep=nan_rep,
1969 data_columns=data_columns,
1970 track_times=track_times,
1971 )
1973 if isinstance(s, Table) and index:
1974 s.create_index(columns=index)
1976 def _read_group(self, group: Node):
1977 s = self._create_storer(group)
1978 s.infer_axes()
1979 return s.read()
1981 def _identify_group(self, key: str, append: bool) -> Node:
1982 """Identify HDF5 group based on key, delete/create group if needed."""
1983 group = self.get_node(key)
1985 # we make this assertion for mypy; the get_node call will already
1986 # have raised if this is incorrect
1987 assert self._handle is not None
1989 # remove the node if we are not appending
1990 if group is not None and not append:
1991 self._handle.remove_node(group, recursive=True)
1992 group = None
1994 if group is None:
1995 group = self._create_nodes_and_group(key)
1997 return group
1999 def _create_nodes_and_group(self, key: str) -> Node:
2000 """Create nodes from key and return group name."""
2001 # assertion for mypy
2002 assert self._handle is not None
2004 paths = key.split("/")
2005 # recursively create the groups
2006 path = "/"
2007 for p in paths:
2008 if not len(p):
2009 continue
2010 new_path = path
2011 if not path.endswith("/"):
2012 new_path += "/"
2013 new_path += p
2014 group = self.get_node(new_path)
2015 if group is None:
2016 group = self._handle.create_group(path, p)
2017 path = new_path
2018 return group
2021class TableIterator:
2022 """
2023 Define the iteration interface on a table
2025 Parameters
2026 ----------
2027 store : HDFStore
2028 s : the referred storer
2029 func : the function to execute the query
2030 where : the where of the query
2031 nrows : the rows to iterate on
2032 start : the passed start value (default is None)
2033 stop : the passed stop value (default is None)
2034 iterator : bool, default False
2035 Whether to use the default iterator.
2036 chunksize : the passed chunking value (default is 100000)
2037 auto_close : bool, default False
2038 Whether to automatically close the store at the end of iteration.
2039 """
2041 chunksize: int | None
2042 store: HDFStore
2043 s: GenericFixed | Table
2045 def __init__(
2046 self,
2047 store: HDFStore,
2048 s: GenericFixed | Table,
2049 func,
2050 where,
2051 nrows,
2052 start=None,
2053 stop=None,
2054 iterator: bool = False,
2055 chunksize: int | None = None,
2056 auto_close: bool = False,
2057 ) -> None:
2058 self.store = store
2059 self.s = s
2060 self.func = func
2061 self.where = where
2063 # set start/stop if they are not set if we are a table
2064 if self.s.is_table:
2065 if nrows is None:
2066 nrows = 0
2067 if start is None:
2068 start = 0
2069 if stop is None:
2070 stop = nrows
2071 stop = min(nrows, stop)
2073 self.nrows = nrows
2074 self.start = start
2075 self.stop = stop
2077 self.coordinates = None
2078 if iterator or chunksize is not None:
2079 if chunksize is None:
2080 chunksize = 100000
2081 self.chunksize = int(chunksize)
2082 else:
2083 self.chunksize = None
2085 self.auto_close = auto_close
2087 def __iter__(self) -> Iterator:
2088 # iterate
2089 current = self.start
2090 if self.coordinates is None:
2091 raise ValueError("Cannot iterate until get_result is called.")
2092 while current < self.stop:
2093 stop = min(current + self.chunksize, self.stop)
2094 value = self.func(None, None, self.coordinates[current:stop])
2095 current = stop
2096 if value is None or not len(value):
2097 continue
2099 yield value
2101 self.close()
2103 def close(self) -> None:
2104 if self.auto_close:
2105 self.store.close()
2107 def get_result(self, coordinates: bool = False):
2108 # return the actual iterator
2109 if self.chunksize is not None:
2110 if not isinstance(self.s, Table):
2111 raise TypeError("can only use an iterator or chunksize on a table")
2113 self.coordinates = self.s.read_coordinates(where=self.where)
2115 return self
2117 # if specified read via coordinates (necessary for multiple selections
2118 if coordinates:
2119 if not isinstance(self.s, Table):
2120 raise TypeError("can only read_coordinates on a table")
2121 where = self.s.read_coordinates(
2122 where=self.where, start=self.start, stop=self.stop
2123 )
2124 else:
2125 where = self.where
2127 # directly return the result
2128 results = self.func(self.start, self.stop, where)
2129 self.close()
2130 return results
2133class IndexCol:
2134 """
2135 an index column description class
2137 Parameters
2138 ----------
2139 axis : axis which I reference
2140 values : the ndarray like converted values
2141 kind : a string description of this type
2142 typ : the pytables type
2143 pos : the position in the pytables
2145 """
2147 is_an_indexable: bool = True
2148 is_data_indexable: bool = True
2149 _info_fields = ["freq", "tz", "index_name"]
2151 def __init__(
2152 self,
2153 name: str,
2154 values=None,
2155 kind=None,
2156 typ=None,
2157 cname: str | None = None,
2158 axis=None,
2159 pos=None,
2160 freq=None,
2161 tz=None,
2162 index_name=None,
2163 ordered=None,
2164 table=None,
2165 meta=None,
2166 metadata=None,
2167 ) -> None:
2168 if not isinstance(name, str):
2169 raise ValueError("`name` must be a str.")
2171 self.values = values
2172 self.kind = kind
2173 self.typ = typ
2174 self.name = name
2175 self.cname = cname or name
2176 self.axis = axis
2177 self.pos = pos
2178 self.freq = freq
2179 self.tz = tz
2180 self.index_name = index_name
2181 self.ordered = ordered
2182 self.table = table
2183 self.meta = meta
2184 self.metadata = metadata
2186 if pos is not None:
2187 self.set_pos(pos)
2189 # These are ensured as long as the passed arguments match the
2190 # constructor annotations.
2191 assert isinstance(self.name, str)
2192 assert isinstance(self.cname, str)
2194 @property
2195 def itemsize(self) -> int:
2196 # Assumes self.typ has already been initialized
2197 return self.typ.itemsize
2199 @property
2200 def kind_attr(self) -> str:
2201 return f"{self.name}_kind"
2203 def set_pos(self, pos: int) -> None:
2204 """set the position of this column in the Table"""
2205 self.pos = pos
2206 if pos is not None and self.typ is not None:
2207 self.typ._v_pos = pos
2209 def __repr__(self) -> str:
2210 temp = tuple(
2211 map(pprint_thing, (self.name, self.cname, self.axis, self.pos, self.kind))
2212 )
2213 return ",".join(
2214 [
2215 f"{key}->{value}"
2216 for key, value in zip(
2217 ["name", "cname", "axis", "pos", "kind"], temp, strict=True
2218 )
2219 ]
2220 )
2222 def __eq__(self, other: object) -> bool:
2223 """compare 2 col items"""
2224 return all(
2225 getattr(self, a, None) == getattr(other, a, None)
2226 for a in ["name", "cname", "axis", "pos"]
2227 )
2229 def __ne__(self, other) -> bool:
2230 return not self.__eq__(other)
2232 @property
2233 def is_indexed(self) -> bool:
2234 """return whether I am an indexed column"""
2235 if not hasattr(self.table, "cols"):
2236 # e.g. if infer hasn't been called yet, self.table will be None.
2237 return False
2238 return getattr(self.table.cols, self.cname).is_indexed
2240 def convert(
2241 self, values: np.ndarray, nan_rep, encoding: str, errors: str
2242 ) -> tuple[np.ndarray, np.ndarray] | tuple[Index, Index]:
2243 """
2244 Convert the data from this selection to the appropriate pandas type.
2245 """
2246 assert isinstance(values, np.ndarray), type(values)
2248 # values is a recarray
2249 if values.dtype.fields is not None:
2250 # Copy, otherwise values will be a view
2251 # preventing the original recarry from being free'ed
2252 values = values[self.cname].copy()
2254 val_kind = self.kind
2255 values = _maybe_convert(values, val_kind, encoding, errors)
2256 kwargs = {}
2257 kwargs["name"] = self.index_name
2259 if self.freq is not None:
2260 kwargs["freq"] = self.freq
2262 factory: type[Index | DatetimeIndex] = Index
2263 if lib.is_np_dtype(values.dtype, "M") or isinstance(
2264 values.dtype, DatetimeTZDtype
2265 ):
2266 factory = DatetimeIndex
2267 elif values.dtype == "i8" and "freq" in kwargs:
2268 # PeriodIndex data is stored as i8
2269 # error: Incompatible types in assignment (expression has type
2270 # "Callable[[Any, KwArg(Any)], PeriodIndex]", variable has type
2271 # "Union[Type[Index], Type[DatetimeIndex]]")
2272 factory = lambda x, **kwds: PeriodIndex.from_ordinals( # type: ignore[assignment]
2273 x, freq=kwds.get("freq", None)
2274 )._rename(kwds["name"])
2276 # making an Index instance could throw a number of different errors
2277 try:
2278 new_pd_index = factory(values, **kwargs)
2279 except UnicodeEncodeError as err:
2280 if (
2281 errors == "surrogatepass"
2282 and using_string_dtype()
2283 and str(err).endswith("surrogates not allowed")
2284 and HAS_PYARROW
2285 ):
2286 new_pd_index = factory(
2287 values,
2288 dtype=StringDtype(storage="python", na_value=np.nan),
2289 **kwargs,
2290 )
2291 else:
2292 raise
2293 except ValueError:
2294 # if the output freq is different that what we recorded,
2295 # it should be None (see also 'doc example part 2')
2296 if "freq" in kwargs:
2297 kwargs["freq"] = None
2298 new_pd_index = factory(values, **kwargs)
2300 final_pd_index: Index
2301 if self.tz is not None and isinstance(new_pd_index, DatetimeIndex):
2302 final_pd_index = new_pd_index.tz_localize("UTC").tz_convert(self.tz)
2303 else:
2304 final_pd_index = new_pd_index
2305 return final_pd_index, final_pd_index
2307 def take_data(self):
2308 """return the values"""
2309 return self.values
2311 @property
2312 def attrs(self):
2313 return self.table._v_attrs
2315 @property
2316 def description(self):
2317 return self.table.description
2319 @property
2320 def col(self):
2321 """return my current col description"""
2322 return getattr(self.description, self.cname, None)
2324 @property
2325 def cvalues(self):
2326 """return my cython values"""
2327 return self.values
2329 def __iter__(self) -> Iterator:
2330 return iter(self.values)
2332 def maybe_set_size(self, min_itemsize=None) -> None:
2333 """
2334 maybe set a string col itemsize:
2335 min_itemsize can be an integer or a dict with this columns name
2336 with an integer size
2337 """
2338 if self.kind == "string":
2339 if isinstance(min_itemsize, dict):
2340 min_itemsize = min_itemsize.get(self.name)
2342 if min_itemsize is not None and self.typ.itemsize < min_itemsize:
2343 self.typ = _tables().StringCol(itemsize=min_itemsize, pos=self.pos)
2345 def validate_names(self) -> None:
2346 pass
2348 def validate_and_set(self, handler: AppendableTable, append: bool) -> None:
2349 self.table = handler.table
2350 self.validate_col()
2351 self.validate_attr(append)
2352 self.validate_metadata(handler)
2353 self.write_metadata(handler)
2354 self.set_attr()
2356 def validate_col(self, itemsize=None):
2357 """validate this column: return the compared against itemsize"""
2358 # validate this column for string truncation (or reset to the max size)
2359 if self.kind == "string":
2360 c = self.col
2361 if c is not None:
2362 if itemsize is None:
2363 itemsize = self.itemsize
2364 if c.itemsize < itemsize:
2365 raise ValueError(
2366 f"Trying to store a string with len [{itemsize}] in "
2367 f"[{self.cname}] column but\nthis column has a limit of "
2368 f"[{c.itemsize}]!\nConsider using min_itemsize to "
2369 "preset the sizes on these columns"
2370 )
2371 return c.itemsize
2373 return None
2375 def validate_attr(self, append: bool) -> None:
2376 # check for backwards incompatibility
2377 if append:
2378 existing_kind = getattr(self.attrs, self.kind_attr, None)
2379 if existing_kind is not None and existing_kind != self.kind:
2380 raise TypeError(
2381 f"incompatible kind in col [{existing_kind} - {self.kind}]"
2382 )
2384 def update_info(self, info) -> None:
2385 """
2386 set/update the info for this indexable with the key/value
2387 if there is a conflict raise/warn as needed
2388 """
2389 for key in self._info_fields:
2390 value = getattr(self, key, None)
2391 idx = info.setdefault(self.name, {})
2393 existing_value = idx.get(key)
2394 if key in idx and value is not None and existing_value != value:
2395 # frequency/name just warn
2396 if key in ["freq", "index_name"]:
2397 ws = attribute_conflict_doc % (key, existing_value, value)
2398 warnings.warn(
2399 ws, AttributeConflictWarning, stacklevel=find_stack_level()
2400 )
2402 # reset
2403 idx[key] = None
2404 setattr(self, key, None)
2406 else:
2407 raise ValueError(
2408 f"invalid info for [{self.name}] for [{key}], "
2409 f"existing_value [{existing_value}] conflicts with "
2410 f"new value [{value}]"
2411 )
2412 elif value is not None or existing_value is not None:
2413 idx[key] = value
2415 def set_info(self, info) -> None:
2416 """set my state from the passed info"""
2417 idx = info.get(self.name)
2418 if idx is not None:
2419 self.__dict__.update(idx)
2421 def set_attr(self) -> None:
2422 """set the kind for this column"""
2423 setattr(self.attrs, self.kind_attr, self.kind)
2425 def validate_metadata(self, handler: AppendableTable) -> None:
2426 """validate that kind=category does not change the categories"""
2427 if self.meta == "category":
2428 new_metadata = self.metadata
2429 cur_metadata = handler.read_metadata(self.cname)
2430 if (
2431 new_metadata is not None
2432 and cur_metadata is not None
2433 and not array_equivalent(
2434 new_metadata, cur_metadata, strict_nan=True, dtype_equal=True
2435 )
2436 ):
2437 raise ValueError(
2438 "cannot append a categorical with "
2439 "different categories to the existing"
2440 )
2442 def write_metadata(self, handler: AppendableTable) -> None:
2443 """set the meta data"""
2444 if self.metadata is not None:
2445 handler.write_metadata(self.cname, self.metadata)
2448class GenericIndexCol(IndexCol):
2449 """an index which is not represented in the data of the table"""
2451 @property
2452 def is_indexed(self) -> bool:
2453 return False
2455 def convert(
2456 self, values: np.ndarray, nan_rep, encoding: str, errors: str
2457 ) -> tuple[Index, Index]:
2458 """
2459 Convert the data from this selection to the appropriate pandas type.
2461 Parameters
2462 ----------
2463 values : np.ndarray
2464 nan_rep : str
2465 encoding : str
2466 errors : str
2467 """
2468 assert isinstance(values, np.ndarray), type(values)
2470 index = RangeIndex(len(values))
2471 return index, index
2473 def set_attr(self) -> None:
2474 pass
2477class DataCol(IndexCol):
2478 """
2479 a data holding column, by definition this is not indexable
2481 Parameters
2482 ----------
2483 data : the actual data
2484 cname : the column name in the table to hold the data (typically
2485 values)
2486 meta : a string description of the metadata
2487 metadata : the actual metadata
2488 """
2490 is_an_indexable = False
2491 is_data_indexable = False
2492 _info_fields = ["tz", "ordered"]
2494 def __init__(
2495 self,
2496 name: str,
2497 values=None,
2498 kind=None,
2499 typ=None,
2500 cname: str | None = None,
2501 pos=None,
2502 tz=None,
2503 ordered=None,
2504 table=None,
2505 meta=None,
2506 metadata=None,
2507 dtype: DtypeArg | None = None,
2508 data=None,
2509 ) -> None:
2510 super().__init__(
2511 name=name,
2512 values=values,
2513 kind=kind,
2514 typ=typ,
2515 pos=pos,
2516 cname=cname,
2517 tz=tz,
2518 ordered=ordered,
2519 table=table,
2520 meta=meta,
2521 metadata=metadata,
2522 )
2523 self.dtype = dtype
2524 self.data = data
2526 @property
2527 def dtype_attr(self) -> str:
2528 return f"{self.name}_dtype"
2530 @property
2531 def meta_attr(self) -> str:
2532 return f"{self.name}_meta"
2534 def __repr__(self) -> str:
2535 temp = tuple(
2536 map(
2537 pprint_thing, (self.name, self.cname, self.dtype, self.kind, self.shape)
2538 )
2539 )
2540 return ",".join(
2541 [
2542 f"{key}->{value}"
2543 for key, value in zip(
2544 ["name", "cname", "dtype", "kind", "shape"], temp, strict=True
2545 )
2546 ]
2547 )
2549 def __eq__(self, other: object) -> bool:
2550 """compare 2 col items"""
2551 return all(
2552 getattr(self, a, None) == getattr(other, a, None)
2553 for a in ["name", "cname", "dtype", "pos"]
2554 )
2556 def set_data(self, data: ArrayLike) -> None:
2557 assert data is not None
2558 assert self.dtype is None
2560 data, dtype_name = _get_data_and_dtype_name(data)
2562 self.data = data
2563 self.dtype = dtype_name
2564 self.kind = _dtype_to_kind(dtype_name)
2566 def take_data(self):
2567 """return the data"""
2568 return self.data
2570 @classmethod
2571 def _get_atom(cls, values: ArrayLike) -> Col:
2572 """
2573 Get an appropriately typed and shaped pytables.Col object for values.
2574 """
2575 dtype = values.dtype
2576 # error: Item "ExtensionDtype" of "Union[ExtensionDtype, dtype[Any]]" has no
2577 # attribute "itemsize"
2578 itemsize = dtype.itemsize # type: ignore[union-attr]
2580 shape = values.shape
2581 if values.ndim == 1:
2582 # EA, use block shape pretending it is 2D
2583 # TODO(EA2D): not necessary with 2D EAs
2584 shape = (1, values.size)
2586 if isinstance(values, Categorical):
2587 codes = values.codes
2588 atom = cls.get_atom_data(shape, kind=codes.dtype.name)
2589 elif lib.is_np_dtype(dtype, "M") or isinstance(dtype, DatetimeTZDtype):
2590 atom = cls.get_atom_datetime64(shape)
2591 elif lib.is_np_dtype(dtype, "m"):
2592 atom = cls.get_atom_timedelta64(shape)
2593 elif is_complex_dtype(dtype):
2594 atom = _tables().ComplexCol(itemsize=itemsize, shape=shape[0])
2595 elif is_string_dtype(dtype):
2596 atom = cls.get_atom_string(shape, itemsize)
2597 else:
2598 atom = cls.get_atom_data(shape, kind=dtype.name)
2600 return atom
2602 @classmethod
2603 def get_atom_string(cls, shape, itemsize):
2604 return _tables().StringCol(itemsize=itemsize, shape=shape[0])
2606 @classmethod
2607 def get_atom_coltype(cls, kind: str) -> type[Col]:
2608 """return the PyTables column class for this column"""
2609 if kind.startswith("uint"):
2610 k4 = kind[4:]
2611 col_name = f"UInt{k4}Col"
2612 elif kind.startswith("period"):
2613 # we store as integer
2614 col_name = "Int64Col"
2615 else:
2616 kcap = kind.capitalize()
2617 col_name = f"{kcap}Col"
2619 return getattr(_tables(), col_name)
2621 @classmethod
2622 def get_atom_data(cls, shape, kind: str) -> Col:
2623 return cls.get_atom_coltype(kind=kind)(shape=shape[0])
2625 @classmethod
2626 def get_atom_datetime64(cls, shape):
2627 return _tables().Int64Col(shape=shape[0])
2629 @classmethod
2630 def get_atom_timedelta64(cls, shape):
2631 return _tables().Int64Col(shape=shape[0])
2633 @property
2634 def shape(self):
2635 return getattr(self.data, "shape", None)
2637 @property
2638 def cvalues(self):
2639 """return my cython values"""
2640 return self.data
2642 def validate_attr(self, append) -> None:
2643 """validate that we have the same order as the existing & same dtype"""
2644 if append:
2645 existing_fields = getattr(self.attrs, self.kind_attr, None)
2646 if existing_fields is not None and existing_fields != list(self.values):
2647 raise ValueError("appended items do not match existing items in table!")
2649 existing_dtype = getattr(self.attrs, self.dtype_attr, None)
2650 if existing_dtype is not None and existing_dtype != self.dtype:
2651 raise ValueError(
2652 "appended items dtype do not match existing items dtype in table!"
2653 )
2655 def convert(self, values: np.ndarray, nan_rep, encoding: str, errors: str):
2656 """
2657 Convert the data from this selection to the appropriate pandas type.
2659 Parameters
2660 ----------
2661 values : np.ndarray
2662 nan_rep :
2663 encoding : str
2664 errors : str
2666 Returns
2667 -------
2668 index : listlike to become an Index
2669 data : ndarraylike to become a column
2670 """
2671 assert isinstance(values, np.ndarray), type(values)
2673 # values is a recarray
2674 if values.dtype.fields is not None:
2675 values = values[self.cname]
2677 assert self.typ is not None
2678 if self.dtype is None:
2679 # Note: in tests we never have timedelta64 or datetime64,
2680 # so the _get_data_and_dtype_name may be unnecessary
2681 converted, dtype_name = _get_data_and_dtype_name(values)
2682 kind = _dtype_to_kind(dtype_name)
2683 else:
2684 converted = values
2685 dtype_name = self.dtype
2686 kind = self.kind
2688 assert isinstance(converted, np.ndarray) # for mypy
2690 # use the meta if needed
2691 meta = self.meta
2692 metadata = self.metadata
2693 ordered = self.ordered
2694 tz = self.tz
2696 assert dtype_name is not None
2697 # convert to the correct dtype
2698 dtype = dtype_name
2700 # reverse converts
2701 if dtype.startswith("datetime64"):
2702 # recreate with tz if indicated
2703 if dtype == "datetime64":
2704 dtype = "datetime64[ns]"
2705 converted = _set_tz(converted, tz, dtype)
2707 elif dtype.startswith("timedelta64"):
2708 if dtype == "timedelta64":
2709 # from before we started storing timedelta64 unit
2710 converted = np.asarray(converted, dtype="m8[ns]")
2711 else:
2712 converted = np.asarray(converted, dtype=dtype)
2713 elif dtype == "date":
2714 try:
2715 converted = np.asarray(
2716 [date.fromordinal(v) for v in converted], dtype=object
2717 )
2718 except ValueError:
2719 converted = np.asarray(
2720 [date.fromtimestamp(v) for v in converted], dtype=object
2721 )
2723 elif meta == "category":
2724 # we have a categorical
2725 categories = metadata
2726 codes = converted.ravel()
2728 # if we have stored a NaN in the categories
2729 # then strip it; in theory we could have BOTH
2730 # -1s in the codes and nulls :<
2731 if categories is None:
2732 # Handle case of NaN-only categorical columns in which case
2733 # the categories are an empty array; when this is stored,
2734 # pytables cannot write a zero-len array, so on readback
2735 # the categories would be None and `read_hdf()` would fail.
2736 categories = Index([], dtype=np.float64)
2737 else:
2738 mask = isna(categories)
2739 if mask.any():
2740 categories = categories[~mask]
2741 codes[codes != -1] -= mask.astype(int).cumsum()._values
2743 converted = Categorical.from_codes(
2744 codes, categories=categories, ordered=ordered, validate=False
2745 )
2747 else:
2748 try:
2749 converted = converted.astype(dtype, copy=False)
2750 except TypeError:
2751 converted = converted.astype("O", copy=False)
2753 # convert nans / decode
2754 if kind == "string":
2755 converted = _unconvert_string_array(
2756 converted, nan_rep=nan_rep, encoding=encoding, errors=errors
2757 )
2759 return self.values, converted
2761 def set_attr(self) -> None:
2762 """set the data for this column"""
2763 setattr(self.attrs, self.kind_attr, self.values)
2764 setattr(self.attrs, self.meta_attr, self.meta)
2765 assert self.dtype is not None
2766 setattr(self.attrs, self.dtype_attr, self.dtype)
2769class DataIndexableCol(DataCol):
2770 """represent a data column that can be indexed"""
2772 is_data_indexable = True
2774 def validate_names(self) -> None:
2775 if not is_string_dtype(Index(self.values).dtype):
2776 # TODO: should the message here be more specifically non-str?
2777 raise ValueError("cannot have non-object label DataIndexableCol")
2779 @classmethod
2780 def get_atom_string(cls, shape, itemsize):
2781 return _tables().StringCol(itemsize=itemsize)
2783 @classmethod
2784 def get_atom_data(cls, shape, kind: str) -> Col:
2785 return cls.get_atom_coltype(kind=kind)()
2787 @classmethod
2788 def get_atom_datetime64(cls, shape):
2789 return _tables().Int64Col()
2791 @classmethod
2792 def get_atom_timedelta64(cls, shape):
2793 return _tables().Int64Col()
2796class GenericDataIndexableCol(DataIndexableCol):
2797 """represent a generic pytables data column"""
2800class Fixed:
2801 """
2802 represent an object in my store
2803 facilitate read/write of various types of objects
2804 this is an abstract base class
2806 Parameters
2807 ----------
2808 parent : HDFStore
2809 group : Node
2810 The group node where the table resides.
2811 """
2813 pandas_kind: str
2814 format_type: str = "fixed" # GH#30962 needed by dask
2815 obj_type: type[DataFrame | Series]
2816 ndim: int
2817 parent: HDFStore
2818 is_table: bool = False
2820 def __init__(
2821 self,
2822 parent: HDFStore,
2823 group: Node,
2824 encoding: str | None = "UTF-8",
2825 errors: str = "strict",
2826 ) -> None:
2827 assert isinstance(parent, HDFStore), type(parent)
2828 assert _table_mod is not None # needed for mypy
2829 assert isinstance(group, _table_mod.Node), type(group)
2830 self.parent = parent
2831 self.group = group
2832 self.encoding = _ensure_encoding(encoding)
2833 self.errors = errors
2835 @property
2836 def is_old_version(self) -> bool:
2837 return self.version[0] <= 0 and self.version[1] <= 10 and self.version[2] < 1
2839 @property
2840 def version(self) -> tuple[int, int, int]:
2841 """compute and set our version"""
2842 version = getattr(self.group._v_attrs, "pandas_version", None)
2843 if isinstance(version, str):
2844 version_tup = tuple(int(x) for x in version.split("."))
2845 if len(version_tup) == 2:
2846 version_tup = (*version_tup, 0)
2847 assert len(version_tup) == 3 # needed for mypy
2848 return version_tup
2849 else:
2850 return (0, 0, 0)
2852 @property
2853 def pandas_type(self):
2854 return getattr(self.group._v_attrs, "pandas_type", None)
2856 def __repr__(self) -> str:
2857 """return a pretty representation of myself"""
2858 self.infer_axes()
2859 s = self.shape
2860 if s is not None:
2861 if isinstance(s, (list, tuple)):
2862 jshape = ",".join([pprint_thing(x) for x in s])
2863 s = f"[{jshape}]"
2864 return f"{self.pandas_type:12.12} (shape->{s})"
2865 return self.pandas_type
2867 def set_object_info(self) -> None:
2868 """set my pandas type & version"""
2869 self.attrs.pandas_type = str(self.pandas_kind)
2870 self.attrs.pandas_version = str(_version)
2872 def copy(self) -> Fixed:
2873 new_self = copy.copy(self)
2874 return new_self
2876 @property
2877 def shape(self):
2878 return self.nrows
2880 @property
2881 def pathname(self):
2882 return self.group._v_pathname
2884 @property
2885 def _handle(self):
2886 return self.parent._handle
2888 @property
2889 def _filters(self):
2890 return self.parent._filters
2892 @property
2893 def _complevel(self) -> int:
2894 return self.parent._complevel
2896 @property
2897 def _fletcher32(self) -> bool:
2898 return self.parent._fletcher32
2900 @property
2901 def attrs(self):
2902 return self.group._v_attrs
2904 def set_attrs(self) -> None:
2905 """set our object attributes"""
2907 def get_attrs(self) -> None:
2908 """get our object attributes"""
2910 @property
2911 def storable(self):
2912 """return my storable"""
2913 return self.group
2915 @property
2916 def is_exists(self) -> bool:
2917 return False
2919 @property
2920 def nrows(self):
2921 return getattr(self.storable, "nrows", None)
2923 def validate(self, other) -> Literal[True] | None:
2924 """validate against an existing storable"""
2925 if other is None:
2926 return None
2927 return True
2929 def validate_version(self, where=None) -> None:
2930 """are we trying to operate on an old version?"""
2932 def infer_axes(self) -> bool:
2933 """
2934 infer the axes of my storer
2935 return a boolean indicating if we have a valid storer or not
2936 """
2937 s = self.storable
2938 if s is None:
2939 return False
2940 self.get_attrs()
2941 return True
2943 def read(
2944 self,
2945 where=None,
2946 columns=None,
2947 start: int | None = None,
2948 stop: int | None = None,
2949 ) -> Series | DataFrame:
2950 raise NotImplementedError(
2951 "cannot read on an abstract storer: subclasses should implement"
2952 )
2954 def write(self, obj, **kwargs) -> None:
2955 raise NotImplementedError(
2956 "cannot write on an abstract storer: subclasses should implement"
2957 )
2959 def delete(
2960 self, where=None, start: int | None = None, stop: int | None = None
2961 ) -> int | None:
2962 """
2963 support fully deleting the node in its entirety (only) - where
2964 specification must be None
2965 """
2966 if com.all_none(where, start, stop):
2967 self._handle.remove_node(self.group, recursive=True)
2968 return None
2970 raise TypeError("cannot delete on an abstract storer")
2973class GenericFixed(Fixed):
2974 """a generified fixed version"""
2976 _index_type_map = {DatetimeIndex: "datetime", PeriodIndex: "period"}
2977 _reverse_index_map = {v: k for k, v in _index_type_map.items()}
2978 attributes: list[str] = []
2980 # indexer helpers
2981 def _class_to_alias(self, cls) -> str:
2982 return self._index_type_map.get(cls, "")
2984 def _alias_to_class(self, alias):
2985 if isinstance(alias, type): # pragma: no cover
2986 # compat: for a short period of time master stored types
2987 return alias
2988 return self._reverse_index_map.get(alias, Index)
2990 def _get_index_factory(self, attrs):
2991 index_class = self._alias_to_class(getattr(attrs, "index_class", ""))
2993 factory: Callable
2995 kwargs = {}
2996 if index_class == DatetimeIndex:
2998 def f(values, freq=None, tz=None):
2999 # data are already in UTC, localize and convert if tz present
3000 dta = DatetimeArray._simple_new(
3001 values.values, dtype=values.dtype, freq=freq
3002 )
3003 result = DatetimeIndex._simple_new(dta, name=None)
3004 if tz is not None:
3005 result = result.tz_localize("UTC").tz_convert(tz)
3006 return result
3008 factory = f
3009 elif index_class == PeriodIndex:
3011 def f(values, freq=None, tz=None):
3012 dtype = PeriodDtype(freq)
3013 parr = PeriodArray._simple_new(values, dtype=dtype)
3014 return PeriodIndex._simple_new(parr, name=None)
3016 factory = f
3017 else:
3018 factory = index_class
3019 kwargs["copy"] = False
3021 if "freq" in attrs:
3022 kwargs["freq"] = attrs["freq"]
3023 if index_class is Index:
3024 # DTI/PI would be gotten by _alias_to_class
3025 factory = TimedeltaIndex
3027 if "tz" in attrs:
3028 kwargs["tz"] = attrs["tz"]
3029 assert index_class is DatetimeIndex # just checking
3031 return factory, kwargs
3033 def validate_read(self, columns, where) -> None:
3034 """
3035 raise if any keywords are passed which are not-None
3036 """
3037 if columns is not None:
3038 raise TypeError(
3039 "cannot pass a column specification when reading "
3040 "a Fixed format store. this store must be selected in its entirety"
3041 )
3042 if where is not None:
3043 raise TypeError(
3044 "cannot pass a where specification when reading "
3045 "from a Fixed format store. this store must be selected in its entirety"
3046 )
3048 @property
3049 def is_exists(self) -> bool:
3050 return True
3052 def set_attrs(self) -> None:
3053 """set our object attributes"""
3054 self.attrs.encoding = self.encoding
3055 self.attrs.errors = self.errors
3057 def get_attrs(self) -> None:
3058 """retrieve our attributes"""
3059 self.encoding = _ensure_encoding(getattr(self.attrs, "encoding", None))
3060 self.errors = getattr(self.attrs, "errors", "strict")
3061 for n in self.attributes:
3062 setattr(self, n, getattr(self.attrs, n, None))
3064 def write(self, obj, **kwargs) -> None:
3065 self.set_attrs()
3067 def read_array(self, key: str, start: int | None = None, stop: int | None = None):
3068 """read an array for the specified node (off of group"""
3069 import tables
3071 node = getattr(self.group, key)
3072 attrs = node._v_attrs
3074 transposed = getattr(attrs, "transposed", False)
3076 if isinstance(node, tables.VLArray):
3077 ret = node[0][start:stop]
3078 dtype = getattr(attrs, "value_type", None)
3079 if dtype is not None:
3080 ret = pd_array(ret, dtype=dtype)
3081 else:
3082 dtype = getattr(attrs, "value_type", None)
3083 shape = getattr(attrs, "shape", None)
3085 if shape is not None:
3086 # length 0 axis
3087 ret = np.empty(shape, dtype=dtype)
3088 else:
3089 ret = node[start:stop]
3091 if dtype and dtype.startswith("datetime64"):
3092 # reconstruct a timezone if indicated
3093 if dtype == "datetime64":
3094 dtype = "datetime64[ns]"
3095 tz = getattr(attrs, "tz", None)
3096 ret = _set_tz(ret, tz, dtype)
3098 elif dtype and dtype.startswith("timedelta64"):
3099 if dtype == "timedelta64":
3100 # This was written back before we started writing
3101 # timedelta64 units
3102 ret = np.asarray(ret, dtype="m8[ns]")
3103 else:
3104 ret = np.asarray(ret, dtype=dtype)
3106 if transposed:
3107 return ret.T
3108 else:
3109 return ret
3111 def read_index(
3112 self, key: str, start: int | None = None, stop: int | None = None
3113 ) -> Index:
3114 variety = getattr(self.attrs, f"{key}_variety")
3116 if variety == "multi":
3117 return self.read_multi_index(key, start=start, stop=stop)
3118 elif variety == "regular":
3119 node = getattr(self.group, key)
3120 index = self.read_index_node(node, start=start, stop=stop)
3121 return index
3122 else: # pragma: no cover
3123 raise TypeError(f"unrecognized index variety: {variety}")
3125 def write_index(self, key: str, index: Index) -> None:
3126 if isinstance(index, MultiIndex):
3127 setattr(self.attrs, f"{key}_variety", "multi")
3128 self.write_multi_index(key, index)
3129 else:
3130 setattr(self.attrs, f"{key}_variety", "regular")
3131 converted = _convert_index("index", index, self.encoding, self.errors)
3133 self.write_array(key, converted.values)
3135 node = getattr(self.group, key)
3136 node._v_attrs.kind = converted.kind
3137 node._v_attrs.name = index.name
3139 if isinstance(index, (DatetimeIndex, PeriodIndex)):
3140 node._v_attrs.index_class = self._class_to_alias(type(index))
3142 if isinstance(index, (DatetimeIndex, PeriodIndex, TimedeltaIndex)):
3143 node._v_attrs.freq = index.freq
3145 if isinstance(index, DatetimeIndex) and index.tz is not None:
3146 node._v_attrs.tz = _get_tz(index.tz)
3148 def write_multi_index(self, key: str, index: MultiIndex) -> None:
3149 setattr(self.attrs, f"{key}_nlevels", index.nlevels)
3151 for i, (lev, level_codes, name) in enumerate(
3152 zip(index.levels, index.codes, index.names, strict=True)
3153 ):
3154 # write the level
3155 if isinstance(lev.dtype, ExtensionDtype) and not isinstance(
3156 lev.dtype, StringDtype
3157 ):
3158 raise NotImplementedError(
3159 "Saving a MultiIndex with an extension dtype is not supported."
3160 )
3161 level_key = f"{key}_level{i}"
3162 conv_level = _convert_index(level_key, lev, self.encoding, self.errors)
3163 self.write_array(level_key, conv_level.values)
3164 node = getattr(self.group, level_key)
3165 node._v_attrs.kind = conv_level.kind
3166 node._v_attrs.name = name
3168 # write the name
3169 setattr(node._v_attrs, f"{key}_name{name}", name)
3171 # write the labels
3172 label_key = f"{key}_label{i}"
3173 self.write_array(label_key, level_codes)
3175 def read_multi_index(
3176 self, key: str, start: int | None = None, stop: int | None = None
3177 ) -> MultiIndex:
3178 nlevels = getattr(self.attrs, f"{key}_nlevels")
3180 levels = []
3181 codes = []
3182 names: list[Hashable] = []
3183 for i in range(nlevels):
3184 level_key = f"{key}_level{i}"
3185 node = getattr(self.group, level_key)
3186 lev = self.read_index_node(node, start=start, stop=stop)
3187 levels.append(lev)
3188 names.append(lev.name)
3190 label_key = f"{key}_label{i}"
3191 level_codes = self.read_array(label_key, start=start, stop=stop)
3192 codes.append(level_codes)
3194 return MultiIndex(
3195 levels=levels, codes=codes, names=names, verify_integrity=True
3196 )
3198 def read_index_node(
3199 self, node: Node, start: int | None = None, stop: int | None = None
3200 ) -> Index:
3201 data = node[start:stop]
3202 # If the index was an empty array write_array_empty() will
3203 # have written a sentinel. Here we replace it with the original.
3204 if "shape" in node._v_attrs and np.prod(node._v_attrs.shape) == 0:
3205 data = np.empty(node._v_attrs.shape, dtype=node._v_attrs.value_type)
3206 kind = node._v_attrs.kind
3207 name = None
3209 if "name" in node._v_attrs:
3210 name = _ensure_str(node._v_attrs.name)
3212 attrs = node._v_attrs
3213 factory, kwargs = self._get_index_factory(attrs)
3215 if kind in ("date", "object"):
3216 index = factory(
3217 _unconvert_index(
3218 data, kind, encoding=self.encoding, errors=self.errors
3219 ),
3220 dtype=object,
3221 **kwargs,
3222 )
3223 else:
3224 try:
3225 index = factory(
3226 _unconvert_index(
3227 data, kind, encoding=self.encoding, errors=self.errors
3228 ),
3229 **kwargs,
3230 )
3231 except UnicodeEncodeError as err:
3232 if (
3233 self.errors == "surrogatepass"
3234 and using_string_dtype()
3235 and str(err).endswith("surrogates not allowed")
3236 and HAS_PYARROW
3237 ):
3238 index = factory(
3239 _unconvert_index(
3240 data, kind, encoding=self.encoding, errors=self.errors
3241 ),
3242 dtype=StringDtype(storage="python", na_value=np.nan),
3243 **kwargs,
3244 )
3245 else:
3246 raise
3248 index.name = name
3250 return index
3252 def write_array_empty(self, key: str, value: ArrayLike) -> None:
3253 """write a 0-len array"""
3254 # ugly hack for length 0 axes
3255 arr = np.empty((1,) * value.ndim)
3256 self._handle.create_array(self.group, key, arr)
3257 node = getattr(self.group, key)
3258 node._v_attrs.value_type = str(value.dtype)
3259 node._v_attrs.shape = value.shape
3261 def write_array(
3262 self, key: str, obj: AnyArrayLike, items: Index | None = None
3263 ) -> None:
3264 value = extract_array(obj, extract_numpy=True)
3266 if key in self.group:
3267 self._handle.remove_node(self.group, key)
3269 # Transform needed to interface with pytables row/col notation
3270 empty_array = value.size == 0
3271 transposed = False
3273 if isinstance(value.dtype, CategoricalDtype):
3274 raise NotImplementedError(
3275 "Cannot store a category dtype in an HDF5 dataset that uses format="
3276 '"fixed". Use format="table".'
3277 )
3278 if not empty_array:
3279 if hasattr(value, "T"):
3280 # ExtensionArrays (1d) may not have transpose.
3281 value = value.T
3282 transposed = True
3284 if isinstance(value, BaseStringArray):
3285 # GH#64180: BaseStringArray must use the VLArray path.
3286 # Atom.from_dtype does not handle ExtensionDtype.
3287 vlarr = self._handle.create_vlarray(
3288 self.group, key, _tables().ObjectAtom(), filters=self._filters
3289 )
3290 vlarr.append(value.to_numpy())
3291 node = getattr(self.group, key)
3292 node._v_attrs.value_type = str(value.dtype)
3294 else:
3295 atom = None
3296 if self._filters is not None:
3297 with suppress(ValueError):
3298 # get the atom for this datatype
3299 atom = _tables().Atom.from_dtype(value.dtype)
3301 if atom is not None:
3302 # We only get here if self._filters is non-None and
3303 # the Atom.from_dtype call succeeded
3305 # create an empty chunked array and fill it from value
3306 if not empty_array:
3307 ca = self._handle.create_carray(
3308 self.group, key, atom, value.shape, filters=self._filters
3309 )
3310 ca[:] = value
3312 else:
3313 self.write_array_empty(key, value)
3315 elif value.dtype.type == np.object_:
3316 # infer the type, warn if we have a non-string type here
3317 # (for performance)
3318 inferred_type = lib.infer_dtype(value, skipna=False)
3319 if empty_array:
3320 pass
3321 elif inferred_type == "string":
3322 pass
3323 elif get_option("performance_warnings"):
3324 ws = performance_doc % (inferred_type, key, items)
3325 warnings.warn(ws, PerformanceWarning, stacklevel=find_stack_level())
3327 vlarr = self._handle.create_vlarray(
3328 self.group, key, _tables().ObjectAtom()
3329 )
3330 vlarr.append(value)
3332 elif lib.is_np_dtype(value.dtype, "M"):
3333 self._handle.create_array(self.group, key, value.view("i8"))
3334 getattr(self.group, key)._v_attrs.value_type = str(value.dtype)
3335 elif isinstance(value.dtype, DatetimeTZDtype):
3336 # store as UTC
3337 # with a zone
3339 # error: "ExtensionArray" has no attribute "asi8"
3340 self._handle.create_array(
3341 self.group,
3342 key,
3343 value.asi8, # type: ignore[attr-defined]
3344 )
3346 node = getattr(self.group, key)
3347 # error: "ExtensionArray" has no attribute "tz"
3348 node._v_attrs.tz = _get_tz(value.tz) # type: ignore[attr-defined]
3349 node._v_attrs.value_type = f"datetime64[{value.dtype.unit}]"
3350 elif lib.is_np_dtype(value.dtype, "m"):
3351 self._handle.create_array(self.group, key, value.view("i8"))
3352 getattr(self.group, key)._v_attrs.value_type = str(value.dtype)
3353 elif empty_array:
3354 self.write_array_empty(key, value)
3355 else:
3356 self._handle.create_array(self.group, key, value)
3358 getattr(self.group, key)._v_attrs.transposed = transposed
3361class SeriesFixed(GenericFixed):
3362 pandas_kind = "series"
3363 attributes = ["name"]
3365 name: Hashable
3367 @property
3368 def shape(self) -> tuple[int] | None:
3369 try:
3370 return (len(self.group.values),)
3371 except (TypeError, AttributeError):
3372 return None
3374 def read(
3375 self,
3376 where=None,
3377 columns=None,
3378 start: int | None = None,
3379 stop: int | None = None,
3380 ) -> Series:
3381 self.validate_read(columns, where)
3382 index = self.read_index("index", start=start, stop=stop)
3383 values = self.read_array("values", start=start, stop=stop)
3384 try:
3385 result = Series(values, index=index, name=self.name, copy=False)
3386 except UnicodeEncodeError as err:
3387 if (
3388 self.errors == "surrogatepass"
3389 and using_string_dtype()
3390 and str(err).endswith("surrogates not allowed")
3391 and HAS_PYARROW
3392 ):
3393 result = Series(
3394 values,
3395 index=index,
3396 name=self.name,
3397 copy=False,
3398 dtype=StringDtype(storage="python", na_value=np.nan),
3399 )
3400 else:
3401 raise
3402 return result
3404 def write(self, obj, **kwargs) -> None:
3405 super().write(obj, **kwargs)
3406 self.write_index("index", obj.index)
3407 self.write_array("values", obj)
3408 self.attrs.name = obj.name
3411class BlockManagerFixed(GenericFixed):
3412 attributes = ["ndim", "nblocks"]
3414 nblocks: int
3416 @property
3417 def shape(self) -> list[int] | None:
3418 try:
3419 ndim = self.ndim
3421 # items
3422 items = 0
3423 for i in range(self.nblocks):
3424 node = getattr(self.group, f"block{i}_items")
3425 shape = getattr(node, "shape", None)
3426 if shape is not None:
3427 items += shape[0]
3429 # data shape
3430 node = self.group.block0_values
3431 shape = getattr(node, "shape", None)
3432 if shape is not None:
3433 shape = list(shape[0 : (ndim - 1)])
3434 else:
3435 shape = []
3437 shape.append(items)
3439 return shape
3440 except AttributeError:
3441 return None
3443 def read(
3444 self,
3445 where=None,
3446 columns=None,
3447 start: int | None = None,
3448 stop: int | None = None,
3449 ) -> DataFrame:
3450 # start, stop applied to rows, so 0th axis only
3451 self.validate_read(columns, where)
3452 select_axis = self.obj_type()._get_block_manager_axis(0)
3454 axes = []
3455 for i in range(self.ndim):
3456 _start, _stop = (start, stop) if i == select_axis else (None, None)
3457 ax = self.read_index(f"axis{i}", start=_start, stop=_stop)
3458 axes.append(ax)
3460 items = axes[0]
3461 dfs = []
3463 for i in range(self.nblocks):
3464 blk_items = self.read_index(f"block{i}_items")
3465 values = self.read_array(f"block{i}_values", start=_start, stop=_stop)
3467 columns = items[items.get_indexer(blk_items)]
3468 df = DataFrame(values.T, columns=columns, index=axes[1], copy=False)
3469 if (
3470 using_string_dtype()
3471 and isinstance(values, np.ndarray)
3472 and is_string_array(values, skipna=True)
3473 ):
3474 df = df.astype(StringDtype(na_value=np.nan))
3475 dfs.append(df)
3477 if len(dfs) > 0:
3478 out = concat(dfs, axis=1).copy()
3479 return out.reindex(columns=items)
3481 return DataFrame(columns=axes[0], index=axes[1])
3483 def write(self, obj, **kwargs) -> None:
3484 super().write(obj, **kwargs)
3486 data = obj._mgr
3487 if not data.is_consolidated():
3488 data = data.consolidate()
3490 self.attrs.ndim = data.ndim
3491 for i, ax in enumerate(data.axes):
3492 if i == 0 and (not ax.is_unique):
3493 raise ValueError("Columns index has to be unique for fixed format")
3494 self.write_index(f"axis{i}", ax)
3496 # Supporting mixed-type DataFrame objects...nontrivial
3497 self.attrs.nblocks = len(data.blocks)
3498 for i, blk in enumerate(data.blocks):
3499 # I have no idea why, but writing values before items fixed #2299
3500 blk_items = data.items.take(blk.mgr_locs)
3501 self.write_array(f"block{i}_values", blk.values, items=blk_items)
3502 self.write_index(f"block{i}_items", blk_items)
3505class FrameFixed(BlockManagerFixed):
3506 pandas_kind = "frame"
3507 obj_type = DataFrame
3510class Table(Fixed):
3511 """
3512 represent a table:
3513 facilitate read/write of various types of tables
3515 Attrs in Table Node
3516 -------------------
3517 These are attributes that are store in the main table node, they are
3518 necessary to recreate these tables when read back in.
3520 index_axes : a list of tuples of the (original indexing axis and
3521 index column)
3522 non_index_axes: a list of tuples of the (original index axis and
3523 columns on a non-indexing axis)
3524 values_axes : a list of the columns which comprise the data of this
3525 table
3526 data_columns : a list of the columns that we are allowing indexing
3527 (these become single columns in values_axes)
3528 nan_rep : the string to use for nan representations for string
3529 objects
3530 levels : the names of levels
3531 metadata : the names of the metadata columns
3532 """
3534 pandas_kind = "wide_table"
3535 format_type: str = "table" # GH#30962 needed by dask
3536 table_type: str
3537 levels: int | list[Hashable] = 1
3538 is_table = True
3540 metadata: list
3542 def __init__(
3543 self,
3544 parent: HDFStore,
3545 group: Node,
3546 encoding: str | None = None,
3547 errors: str = "strict",
3548 index_axes: list[IndexCol] | None = None,
3549 non_index_axes: list[tuple[AxisInt, Any]] | None = None,
3550 values_axes: list[DataCol] | None = None,
3551 data_columns: list | None = None,
3552 info: dict | None = None,
3553 nan_rep=None,
3554 ) -> None:
3555 super().__init__(parent, group, encoding=encoding, errors=errors)
3556 self.index_axes = index_axes or []
3557 self.non_index_axes = non_index_axes or []
3558 self.values_axes = values_axes or []
3559 self.data_columns = data_columns or []
3560 self.info = info or {}
3561 self.nan_rep = nan_rep
3563 @property
3564 def table_type_short(self) -> str:
3565 return self.table_type.split("_")[0]
3567 def __repr__(self) -> str:
3568 """return a pretty representation of myself"""
3569 self.infer_axes()
3570 jdc = ",".join(self.data_columns) if len(self.data_columns) else ""
3571 dc = f",dc->[{jdc}]"
3573 ver = ""
3574 if self.is_old_version:
3575 jver = ".".join([str(x) for x in self.version])
3576 ver = f"[{jver}]"
3578 jindex_axes = ",".join([a.name for a in self.index_axes])
3579 return (
3580 f"{self.pandas_type:12.12}{ver} "
3581 f"(typ->{self.table_type_short},nrows->{self.nrows},"
3582 f"ncols->{self.ncols},indexers->[{jindex_axes}]{dc})"
3583 )
3585 def __getitem__(self, c: str):
3586 """return the axis for c"""
3587 for a in self.axes:
3588 if c == a.name:
3589 return a
3590 return None
3592 def validate(self, other) -> None:
3593 """validate against an existing table"""
3594 if other is None:
3595 return
3597 if other.table_type != self.table_type:
3598 raise TypeError(
3599 "incompatible table_type with existing "
3600 f"[{other.table_type} - {self.table_type}]"
3601 )
3603 for c in ["index_axes", "non_index_axes", "values_axes"]:
3604 sv = getattr(self, c, None)
3605 ov = getattr(other, c, None)
3606 if sv != ov:
3607 # show the error for the specific axes
3608 # Argument 1 to "enumerate" has incompatible type
3609 # "Optional[Any]"; expected "Iterable[Any]" [arg-type]
3610 for i, sax in enumerate(sv): # type: ignore[arg-type]
3611 # Value of type "Optional[Any]" is not indexable [index]
3612 oax = ov[i] # type: ignore[index]
3613 if sax != oax:
3614 if c == "values_axes" and sax.kind != oax.kind:
3615 raise ValueError(
3616 f"Cannot serialize the column [{oax.values[0]}] "
3617 f"because its data contents are not [{sax.kind}] "
3618 f"but [{oax.kind}] object dtype"
3619 )
3620 raise ValueError(
3621 f"invalid combination of [{c}] on appending data "
3622 f"[{sax}] vs current table [{oax}]"
3623 )
3625 # should never get here
3626 raise Exception(
3627 f"invalid combination of [{c}] on appending data [{sv}] vs "
3628 f"current table [{ov}]"
3629 )
3631 @property
3632 def is_multi_index(self) -> bool:
3633 """the levels attribute is 1 or a list in the case of a multi-index"""
3634 return isinstance(self.levels, list)
3636 def validate_multiindex(
3637 self, obj: DataFrame | Series
3638 ) -> tuple[DataFrame, list[Hashable]]:
3639 """
3640 validate that we can store the multi-index; reset and return the
3641 new object
3642 """
3643 levels = com.fill_missing_names(obj.index.names)
3644 try:
3645 reset_obj = obj.reset_index()
3646 except ValueError as err:
3647 raise ValueError(
3648 "duplicate names/columns in the multi-index when storing as a table"
3649 ) from err
3650 assert isinstance(reset_obj, DataFrame) # for mypy
3651 return reset_obj, levels
3653 @property
3654 def nrows_expected(self) -> int:
3655 """based on our axes, compute the expected nrows"""
3656 return np.prod([i.cvalues.shape[0] for i in self.index_axes])
3658 @property
3659 def is_exists(self) -> bool:
3660 """has this table been created"""
3661 return "table" in self.group
3663 @property
3664 def storable(self):
3665 return getattr(self.group, "table", None)
3667 @property
3668 def table(self):
3669 """return the table group (this is my storable)"""
3670 return self.storable
3672 @property
3673 def dtype(self):
3674 return self.table.dtype
3676 @property
3677 def description(self):
3678 return self.table.description
3680 @property
3681 def axes(self) -> itertools.chain[IndexCol]:
3682 return itertools.chain(self.index_axes, self.values_axes)
3684 @property
3685 def ncols(self) -> int:
3686 """the number of total columns in the values axes"""
3687 return sum(len(a.values) for a in self.values_axes)
3689 @property
3690 def is_transposed(self) -> bool:
3691 return False
3693 @property
3694 def data_orientation(self) -> tuple[int, ...]:
3695 """return a tuple of my permutated axes, non_indexable at the front"""
3696 return tuple(
3697 itertools.chain(
3698 [int(a[0]) for a in self.non_index_axes],
3699 [int(a.axis) for a in self.index_axes],
3700 )
3701 )
3703 def queryables(self) -> dict[str, Any]:
3704 """return a dict of the kinds allowable columns for this object"""
3705 # mypy doesn't recognize DataFrame._AXIS_NAMES, so we re-write it here
3706 axis_names = {0: "index", 1: "columns"}
3708 # compute the values_axes queryables
3709 d1 = [(a.cname, a) for a in self.index_axes]
3710 d2 = [(axis_names[axis], None) for axis, values in self.non_index_axes]
3711 d3 = [
3712 (v.cname, v) for v in self.values_axes if v.name in set(self.data_columns)
3713 ]
3715 return dict(d1 + d2 + d3)
3717 def index_cols(self) -> list[tuple[Any, Any]]:
3718 """return a list of my index cols"""
3719 # Note: each `i.cname` below is assured to be a str.
3720 return [(i.axis, i.cname) for i in self.index_axes]
3722 def values_cols(self) -> list[str]:
3723 """return a list of my values cols"""
3724 return [i.cname for i in self.values_axes]
3726 def _get_metadata_path(self, key: str) -> str:
3727 """return the metadata pathname for this key"""
3728 group = self.group._v_pathname
3729 return f"{group}/meta/{key}/meta"
3731 def write_metadata(self, key: str, values: np.ndarray) -> None:
3732 """
3733 Write out a metadata array to the key as a fixed-format Series.
3735 Parameters
3736 ----------
3737 key : str
3738 values : ndarray
3739 """
3740 self.parent.put(
3741 self._get_metadata_path(key),
3742 Series(values, copy=False),
3743 format="table",
3744 encoding=self.encoding,
3745 errors=self.errors,
3746 nan_rep=self.nan_rep,
3747 )
3749 def read_metadata(self, key: str):
3750 """return the meta data array for this key"""
3751 if getattr(getattr(self.group, "meta", None), key, None) is not None:
3752 return self.parent.select(self._get_metadata_path(key))
3753 return None
3755 def set_attrs(self) -> None:
3756 """set our table type & indexables"""
3757 self.attrs.table_type = str(self.table_type)
3758 self.attrs.index_cols = self.index_cols()
3759 self.attrs.values_cols = self.values_cols()
3760 self.attrs.non_index_axes = self.non_index_axes
3761 self.attrs.data_columns = self.data_columns
3762 self.attrs.nan_rep = self.nan_rep
3763 self.attrs.encoding = self.encoding
3764 self.attrs.errors = self.errors
3765 self.attrs.levels = self.levels
3766 self.attrs.info = self.info
3768 def get_attrs(self) -> None:
3769 """retrieve our attributes"""
3770 self.non_index_axes = getattr(self.attrs, "non_index_axes", None) or []
3771 self.data_columns = getattr(self.attrs, "data_columns", None) or []
3772 self.info = getattr(self.attrs, "info", None) or {}
3773 self.nan_rep = getattr(self.attrs, "nan_rep", None)
3774 self.encoding = _ensure_encoding(getattr(self.attrs, "encoding", None))
3775 self.errors = getattr(self.attrs, "errors", "strict")
3776 self.levels: list[Hashable] = getattr(self.attrs, "levels", None) or []
3777 self.index_axes = [a for a in self.indexables if a.is_an_indexable]
3778 self.values_axes = [a for a in self.indexables if not a.is_an_indexable]
3780 def validate_version(self, where=None) -> None:
3781 """are we trying to operate on an old version?"""
3782 if where is not None:
3783 if self.is_old_version:
3784 ws = incompatibility_doc % ".".join([str(x) for x in self.version])
3785 warnings.warn(
3786 ws,
3787 IncompatibilityWarning,
3788 stacklevel=find_stack_level(),
3789 )
3791 def validate_min_itemsize(self, min_itemsize) -> None:
3792 """
3793 validate the min_itemsize doesn't contain items that are not in the
3794 axes this needs data_columns to be defined
3795 """
3796 if min_itemsize is None:
3797 return
3798 if not isinstance(min_itemsize, dict):
3799 return
3801 q = self.queryables()
3802 for k in min_itemsize:
3803 # ok, apply generally
3804 if k == "values":
3805 continue
3806 if k not in q:
3807 raise ValueError(
3808 f"min_itemsize has the key [{k}] which is not an axis or "
3809 "data_column"
3810 )
3812 @cache_readonly
3813 def indexables(self):
3814 """create/cache the indexables if they don't exist"""
3815 _indexables = []
3817 desc = self.description
3818 table_attrs = self.table.attrs
3820 # Note: each of the `name` kwargs below are str, ensured
3821 # by the definition in index_cols.
3822 # index columns
3823 for i, (axis, name) in enumerate(self.attrs.index_cols):
3824 atom = getattr(desc, name)
3825 md = self.read_metadata(name)
3826 meta = "category" if md is not None else None
3828 kind_attr = f"{name}_kind"
3829 kind = getattr(table_attrs, kind_attr, None)
3831 index_col = IndexCol(
3832 name=name,
3833 axis=axis,
3834 pos=i,
3835 kind=kind,
3836 typ=atom,
3837 table=self.table,
3838 meta=meta,
3839 metadata=md,
3840 )
3841 _indexables.append(index_col)
3843 # values columns
3844 dc = set(self.data_columns)
3845 base_pos = len(_indexables)
3847 def f(i, c: str) -> DataCol:
3848 assert isinstance(c, str)
3849 klass = DataCol
3850 if c in dc:
3851 klass = DataIndexableCol
3853 atom = getattr(desc, c)
3854 adj_name = _maybe_adjust_name(c, self.version)
3856 # TODO: why kind_attr here?
3857 values = getattr(table_attrs, f"{adj_name}_kind", None)
3858 dtype = getattr(table_attrs, f"{adj_name}_dtype", None)
3859 # Argument 1 to "_dtype_to_kind" has incompatible type
3860 # "Optional[Any]"; expected "str" [arg-type]
3861 kind = _dtype_to_kind(dtype) # type: ignore[arg-type]
3863 md = self.read_metadata(c)
3864 # TODO: figure out why these two versions of `meta` dont always match.
3865 # meta = "category" if md is not None else None
3866 meta = getattr(table_attrs, f"{adj_name}_meta", None)
3868 obj = klass(
3869 name=adj_name,
3870 cname=c,
3871 values=values,
3872 kind=kind,
3873 pos=base_pos + i,
3874 typ=atom,
3875 table=self.table,
3876 meta=meta,
3877 metadata=md,
3878 dtype=dtype,
3879 )
3880 return obj
3882 # Note: the definition of `values_cols` ensures that each
3883 # `c` below is a str.
3884 _indexables.extend([f(i, c) for i, c in enumerate(self.attrs.values_cols)])
3886 return _indexables
3888 def create_index(
3889 self, columns=None, optlevel=None, kind: str | None = None
3890 ) -> None:
3891 """
3892 Create a pytables index on the specified columns.
3894 Parameters
3895 ----------
3896 columns : None, bool, or listlike[str]
3897 Indicate which columns to create an index on.
3899 * False : Do not create any indexes.
3900 * True : Create indexes on all columns.
3901 * None : Create indexes on all columns.
3902 * listlike : Create indexes on the given columns.
3904 optlevel : int or None, default None
3905 Optimization level, if None, pytables defaults to 6.
3906 kind : str or None, default None
3907 Kind of index, if None, pytables defaults to "medium".
3909 Raises
3910 ------
3911 TypeError if trying to create an index on a complex-type column.
3913 Notes
3914 -----
3915 Cannot index Time64Col or ComplexCol.
3916 Pytables must be >= 3.0.
3917 """
3918 if not self.infer_axes():
3919 return
3920 if columns is False:
3921 return
3923 # index all indexables and data_columns
3924 if columns is None or columns is True:
3925 columns = [a.cname for a in self.axes if a.is_data_indexable]
3926 if not isinstance(columns, (tuple, list)):
3927 columns = [columns]
3929 kw = {}
3930 if optlevel is not None:
3931 kw["optlevel"] = optlevel
3932 if kind is not None:
3933 kw["kind"] = kind
3935 table = self.table
3936 for c in columns:
3937 v = getattr(table.cols, c, None)
3938 if v is not None:
3939 # remove the index if the kind/optlevel have changed
3940 if v.is_indexed:
3941 index = v.index
3942 cur_optlevel = index.optlevel
3943 cur_kind = index.kind
3945 if kind is not None and cur_kind != kind:
3946 v.remove_index()
3947 else:
3948 kw["kind"] = cur_kind
3950 if optlevel is not None and cur_optlevel != optlevel:
3951 v.remove_index()
3952 else:
3953 kw["optlevel"] = cur_optlevel
3955 # create the index
3956 if not v.is_indexed:
3957 if v.type.startswith("complex"):
3958 raise TypeError(
3959 "Columns containing complex values can be stored but "
3960 "cannot be indexed when using table format. Either use "
3961 "fixed format, set index=False, or do not include "
3962 "the columns containing complex values to "
3963 "data_columns when initializing the table."
3964 )
3965 v.create_index(**kw)
3966 elif c in self.non_index_axes[0][1]:
3967 # GH 28156
3968 raise AttributeError(
3969 f"column {c} is not a data_column.\n"
3970 f"In order to read column {c} you must reload the dataframe \n"
3971 f"into HDFStore and include {c} with the data_columns argument."
3972 )
3974 def _read_axes(
3975 self, where, start: int | None = None, stop: int | None = None
3976 ) -> list[tuple[np.ndarray, np.ndarray] | tuple[Index, Index]]:
3977 """
3978 Create the axes sniffed from the table.
3980 Parameters
3981 ----------
3982 where : ???
3983 start : int or None, default None
3984 stop : int or None, default None
3986 Returns
3987 -------
3988 List[Tuple[index_values, column_values]]
3989 """
3990 # create the selection
3991 selection = Selection(self, where=where, start=start, stop=stop)
3992 values = selection.select()
3994 results = []
3995 # convert the data
3996 for a in self.axes:
3997 a.set_info(self.info)
3998 res = a.convert(
3999 values,
4000 nan_rep=self.nan_rep,
4001 encoding=self.encoding,
4002 errors=self.errors,
4003 )
4004 results.append(res)
4006 return results
4008 @classmethod
4009 def get_object(cls, obj, transposed: bool):
4010 """return the data for this obj"""
4011 return obj
4013 def validate_data_columns(self, data_columns, min_itemsize, non_index_axes) -> list:
4014 """
4015 take the input data_columns and min_itemize and create a data
4016 columns spec
4017 """
4018 if not len(non_index_axes):
4019 return []
4021 axis, axis_labels = non_index_axes[0]
4022 info = self.info.get(axis, {})
4023 if info.get("type") == "MultiIndex" and data_columns:
4024 raise ValueError(
4025 f"cannot use a multi-index on axis [{axis}] with "
4026 f"data_columns {data_columns}"
4027 )
4029 # evaluate the passed data_columns, True == use all columns
4030 # take only valid axis labels
4031 if data_columns is True:
4032 data_columns = list(axis_labels)
4033 elif data_columns is None:
4034 data_columns = []
4036 # if min_itemsize is a dict, add the keys (exclude 'values')
4037 if isinstance(min_itemsize, dict):
4038 existing_data_columns = set(data_columns)
4039 data_columns = list(data_columns) # ensure we do not modify
4040 data_columns.extend(
4041 [
4042 k
4043 for k in min_itemsize.keys()
4044 if k != "values" and k not in existing_data_columns
4045 ]
4046 )
4048 # return valid columns in the order of our axis
4049 return [c for c in data_columns if c in axis_labels]
4051 def _create_axes(
4052 self,
4053 axes,
4054 obj: DataFrame,
4055 validate: bool = True,
4056 nan_rep=None,
4057 data_columns=None,
4058 min_itemsize=None,
4059 ):
4060 """
4061 Create and return the axes.
4063 Parameters
4064 ----------
4065 axes: list or None
4066 The names or numbers of the axes to create.
4067 obj : DataFrame
4068 The object to create axes on.
4069 validate: bool, default True
4070 Whether to validate the obj against an existing object already written.
4071 nan_rep :
4072 A value to use for string column nan_rep.
4073 data_columns : List[str], True, or None, default None
4074 Specify the columns that we want to create to allow indexing on.
4076 * True : Use all available columns.
4077 * None : Use no columns.
4078 * List[str] : Use the specified columns.
4080 min_itemsize: Dict[str, int] or None, default None
4081 The min itemsize for a column in bytes.
4082 """
4083 if not isinstance(obj, DataFrame):
4084 group = self.group._v_name
4085 raise TypeError(
4086 f"cannot properly create the storer for: [group->{group},"
4087 f"value->{type(obj)}]"
4088 )
4090 # set the default axes if needed
4091 if axes is None:
4092 axes = [0]
4094 # map axes to numbers
4095 axes = [obj._get_axis_number(a) for a in axes]
4097 # do we have an existing table (if so, use its axes & data_columns)
4098 if self.infer_axes():
4099 table_exists = True
4100 axes = [a.axis for a in self.index_axes]
4101 data_columns = list(self.data_columns)
4102 nan_rep = self.nan_rep
4103 # TODO: do we always have validate=True here?
4104 else:
4105 table_exists = False
4107 new_info = self.info
4109 assert self.ndim == 2 # with next check, we must have len(axes) == 1
4110 # currently support on ndim-1 axes
4111 if len(axes) != self.ndim - 1:
4112 raise ValueError(
4113 "currently only support ndim-1 indexers in an AppendableTable"
4114 )
4116 # create according to the new data
4117 new_non_index_axes: list = []
4119 # nan_representation
4120 if nan_rep is None:
4121 nan_rep = "nan"
4123 # We construct the non-index-axis first, since that alters new_info
4124 idx = next(x for x in [0, 1] if x not in axes)
4126 a = obj.axes[idx]
4127 # we might be able to change the axes on the appending data if necessary
4128 append_axis = list(a)
4129 if table_exists:
4130 indexer = len(new_non_index_axes) # i.e. 0
4131 exist_axis = self.non_index_axes[indexer][1]
4132 if not array_equivalent(
4133 np.array(append_axis),
4134 np.array(exist_axis),
4135 strict_nan=True,
4136 dtype_equal=True,
4137 ):
4138 # ahah! -> reindex
4139 if array_equivalent(
4140 np.array(sorted(append_axis)),
4141 np.array(sorted(exist_axis)),
4142 strict_nan=True,
4143 dtype_equal=True,
4144 ):
4145 append_axis = exist_axis
4147 # the non_index_axes info
4148 info = new_info.setdefault(idx, {})
4149 info["names"] = list(a.names)
4150 info["type"] = type(a).__name__
4152 new_non_index_axes.append((idx, append_axis))
4154 # Now we can construct our new index axis
4155 idx = axes[0]
4156 a = obj.axes[idx]
4157 axis_name = obj._get_axis_name(idx)
4158 new_index = _convert_index(axis_name, a, self.encoding, self.errors)
4159 new_index.axis = idx
4161 # Because we are always 2D, there is only one new_index, so
4162 # we know it will have pos=0
4163 new_index.set_pos(0)
4164 new_index.update_info(new_info)
4165 new_index.maybe_set_size(min_itemsize) # check for column conflicts
4167 new_index_axes = [new_index]
4168 j = len(new_index_axes) # i.e. 1
4169 assert j == 1
4171 # reindex by our non_index_axes & compute data_columns
4172 assert len(new_non_index_axes) == 1
4173 for a in new_non_index_axes:
4174 obj = _reindex_axis(obj, a[0], a[1])
4176 transposed = new_index.axis == 1
4178 # figure out data_columns and get out blocks
4179 data_columns = self.validate_data_columns(
4180 data_columns, min_itemsize, new_non_index_axes
4181 )
4183 frame = self.get_object(obj, transposed)._consolidate()
4185 blocks, blk_items = self._get_blocks_and_items(
4186 frame, table_exists, new_non_index_axes, self.values_axes, data_columns
4187 )
4189 # add my values
4190 vaxes = []
4191 for i, (blk, b_items) in enumerate(zip(blocks, blk_items, strict=True)):
4192 # shape of the data column are the indexable axes
4193 klass = DataCol
4194 name = None
4196 # we have a data_column
4197 if data_columns and len(b_items) == 1 and b_items[0] in data_columns:
4198 klass = DataIndexableCol
4199 name = b_items[0]
4200 if not (name is None or isinstance(name, str)):
4201 # TODO: should the message here be more specifically non-str?
4202 raise ValueError("cannot have non-object label DataIndexableCol")
4204 # make sure that we match up the existing columns
4205 # if we have an existing table
4206 existing_col: DataCol | None
4208 if table_exists and validate:
4209 try:
4210 existing_col = self.values_axes[i]
4211 except (IndexError, KeyError) as err:
4212 raise ValueError(
4213 f"Incompatible appended table [{blocks}]"
4214 f"with existing table [{self.values_axes}]"
4215 ) from err
4216 else:
4217 existing_col = None
4219 new_name = name or f"values_block_{i}"
4220 data_converted = _maybe_convert_for_string_atom(
4221 new_name,
4222 blk.values,
4223 existing_col=existing_col,
4224 min_itemsize=min_itemsize,
4225 nan_rep=nan_rep,
4226 encoding=self.encoding,
4227 errors=self.errors,
4228 columns=b_items,
4229 )
4230 adj_name = _maybe_adjust_name(new_name, self.version)
4232 typ = klass._get_atom(data_converted)
4233 kind = _dtype_to_kind(data_converted.dtype.name)
4234 tz = None
4235 if getattr(data_converted, "tz", None) is not None:
4236 tz = _get_tz(data_converted.tz)
4238 meta = metadata = ordered = None
4239 if isinstance(data_converted.dtype, CategoricalDtype):
4240 ordered = data_converted.ordered
4241 meta = "category"
4242 metadata = np.asarray(data_converted.categories).ravel()
4243 elif isinstance(blk.dtype, StringDtype):
4244 meta = str(blk.dtype)
4246 data, dtype_name = _get_data_and_dtype_name(data_converted)
4248 col = klass(
4249 name=adj_name,
4250 cname=new_name,
4251 values=list(b_items),
4252 typ=typ,
4253 pos=j,
4254 kind=kind,
4255 tz=tz,
4256 ordered=ordered,
4257 meta=meta,
4258 metadata=metadata,
4259 dtype=dtype_name,
4260 data=data,
4261 )
4262 col.update_info(new_info)
4264 vaxes.append(col)
4266 j += 1
4268 dcs = [col.name for col in vaxes if col.is_data_indexable]
4270 new_table = type(self)(
4271 parent=self.parent,
4272 group=self.group,
4273 encoding=self.encoding,
4274 errors=self.errors,
4275 index_axes=new_index_axes,
4276 non_index_axes=new_non_index_axes,
4277 values_axes=vaxes,
4278 data_columns=dcs,
4279 info=new_info,
4280 nan_rep=nan_rep,
4281 )
4282 if hasattr(self, "levels"):
4283 # TODO: get this into constructor, only for appropriate subclass
4284 new_table.levels = self.levels
4286 new_table.validate_min_itemsize(min_itemsize)
4288 if validate and table_exists:
4289 new_table.validate(self)
4291 return new_table
4293 @staticmethod
4294 def _get_blocks_and_items(
4295 frame: DataFrame,
4296 table_exists: bool,
4297 new_non_index_axes,
4298 values_axes,
4299 data_columns,
4300 ):
4301 # Helper to clarify non-state-altering parts of _create_axes
4302 def get_blk_items(mgr):
4303 return [mgr.items.take(blk.mgr_locs) for blk in mgr.blocks]
4305 mgr = frame._mgr
4306 blocks: list[Block] = list(mgr.blocks)
4307 blk_items: list[Index] = get_blk_items(mgr)
4309 if len(data_columns):
4310 # TODO: prove that we only get here with axis == 1?
4311 # It is the case in all extant tests, but NOT the case
4312 # outside this `if len(data_columns)` check.
4314 axis, axis_labels = new_non_index_axes[0]
4315 new_labels = Index(axis_labels).difference(Index(data_columns))
4316 mgr = frame.reindex(new_labels, axis=axis)._mgr
4318 blocks = list(mgr.blocks)
4319 blk_items = get_blk_items(mgr)
4320 for c in data_columns:
4321 # This reindex would raise ValueError if we had a duplicate
4322 # index, so we can infer that (as long as axis==1) we
4323 # get a single column back, so a single block.
4324 mgr = frame.reindex([c], axis=axis)._mgr
4325 blocks.extend(mgr.blocks)
4326 blk_items.extend(get_blk_items(mgr))
4328 # reorder the blocks in the same order as the existing table if we can
4329 if table_exists:
4330 by_items = {
4331 tuple(b_items.tolist()): (b, b_items)
4332 for b, b_items in zip(blocks, blk_items, strict=True)
4333 }
4334 new_blocks: list[Block] = []
4335 new_blk_items = []
4336 for ea in values_axes:
4337 items = tuple(ea.values)
4338 try:
4339 b, b_items = by_items.pop(items)
4340 new_blocks.append(b)
4341 new_blk_items.append(b_items)
4342 except (IndexError, KeyError) as err:
4343 jitems = ",".join([pprint_thing(item) for item in items])
4344 raise ValueError(
4345 f"cannot match existing table structure for [{jitems}] "
4346 "on appending data"
4347 ) from err
4348 blocks = new_blocks
4349 blk_items = new_blk_items
4351 return blocks, blk_items
4353 def process_axes(self, obj, selection: Selection, columns=None) -> DataFrame:
4354 """process axes filters"""
4355 # make a copy to avoid side effects
4356 if columns is not None:
4357 columns = list(columns)
4359 # make sure to include levels if we have them
4360 if columns is not None and self.is_multi_index:
4361 assert isinstance(self.levels, list) # assured by is_multi_index
4362 for n in self.levels:
4363 if n not in columns:
4364 columns.insert(0, n)
4366 # reorder by any non_index_axes & limit to the select columns
4367 for axis, labels in self.non_index_axes:
4368 obj = _reindex_axis(obj, axis, labels, columns)
4370 def process_filter(field, filt, op):
4371 for axis_name in obj._AXIS_ORDERS:
4372 axis_number = obj._get_axis_number(axis_name)
4373 axis_values = obj._get_axis(axis_name)
4374 assert axis_number is not None
4376 # see if the field is the name of an axis
4377 if field == axis_name:
4378 # if we have a multi-index, then need to include
4379 # the levels
4380 if self.is_multi_index:
4381 filt = filt.union(Index(self.levels))
4383 takers = op(axis_values, filt)
4384 return obj.loc(axis=axis_number)[takers]
4386 # this might be the name of a file IN an axis
4387 elif field in axis_values:
4388 # we need to filter on this dimension
4389 values = ensure_index(getattr(obj, field).values)
4390 filt = ensure_index(filt)
4392 # hack until we support reversed dim flags
4393 if isinstance(obj, DataFrame):
4394 axis_number = 1 - axis_number
4396 takers = op(values, filt)
4397 return obj.loc(axis=axis_number)[takers]
4399 raise ValueError(f"cannot find the field [{field}] for filtering!")
4401 # apply the selection filters (but keep in the same order)
4402 if selection.filter is not None:
4403 for field, op, filt in selection.filter.format():
4404 obj = process_filter(field, filt, op)
4406 return obj
4408 def create_description(
4409 self,
4410 complib,
4411 complevel: int | None,
4412 fletcher32: bool,
4413 expectedrows: int | None,
4414 ) -> dict[str, Any]:
4415 """create the description of the table from the axes & values"""
4416 # provided expected rows if its passed
4417 if expectedrows is None:
4418 expectedrows = max(self.nrows_expected, 10000)
4420 d = {"name": "table", "expectedrows": expectedrows}
4422 # description from the axes & values
4423 d["description"] = {a.cname: a.typ for a in self.axes}
4425 if complib:
4426 if complevel is None:
4427 complevel = self._complevel or 9
4428 filters = _tables().Filters(
4429 complevel=complevel,
4430 complib=complib,
4431 fletcher32=fletcher32 or self._fletcher32,
4432 )
4433 d["filters"] = filters
4434 elif self._filters is not None:
4435 d["filters"] = self._filters
4437 return d
4439 def read_coordinates(
4440 self, where=None, start: int | None = None, stop: int | None = None
4441 ):
4442 """
4443 select coordinates (row numbers) from a table; return the
4444 coordinates object
4445 """
4446 # validate the version
4447 self.validate_version(where)
4449 # infer the data kind
4450 if not self.infer_axes():
4451 return False
4453 # create the selection
4454 selection = Selection(self, where=where, start=start, stop=stop)
4455 coords = selection.select_coords()
4456 if selection.filter is not None:
4457 for field, op, filt in selection.filter.format():
4458 data = self.read_column(
4459 field, start=coords.min(), stop=coords.max() + 1
4460 )
4461 coords = coords[op(data.iloc[coords - coords.min()], filt).values]
4463 return Index(coords, copy=False)
4465 def read_column(
4466 self,
4467 column: str,
4468 where=None,
4469 start: int | None = None,
4470 stop: int | None = None,
4471 ):
4472 """
4473 return a single column from the table, generally only indexables
4474 are interesting
4475 """
4476 # validate the version
4477 self.validate_version()
4479 # infer the data kind
4480 if not self.infer_axes():
4481 return False
4483 if where is not None:
4484 raise TypeError("read_column does not currently accept a where clause")
4486 # find the axes
4487 for a in self.axes:
4488 if column == a.name:
4489 if not a.is_data_indexable:
4490 raise ValueError(
4491 f"column [{column}] can not be extracted individually; "
4492 "it is not data indexable"
4493 )
4495 # column must be an indexable or a data column
4496 c = getattr(self.table.cols, column)
4497 a.set_info(self.info)
4498 col_values = a.convert(
4499 c[start:stop],
4500 nan_rep=self.nan_rep,
4501 encoding=self.encoding,
4502 errors=self.errors,
4503 )
4504 cvs = col_values[1]
4505 dtype = getattr(self.table.attrs, f"{column}_meta", None)
4506 return Series(cvs, name=column, copy=False, dtype=dtype)
4508 raise KeyError(f"column [{column}] not found in the table")
4511class WORMTable(Table):
4512 """
4513 a write-once read-many table: this format DOES NOT ALLOW appending to a
4514 table. writing is a one-time operation the data are stored in a format
4515 that allows for searching the data on disk
4516 """
4518 table_type = "worm"
4520 def read(
4521 self,
4522 where=None,
4523 columns=None,
4524 start: int | None = None,
4525 stop: int | None = None,
4526 ):
4527 """
4528 read the indices and the indexing array, calculate offset rows and return
4529 """
4530 raise NotImplementedError("WORMTable needs to implement read")
4532 def write(self, obj, **kwargs) -> None:
4533 """
4534 write in a format that we can search later on (but cannot append
4535 to): write out the indices and the values using _write_array
4536 (e.g. a CArray) create an indexing table so that we can search
4537 """
4538 raise NotImplementedError("WORMTable needs to implement write")
4541class AppendableTable(Table):
4542 """support the new appendable table formats"""
4544 table_type = "appendable"
4546 # error: Signature of "write" incompatible with supertype "Fixed"
4547 def write( # type: ignore[override]
4548 self,
4549 obj,
4550 axes=None,
4551 append: bool = False,
4552 complib=None,
4553 complevel=None,
4554 fletcher32=None,
4555 min_itemsize=None,
4556 chunksize: int | None = None,
4557 expectedrows=None,
4558 dropna: bool = False,
4559 nan_rep=None,
4560 data_columns=None,
4561 track_times: bool = True,
4562 ) -> None:
4563 if not append and self.is_exists:
4564 self._handle.remove_node(self.group, "table")
4566 # create the axes
4567 table = self._create_axes(
4568 axes=axes,
4569 obj=obj,
4570 validate=append,
4571 min_itemsize=min_itemsize,
4572 nan_rep=nan_rep,
4573 data_columns=data_columns,
4574 )
4576 for a in table.axes:
4577 a.validate_names()
4579 if not table.is_exists:
4580 # create the table
4581 options = table.create_description(
4582 complib=complib,
4583 complevel=complevel,
4584 fletcher32=fletcher32,
4585 expectedrows=expectedrows,
4586 )
4588 # set the table attributes
4589 table.set_attrs()
4591 options["track_times"] = track_times
4593 # create the table
4594 table._handle.create_table(table.group, **options)
4596 # update my info
4597 table.attrs.info = table.info
4599 # validate the axes and set the kinds
4600 for a in table.axes:
4601 a.validate_and_set(table, append)
4603 # add the rows
4604 table.write_data(chunksize, dropna=dropna)
4606 def write_data(self, chunksize: int | None, dropna: bool = False) -> None:
4607 """
4608 we form the data into a 2-d including indexes,values,mask write chunk-by-chunk
4609 """
4610 names = self.dtype.names
4611 nrows = self.nrows_expected
4613 # if dropna==True, then drop ALL nan rows
4614 masks = []
4615 if dropna:
4616 for a in self.values_axes:
4617 # figure the mask: only do if we can successfully process this
4618 # column, otherwise ignore the mask
4619 mask = isna(a.data).all(axis=0)
4620 if isinstance(mask, np.ndarray):
4621 masks.append(mask.astype("u1", copy=False))
4623 # consolidate masks
4624 if masks:
4625 mask = masks[0]
4626 for m in masks[1:]:
4627 mask = mask & m
4628 mask = mask.ravel()
4629 else:
4630 mask = None
4632 # broadcast the indexes if needed
4633 indexes = [a.cvalues for a in self.index_axes]
4634 nindexes = len(indexes)
4635 assert nindexes == 1, nindexes # ensures we dont need to broadcast
4637 # transpose the values so first dimension is last
4638 # reshape the values if needed
4639 values = [a.take_data() for a in self.values_axes]
4640 values = [v.transpose(np.roll(np.arange(v.ndim), v.ndim - 1)) for v in values]
4641 bvalues = []
4642 for i, v in enumerate(values):
4643 new_shape = (nrows, *self.dtype[names[nindexes + i]].shape)
4644 bvalues.append(v.reshape(new_shape))
4646 # write the chunks
4647 if chunksize is None:
4648 chunksize = 100000
4650 rows = np.empty(min(chunksize, nrows), dtype=self.dtype)
4651 chunks = nrows // chunksize + 1
4652 for i in range(chunks):
4653 start_i = i * chunksize
4654 end_i = min((i + 1) * chunksize, nrows)
4655 if start_i >= end_i:
4656 break
4658 self.write_data_chunk(
4659 rows,
4660 indexes=[a[start_i:end_i] for a in indexes],
4661 mask=mask[start_i:end_i] if mask is not None else None,
4662 values=[v[start_i:end_i] for v in bvalues],
4663 )
4665 def write_data_chunk(
4666 self,
4667 rows: np.ndarray,
4668 indexes: list[np.ndarray],
4669 mask: npt.NDArray[np.bool_] | None,
4670 values: list[np.ndarray],
4671 ) -> None:
4672 """
4673 Parameters
4674 ----------
4675 rows : an empty memory space where we are putting the chunk
4676 indexes : an array of the indexes
4677 mask : an array of the masks
4678 values : an array of the values
4679 """
4680 # 0 len
4681 for v in values:
4682 if not np.prod(v.shape):
4683 return
4685 nrows = indexes[0].shape[0]
4686 if nrows != len(rows):
4687 rows = np.empty(nrows, dtype=self.dtype)
4688 names = self.dtype.names
4689 nindexes = len(indexes)
4691 # indexes
4692 for i, idx in enumerate(indexes):
4693 rows[names[i]] = idx
4695 # values
4696 for i, v in enumerate(values):
4697 rows[names[i + nindexes]] = v
4699 # mask
4700 if mask is not None:
4701 m = ~mask.ravel().astype(bool, copy=False)
4702 if not m.all():
4703 rows = rows[m]
4705 if len(rows):
4706 self.table.append(rows)
4707 self.table.flush()
4709 def delete(
4710 self, where=None, start: int | None = None, stop: int | None = None
4711 ) -> int | None:
4712 # delete all rows (and return the nrows)
4713 if where is None or not len(where):
4714 if start is None and stop is None:
4715 nrows = self.nrows
4716 self._handle.remove_node(self.group, recursive=True)
4717 else:
4718 # pytables<3.0 would remove a single row with stop=None
4719 if stop is None:
4720 stop = self.nrows
4721 nrows = self.table.remove_rows(start=start, stop=stop)
4722 self.table.flush()
4723 return nrows
4725 # infer the data kind
4726 if not self.infer_axes():
4727 return None
4729 # create the selection
4730 table = self.table
4731 selection = Selection(self, where, start=start, stop=stop)
4732 values = selection.select_coords()
4734 # delete the rows in reverse order
4735 sorted_series = Series(values, copy=False).sort_values()
4736 ln = len(sorted_series)
4738 if ln:
4739 # construct groups of consecutive rows
4740 diff = sorted_series.diff()
4741 groups = list(diff[diff > 1].index)
4743 # 1 group
4744 if not groups:
4745 groups = [0]
4747 # final element
4748 if groups[-1] != ln:
4749 groups.append(ln)
4751 # initial element
4752 if groups[0] != 0:
4753 groups.insert(0, 0)
4755 # we must remove in reverse order!
4756 pg = groups.pop()
4757 for g in reversed(groups):
4758 rows = sorted_series.take(range(g, pg))
4759 table.remove_rows(
4760 start=rows[rows.index[0]], stop=rows[rows.index[-1]] + 1
4761 )
4762 pg = g
4764 self.table.flush()
4766 # return the number of rows removed
4767 return ln
4770class AppendableFrameTable(AppendableTable):
4771 """support the new appendable table formats"""
4773 pandas_kind = "frame_table"
4774 table_type = "appendable_frame"
4775 ndim = 2
4776 obj_type: type[DataFrame | Series] = DataFrame
4778 @property
4779 def is_transposed(self) -> bool:
4780 return self.index_axes[0].axis == 1
4782 @classmethod
4783 def get_object(cls, obj, transposed: bool):
4784 """these are written transposed"""
4785 if transposed:
4786 obj = obj.T
4787 return obj
4789 def read(
4790 self,
4791 where=None,
4792 columns=None,
4793 start: int | None = None,
4794 stop: int | None = None,
4795 ):
4796 # validate the version
4797 self.validate_version(where)
4799 # infer the data kind
4800 if not self.infer_axes():
4801 return None
4803 result = self._read_axes(where=where, start=start, stop=stop)
4805 info = (
4806 self.info.get(self.non_index_axes[0][0], {})
4807 if len(self.non_index_axes)
4808 else {}
4809 )
4811 inds = [i for i, ax in enumerate(self.axes) if ax is self.index_axes[0]]
4812 assert len(inds) == 1
4813 ind = inds[0]
4815 index = result[ind][0]
4817 frames = []
4818 for i, a in enumerate(self.axes):
4819 if a not in self.values_axes:
4820 continue
4821 index_vals, cvalues = result[i]
4823 # we could have a multi-index constructor here
4824 # ensure_index doesn't recognized our list-of-tuples here
4825 if info.get("type") != "MultiIndex":
4826 cols = Index(index_vals)
4827 else:
4828 cols = MultiIndex.from_tuples(index_vals)
4830 names = info.get("names")
4831 if names is not None:
4832 cols.set_names(names, inplace=True)
4834 if self.is_transposed:
4835 values = cvalues
4836 index_ = cols
4837 cols_ = Index(index, name=getattr(index, "name", None))
4838 else:
4839 values = cvalues.T
4840 index_ = Index(index, name=getattr(index, "name", None))
4841 cols_ = cols
4843 # if we have a DataIndexableCol, its shape will only be 1 dim
4844 if values.ndim == 1 and isinstance(values, np.ndarray):
4845 values = values.reshape((1, values.shape[0]))
4847 if isinstance(values, (np.ndarray, DatetimeArray)):
4848 try:
4849 df = DataFrame(values.T, columns=cols_, index=index_, copy=False)
4850 except UnicodeEncodeError as err:
4851 if (
4852 self.errors == "surrogatepass"
4853 and using_string_dtype()
4854 and str(err).endswith("surrogates not allowed")
4855 and HAS_PYARROW
4856 ):
4857 df = DataFrame(
4858 values.T,
4859 columns=cols_,
4860 index=index_,
4861 copy=False,
4862 dtype=StringDtype(storage="python", na_value=np.nan),
4863 )
4864 else:
4865 raise
4866 elif isinstance(values, Index):
4867 df = DataFrame(values, columns=cols_, index=index_)
4868 else:
4869 # Categorical
4870 df = DataFrame._from_arrays([values], columns=cols_, index=index_)
4871 if not (using_string_dtype() and values.dtype.kind == "O"):
4872 assert (df.dtypes == values.dtype).all(), (df.dtypes, values.dtype)
4874 # If str / string dtype is stored in meta, use that.
4875 for column in cols_:
4876 dtype = getattr(self.table.attrs, f"{column}_meta", None)
4877 if dtype in ["str", "string"]:
4878 df[column] = df[column].astype(dtype)
4879 frames.append(df)
4881 if len(frames) == 1:
4882 df = frames[0]
4883 else:
4884 df = concat(frames, axis=1)
4886 selection = Selection(self, where=where, start=start, stop=stop)
4887 # apply the selection filters & axis orderings
4888 df = self.process_axes(df, selection=selection, columns=columns)
4889 return df
4892class AppendableSeriesTable(AppendableFrameTable):
4893 """support the new appendable table formats"""
4895 pandas_kind = "series_table"
4896 table_type = "appendable_series"
4897 ndim = 2
4898 obj_type = Series
4900 @property
4901 def is_transposed(self) -> bool:
4902 return False
4904 @classmethod
4905 def get_object(cls, obj, transposed: bool):
4906 return obj
4908 # error: Signature of "write" incompatible with supertype "Fixed"
4909 def write(self, obj, data_columns=None, **kwargs) -> None: # type: ignore[override]
4910 """we are going to write this as a frame table"""
4911 if not isinstance(obj, DataFrame):
4912 name = obj.name or "values"
4913 obj = obj.to_frame(name)
4914 super().write(obj=obj, data_columns=obj.columns.tolist(), **kwargs)
4916 def read(
4917 self,
4918 where=None,
4919 columns=None,
4920 start: int | None = None,
4921 stop: int | None = None,
4922 ) -> Series:
4923 is_multi_index = self.is_multi_index
4924 if columns is not None and is_multi_index:
4925 assert isinstance(self.levels, list) # needed for mypy
4926 for n in self.levels:
4927 if n not in columns:
4928 columns.insert(0, n)
4929 s = super().read(where=where, columns=columns, start=start, stop=stop)
4930 if is_multi_index:
4931 s.set_index(self.levels, inplace=True)
4933 s = s.iloc[:, 0]
4935 # remove the default name
4936 if s.name == "values":
4937 s.name = None
4938 return s
4941class AppendableMultiSeriesTable(AppendableSeriesTable):
4942 """support the new appendable table formats"""
4944 pandas_kind = "series_table"
4945 table_type = "appendable_multiseries"
4947 # error: Signature of "write" incompatible with supertype "Fixed"
4948 def write(self, obj, **kwargs) -> None: # type: ignore[override]
4949 """we are going to write this as a frame table"""
4950 name = obj.name or "values"
4951 newobj, self.levels = self.validate_multiindex(obj)
4952 assert isinstance(self.levels, list) # for mypy
4953 cols = list(self.levels)
4954 cols.append(name)
4955 newobj.columns = Index(cols)
4956 super().write(obj=newobj, **kwargs)
4959class GenericTable(AppendableFrameTable):
4960 """a table that read/writes the generic pytables table format"""
4962 pandas_kind = "frame_table"
4963 table_type = "generic_table"
4964 ndim = 2
4965 obj_type = DataFrame
4966 levels: list[Hashable]
4968 @property
4969 def pandas_type(self) -> str:
4970 return self.pandas_kind
4972 @property
4973 def storable(self):
4974 return getattr(self.group, "table", None) or self.group
4976 def get_attrs(self) -> None:
4977 """retrieve our attributes"""
4978 self.non_index_axes = []
4979 self.nan_rep = None
4980 self.levels = []
4982 self.index_axes = [a for a in self.indexables if a.is_an_indexable]
4983 self.values_axes = [a for a in self.indexables if not a.is_an_indexable]
4984 self.data_columns = [a.name for a in self.values_axes]
4986 @cache_readonly
4987 def indexables(self):
4988 """create the indexables from the table description"""
4989 d = self.description
4991 # TODO: can we get a typ for this? AFAICT it is the only place
4992 # where we aren't passing one
4993 # the index columns is just a simple index
4994 md = self.read_metadata("index")
4995 meta = "category" if md is not None else None
4996 index_col = GenericIndexCol(
4997 name="index", axis=0, table=self.table, meta=meta, metadata=md
4998 )
5000 _indexables: list[GenericIndexCol | GenericDataIndexableCol] = [index_col]
5002 for i, n in enumerate(d._v_names):
5003 assert isinstance(n, str)
5005 atom = getattr(d, n)
5006 md = self.read_metadata(n)
5007 meta = "category" if md is not None else None
5008 dc = GenericDataIndexableCol(
5009 name=n,
5010 pos=i,
5011 values=[n],
5012 typ=atom,
5013 table=self.table,
5014 meta=meta,
5015 metadata=md,
5016 )
5017 _indexables.append(dc)
5019 return _indexables
5021 # error: Signature of "write" incompatible with supertype "AppendableTable"
5022 def write(self, **kwargs) -> None: # type: ignore[override]
5023 raise NotImplementedError("cannot write on a generic table")
5026class AppendableMultiFrameTable(AppendableFrameTable):
5027 """a frame with a multi-index"""
5029 table_type = "appendable_multiframe"
5030 obj_type = DataFrame
5031 ndim = 2
5032 _re_levels = re.compile(r"^level_\d+$")
5034 @property
5035 def table_type_short(self) -> str:
5036 return "appendable_multi"
5038 # error: Signature of "write" incompatible with supertype "Fixed"
5039 def write(self, obj, data_columns=None, **kwargs) -> None: # type: ignore[override]
5040 if data_columns is None:
5041 data_columns = []
5042 elif data_columns is True:
5043 data_columns = obj.columns.tolist()
5044 obj, self.levels = self.validate_multiindex(obj)
5045 assert isinstance(self.levels, list) # for mypy
5046 for n in self.levels:
5047 if n not in data_columns:
5048 data_columns.insert(0, n)
5049 super().write(obj=obj, data_columns=data_columns, **kwargs)
5051 def read(
5052 self,
5053 where=None,
5054 columns=None,
5055 start: int | None = None,
5056 stop: int | None = None,
5057 ) -> DataFrame:
5058 df = super().read(where=where, columns=columns, start=start, stop=stop)
5059 df = df.set_index(self.levels)
5061 # remove names for 'level_%d'
5062 df.index = df.index.set_names(
5063 [None if self._re_levels.search(name) else name for name in df.index.names]
5064 )
5066 return df
5069def _reindex_axis(
5070 obj: DataFrame, axis: AxisInt, labels: Index, other=None
5071) -> DataFrame:
5072 ax = obj._get_axis(axis)
5073 labels = ensure_index(labels)
5075 # try not to reindex even if other is provided
5076 # if it equals our current index
5077 if other is not None:
5078 other = ensure_index(other)
5079 if (other is None or labels.equals(other)) and labels.equals(ax):
5080 return obj
5082 labels = ensure_index(labels.unique())
5083 if other is not None:
5084 labels = ensure_index(other.unique()).intersection(labels, sort=False)
5085 if not labels.equals(ax):
5086 slicer: list[slice | Index] = [slice(None, None)] * obj.ndim
5087 slicer[axis] = labels
5088 obj = obj.loc[tuple(slicer)]
5089 return obj
5092# tz to/from coercion
5095def _get_tz(tz: tzinfo) -> str | tzinfo:
5096 """for a tz-aware type, return an encoded zone"""
5097 zone = timezones.get_timezone(tz)
5098 return zone
5101def _set_tz(
5102 values: npt.NDArray[np.int64], tz: str | tzinfo | None, datetime64_dtype: str
5103) -> DatetimeArray:
5104 """
5105 Coerce the values to a DatetimeArray with appropriate tz.
5107 Parameters
5108 ----------
5109 values : ndarray[int64]
5110 tz : str, tzinfo, or None
5111 datetime64_dtype : str, e.g. "datetime64[ns]", "datetime64[25s]"
5112 """
5113 assert values.dtype == "i8", values.dtype
5114 # Argument "tz" to "tz_to_dtype" has incompatible type "str | tzinfo | None";
5115 # expected "tzinfo"
5116 unit, _ = np.datetime_data(datetime64_dtype) # parsing dtype: unit, count
5117 unit = cast("TimeUnit", unit)
5118 # error: Argument "tz" to "tz_to_dtype" has incompatible type
5119 # "str | tzinfo | None"; expected "tzinfo"
5120 dtype = tz_to_dtype(tz=tz, unit=unit) # type: ignore[arg-type]
5121 dta = DatetimeArray._from_sequence(values, dtype=dtype)
5122 return dta
5125def _convert_index(name: str, index: Index, encoding: str, errors: str) -> IndexCol:
5126 assert isinstance(name, str)
5128 index_name = index.name
5129 # error: Argument 1 to "_get_data_and_dtype_name" has incompatible type "Index";
5130 # expected "Union[ExtensionArray, ndarray]"
5131 converted, dtype_name = _get_data_and_dtype_name(index) # type: ignore[arg-type]
5132 kind = _dtype_to_kind(dtype_name)
5133 atom = DataIndexableCol._get_atom(converted)
5135 if (
5136 lib.is_np_dtype(index.dtype, "iu")
5137 or needs_i8_conversion(index.dtype)
5138 or is_bool_dtype(index.dtype)
5139 ):
5140 # Includes Index, RangeIndex, DatetimeIndex, TimedeltaIndex, PeriodIndex,
5141 # in which case "kind" is "integer", "integer", "datetime64",
5142 # "timedelta64", and "integer", respectively.
5143 return IndexCol(
5144 name,
5145 values=converted,
5146 kind=kind,
5147 typ=atom,
5148 freq=getattr(index, "freq", None),
5149 tz=getattr(index, "tz", None),
5150 index_name=index_name,
5151 )
5153 if isinstance(index, MultiIndex):
5154 raise TypeError("MultiIndex not supported here!")
5156 inferred_type = lib.infer_dtype(index, skipna=False)
5157 # we won't get inferred_type of "datetime64" or "timedelta64" as these
5158 # would go through the DatetimeIndex/TimedeltaIndex paths above
5160 values = np.asarray(index)
5162 if inferred_type == "date":
5163 converted = np.asarray([v.toordinal() for v in values], dtype=np.int32)
5164 return IndexCol(
5165 name, converted, "date", _tables().Time32Col(), index_name=index_name
5166 )
5167 elif inferred_type == "string":
5168 converted = _convert_string_array(values, encoding, errors)
5169 itemsize = converted.dtype.itemsize
5170 return IndexCol(
5171 name,
5172 converted,
5173 "string",
5174 _tables().StringCol(itemsize),
5175 index_name=index_name,
5176 )
5178 elif inferred_type in ["integer", "floating"]:
5179 return IndexCol(
5180 name, values=converted, kind=kind, typ=atom, index_name=index_name
5181 )
5182 else:
5183 assert isinstance(converted, np.ndarray) and converted.dtype == object
5184 assert kind == "object", kind
5185 atom = _tables().ObjectAtom()
5186 return IndexCol(name, converted, kind, atom, index_name=index_name)
5189def _unconvert_index(data, kind: str, encoding: str, errors: str) -> np.ndarray | Index:
5190 index: Index | np.ndarray
5192 if kind.startswith("datetime64"):
5193 if kind == "datetime64":
5194 # created before we stored resolution information
5195 index = DatetimeIndex(data, copy=False)
5196 else:
5197 index = DatetimeIndex(data.view(kind), copy=False)
5198 elif kind.startswith("timedelta64"):
5199 if kind == "timedelta64":
5200 # created before we stored resolution information
5201 index = TimedeltaIndex(data, copy=False)
5202 else:
5203 index = TimedeltaIndex(data.view(kind), copy=False)
5204 elif kind == "date":
5205 try:
5206 index = np.asarray([date.fromordinal(v) for v in data], dtype=object)
5207 except ValueError:
5208 index = np.asarray([date.fromtimestamp(v) for v in data], dtype=object)
5209 elif kind in ("integer", "float", "bool"):
5210 index = np.asarray(data)
5211 elif kind in ("string"):
5212 index = _unconvert_string_array(
5213 data, nan_rep=None, encoding=encoding, errors=errors
5214 )
5215 elif kind == "object":
5216 index = np.asarray(data[0])
5217 else: # pragma: no cover
5218 raise ValueError(f"unrecognized index type {kind}")
5219 return index
5222def _maybe_convert_for_string_atom(
5223 name: str,
5224 bvalues: ArrayLike,
5225 existing_col,
5226 min_itemsize,
5227 nan_rep,
5228 encoding,
5229 errors,
5230 columns: list[str],
5231):
5232 if isinstance(bvalues.dtype, StringDtype):
5233 bvalues = bvalues.to_numpy()
5234 if bvalues.dtype != object:
5235 return bvalues
5237 bvalues = cast(np.ndarray, bvalues)
5239 dtype_name = bvalues.dtype.name
5240 inferred_type = lib.infer_dtype(bvalues, skipna=False)
5242 if inferred_type == "date":
5243 raise TypeError("[date] is not implemented as a table column")
5244 if inferred_type == "datetime":
5245 # after GH#8260
5246 # this only would be hit for a multi-timezone dtype which is an error
5247 raise TypeError(
5248 "too many timezones in this block, create separate data columns"
5249 )
5251 if not (inferred_type == "string" or dtype_name == "object"):
5252 return bvalues
5254 mask = isna(bvalues)
5255 data = bvalues.copy()
5256 data[mask] = nan_rep
5258 if existing_col and mask.any() and len(nan_rep) > existing_col.itemsize:
5259 raise ValueError("NaN representation is too large for existing column size")
5261 # see if we have a valid string type
5262 inferred_type = lib.infer_dtype(data, skipna=False)
5263 if inferred_type != "string":
5264 # we cannot serialize this data, so report an exception on a column
5265 # by column basis
5267 # expected behaviour:
5268 # search block for a non-string object column by column
5269 for i in range(data.shape[0]):
5270 col = data[i]
5271 inferred_type = lib.infer_dtype(col, skipna=False)
5272 if inferred_type != "string":
5273 error_column_label = columns[i] if len(columns) > i else f"No.{i}"
5274 raise TypeError(
5275 f"Cannot serialize the column [{error_column_label}]\n"
5276 f"because its data contents are not [string] but "
5277 f"[{inferred_type}] object dtype"
5278 )
5280 # itemsize is the maximum length of a string (along any dimension)
5282 data_converted = _convert_string_array(data, encoding, errors).reshape(data.shape)
5283 itemsize = data_converted.itemsize
5285 # specified min_itemsize?
5286 if isinstance(min_itemsize, dict):
5287 min_itemsize = int(min_itemsize.get(name) or min_itemsize.get("values") or 0)
5288 itemsize = max(min_itemsize or 0, itemsize)
5290 # check for column in the values conflicts
5291 if existing_col is not None:
5292 eci = existing_col.validate_col(itemsize)
5293 if eci is not None and eci > itemsize:
5294 itemsize = eci
5296 data_converted = data_converted.astype(f"|S{itemsize}", copy=False)
5297 return data_converted
5300def _convert_string_array(data: np.ndarray, encoding: str, errors: str) -> np.ndarray:
5301 """
5302 Take a string-like that is object dtype and coerce to a fixed size string type.
5304 Parameters
5305 ----------
5306 data : np.ndarray[object]
5307 encoding : str
5308 errors : str
5309 Handler for encoding errors.
5311 Returns
5312 -------
5313 np.ndarray[fixed-length-string]
5314 """
5315 # encode if needed
5316 if len(data):
5317 data = (
5318 Series(data.ravel(), copy=False, dtype="object")
5319 .str.encode(encoding, errors)
5320 ._values.reshape(data.shape)
5321 )
5323 # create the sized dtype
5324 ensured = ensure_object(data.ravel())
5325 itemsize = max(1, libwriters.max_len_string_array(ensured))
5327 data = np.asarray(data, dtype=f"S{itemsize}")
5328 return data
5331def _unconvert_string_array(
5332 data: np.ndarray, nan_rep, encoding: str, errors: str
5333) -> np.ndarray:
5334 """
5335 Inverse of _convert_string_array.
5337 Parameters
5338 ----------
5339 data : np.ndarray[fixed-length-string]
5340 nan_rep : the storage repr of NaN
5341 encoding : str
5342 errors : str
5343 Handler for encoding errors.
5345 Returns
5346 -------
5347 np.ndarray[object]
5348 Decoded data.
5349 """
5350 shape = data.shape
5351 data = np.asarray(data.ravel(), dtype=object)
5353 if len(data):
5354 itemsize = libwriters.max_len_string_array(ensure_object(data))
5355 dtype = f"U{itemsize}"
5357 if isinstance(data[0], bytes):
5358 ser = Series(data, copy=False).str.decode(
5359 encoding, errors=errors, dtype="object"
5360 )
5361 data = ser.to_numpy()
5362 data.flags.writeable = True
5363 else:
5364 data = data.astype(dtype, copy=False).astype(object, copy=False)
5366 if nan_rep is None:
5367 nan_rep = "nan"
5369 libwriters.string_array_replace_from_nan_rep(data, nan_rep)
5370 return data.reshape(shape)
5373def _maybe_convert(values: np.ndarray, val_kind: str, encoding: str, errors: str):
5374 assert isinstance(val_kind, str), type(val_kind)
5375 if _need_convert(val_kind):
5376 conv = _get_converter(val_kind, encoding, errors)
5377 values = conv(values)
5378 return values
5381def _get_converter(kind: str, encoding: str, errors: str):
5382 if kind == "datetime64":
5383 return lambda x: np.asarray(x, dtype="M8[ns]")
5384 elif "datetime64" in kind:
5385 return lambda x: np.asarray(x, dtype=kind)
5386 elif kind == "string":
5387 return lambda x: _unconvert_string_array(
5388 x, nan_rep=None, encoding=encoding, errors=errors
5389 )
5390 else: # pragma: no cover
5391 raise ValueError(f"invalid kind {kind}")
5394def _need_convert(kind: str) -> bool:
5395 if kind in ("datetime64", "string") or "datetime64" in kind:
5396 return True
5397 return False
5400def _maybe_adjust_name(name: str, version: Sequence[int]) -> str:
5401 """
5402 Prior to 0.10.1, we named values blocks like: values_block_0 and the
5403 name values_0, adjust the given name if necessary.
5405 Parameters
5406 ----------
5407 name : str
5408 version : Tuple[int, int, int]
5410 Returns
5411 -------
5412 str
5413 """
5414 if isinstance(version, str) or len(version) < 3:
5415 raise ValueError("Version is incorrect, expected sequence of 3 integers.")
5417 if version[0] == 0 and version[1] <= 10 and version[2] == 0:
5418 m = re.search(r"values_block_(\d+)", name)
5419 if m:
5420 grp = m.groups()[0]
5421 name = f"values_{grp}"
5422 return name
5425def _dtype_to_kind(dtype_str: str) -> str:
5426 """
5427 Find the "kind" string describing the given dtype name.
5428 """
5429 if dtype_str.startswith(("string", "bytes")):
5430 kind = "string"
5431 elif dtype_str.startswith("float"):
5432 kind = "float"
5433 elif dtype_str.startswith("complex"):
5434 kind = "complex"
5435 elif dtype_str.startswith(("int", "uint")):
5436 kind = "integer"
5437 elif dtype_str.startswith("datetime64"):
5438 kind = dtype_str
5439 elif dtype_str.startswith("timedelta"):
5440 kind = dtype_str
5441 elif dtype_str.startswith("bool"):
5442 kind = "bool"
5443 elif dtype_str.startswith("category"):
5444 kind = "category"
5445 elif dtype_str.startswith("period"):
5446 # We store the `freq` attr so we can restore from integers
5447 kind = "integer"
5448 elif dtype_str == "object":
5449 kind = "object"
5450 elif dtype_str == "str":
5451 kind = "str"
5452 else:
5453 raise ValueError(f"cannot interpret dtype of [{dtype_str}]")
5455 return kind
5458def _get_data_and_dtype_name(data: ArrayLike):
5459 """
5460 Convert the passed data into a storable form and a dtype string.
5461 """
5462 if isinstance(data, Categorical):
5463 data = data.codes
5465 if isinstance(data.dtype, DatetimeTZDtype):
5466 # For datetime64tz we need to drop the TZ in tests TODO: why?
5467 dtype_name = f"datetime64[{data.dtype.unit}]"
5468 else:
5469 dtype_name = data.dtype.name
5471 if data.dtype.kind in "mM":
5472 data = np.asarray(data.view("i8"))
5473 # TODO: we used to reshape for the dt64tz case, but no longer
5474 # doing that doesn't seem to break anything. why?
5476 elif isinstance(data, PeriodIndex):
5477 data = data.asi8
5479 data = np.asarray(data)
5480 return data, dtype_name
5483class Selection:
5484 """
5485 Carries out a selection operation on a tables.Table object.
5487 Parameters
5488 ----------
5489 table : a Table object
5490 where : list of Terms (or convertible to)
5491 start, stop: indices to start and/or stop selection
5493 """
5495 def __init__(
5496 self,
5497 table: Table,
5498 where=None,
5499 start: int | None = None,
5500 stop: int | None = None,
5501 ) -> None:
5502 self.table = table
5503 self.where = where
5504 self.start = start
5505 self.stop = stop
5506 self.condition = None
5507 self.filter = None
5508 self.terms = None
5509 self.coordinates = None
5511 if is_list_like(where):
5512 # see if we have a passed coordinate like
5513 with suppress(ValueError):
5514 inferred = lib.infer_dtype(where, skipna=False)
5515 if inferred in ("integer", "boolean"):
5516 where = np.asarray(where)
5517 if where.dtype == np.bool_:
5518 start, stop = self.start, self.stop
5519 if start is None:
5520 start = 0
5521 if stop is None:
5522 stop = self.table.nrows
5523 self.coordinates = np.arange(start, stop)[where]
5524 elif issubclass(where.dtype.type, np.integer):
5525 if (self.start is not None and (where < self.start).any()) or (
5526 self.stop is not None and (where >= self.stop).any()
5527 ):
5528 raise ValueError(
5529 "where must have index locations >= start and < stop"
5530 )
5531 self.coordinates = where
5533 if self.coordinates is None:
5534 self.terms = self.generate(where)
5536 # create the numexpr & the filter
5537 if self.terms is not None:
5538 self.condition, self.filter = self.terms.evaluate()
5540 @overload
5541 def generate(self, where: dict | list | tuple | str) -> PyTablesExpr: ...
5543 @overload
5544 def generate(self, where: None) -> None: ...
5546 def generate(self, where: dict | list | tuple | str | None) -> PyTablesExpr | None:
5547 """where can be a : dict,list,tuple,string"""
5548 if where is None:
5549 return None
5551 q = self.table.queryables()
5552 try:
5553 return PyTablesExpr(where, queryables=q, encoding=self.table.encoding)
5554 except NameError as err:
5555 # raise a nice message, suggesting that the user should use
5556 # data_columns
5557 qkeys = ",".join(q.keys())
5558 msg = dedent(
5559 f"""\
5560 The passed where expression: {where}
5561 contains an invalid variable reference
5562 all of the variable references must be a reference to
5563 an axis (e.g. 'index' or 'columns'), or a data_column
5564 The currently defined references are: {qkeys}
5565 """
5566 )
5567 raise ValueError(msg) from err
5569 def select(self):
5570 """
5571 generate the selection
5572 """
5573 if self.condition is not None:
5574 return self.table.table.read_where(
5575 self.condition.format(), start=self.start, stop=self.stop
5576 )
5577 elif self.coordinates is not None:
5578 return self.table.table.read_coordinates(self.coordinates)
5579 return self.table.table.read(start=self.start, stop=self.stop)
5581 def select_coords(self):
5582 """
5583 generate the selection
5584 """
5585 start, stop = self.start, self.stop
5586 nrows = self.table.nrows
5587 if start is None:
5588 start = 0
5589 elif start < 0:
5590 start += nrows
5591 if stop is None:
5592 stop = nrows
5593 elif stop < 0:
5594 stop += nrows
5596 if self.condition is not None:
5597 return self.table.table.get_where_list(
5598 self.condition.format(), start=start, stop=stop, sort=True
5599 )
5600 elif self.coordinates is not None:
5601 return self.coordinates
5603 return np.arange(start, stop)