Coverage for /pythoncovmergedfiles/medio/medio/usr/local/lib/python3.8/site-packages/tables/description.py: 24%
386 statements
« prev ^ index » next coverage.py v7.2.5, created at 2023-05-10 06:15 +0000
1"""Classes for describing columns for ``Table`` objects."""
3import copy
4import warnings
6import numpy as np
8from . import atom
9from .path import check_name_validity
12__docformat__ = 'reStructuredText'
13"""The format of documentation strings in this module."""
16def same_position(oldmethod):
17 """Decorate `oldmethod` to also compare the `_v_pos` attribute."""
18 def newmethod(self, other):
19 try:
20 other._v_pos
21 except AttributeError:
22 return False # not a column definition
23 return self._v_pos == other._v_pos and oldmethod(self, other)
24 newmethod.__name__ = oldmethod.__name__
25 newmethod.__doc__ = oldmethod.__doc__
26 return newmethod
29class Col(atom.Atom, metaclass=type):
30 """Defines a non-nested column.
32 Col instances are used as a means to declare the different properties of a
33 non-nested column in a table or nested column. Col classes are descendants
34 of their equivalent Atom classes (see :ref:`AtomClassDescr`), but their
35 instances have an additional _v_pos attribute that is used to decide the
36 position of the column inside its parent table or nested column (see the
37 IsDescription class in :ref:`IsDescriptionClassDescr` for more information
38 on column positions).
40 In the same fashion as Atom, you should use a particular Col descendant
41 class whenever you know the exact type you will need when writing your
42 code. Otherwise, you may use one of the Col.from_*() factory methods.
44 Each factory method inherited from the Atom class is available with the
45 same signature, plus an additional pos parameter (placed in last position)
46 which defaults to None and that may take an integer value. This parameter
47 might be used to specify the position of the column in the table.
49 Besides, there are the next additional factory methods, available only for
50 Col objects.
52 The following parameters are available for most Col-derived constructors.
54 Parameters
55 ----------
56 itemsize : int
57 For types with a non-fixed size, this sets the size in bytes of
58 individual items in the column.
59 shape : tuple
60 Sets the shape of the column. An integer shape of N is equivalent to
61 the tuple (N,).
62 dflt
63 Sets the default value for the column.
64 pos : int
65 Sets the position of column in table. If unspecified, the position
66 will be randomly selected.
68 """
70 _class_from_prefix = {} # filled as column classes are created
71 """Maps column prefixes to column classes."""
73 @classmethod
74 def prefix(cls):
75 """Return the column class prefix."""
77 cname = cls.__name__
78 return cname[:cname.rfind('Col')]
80 @classmethod
81 def from_atom(cls, atom, pos=None, _offset=None):
82 """Create a Col definition from a PyTables atom.
84 An optional position may be specified as the pos argument.
86 """
88 prefix = atom.prefix()
89 kwargs = atom._get_init_args()
90 colclass = cls._class_from_prefix[prefix]
91 return colclass(pos=pos, _offset=_offset, **kwargs)
93 @classmethod
94 def from_sctype(cls, sctype, shape=(), dflt=None, pos=None):
95 """Create a `Col` definition from a NumPy scalar type `sctype`.
97 Optional shape, default value and position may be specified as
98 the `shape`, `dflt` and `pos` arguments, respectively.
99 Information in the `sctype` not represented in a `Col` is
100 ignored.
102 """
104 newatom = atom.Atom.from_sctype(sctype, shape, dflt)
105 return cls.from_atom(newatom, pos=pos)
107 @classmethod
108 def from_dtype(cls, dtype, dflt=None, pos=None, _offset=None):
109 """Create a `Col` definition from a NumPy `dtype`.
111 Optional default value and position may be specified as the
112 `dflt` and `pos` arguments, respectively. The `dtype` must have
113 a byte order which is irrelevant or compatible with that of the
114 system. Information in the `dtype` not represented in a `Col`
115 is ignored.
117 """
119 newatom = atom.Atom.from_dtype(dtype, dflt)
120 return cls.from_atom(newatom, pos=pos, _offset=_offset)
122 @classmethod
123 def from_type(cls, type, shape=(), dflt=None, pos=None):
124 """Create a `Col` definition from a PyTables `type`.
126 Optional shape, default value and position may be specified as
127 the `shape`, `dflt` and `pos` arguments, respectively.
129 """
131 newatom = atom.Atom.from_type(type, shape, dflt)
132 return cls.from_atom(newatom, pos=pos)
134 @classmethod
135 def from_kind(cls, kind, itemsize=None, shape=(), dflt=None, pos=None):
136 """Create a `Col` definition from a PyTables `kind`.
138 Optional item size, shape, default value and position may be
139 specified as the `itemsize`, `shape`, `dflt` and `pos`
140 arguments, respectively. Bear in mind that not all columns
141 support a default item size.
143 """
145 newatom = atom.Atom.from_kind(kind, itemsize, shape, dflt)
146 return cls.from_atom(newatom, pos=pos)
148 @classmethod
149 def _subclass_from_prefix(cls, prefix):
150 """Get a column subclass for the given `prefix`."""
152 cname = '%sCol' % prefix
153 class_from_prefix = cls._class_from_prefix
154 if cname in class_from_prefix:
155 return class_from_prefix[cname]
156 atombase = getattr(atom, '%sAtom' % prefix)
158 class NewCol(cls, atombase):
159 """Defines a non-nested column of a particular type.
161 The constructor accepts the same arguments as the equivalent
162 `Atom` class, plus an additional ``pos`` argument for
163 position information, which is assigned to the `_v_pos`
164 attribute.
166 """
168 def __init__(self, *args, **kwargs):
169 pos = kwargs.pop('pos', None)
170 offset = kwargs.pop('_offset', None)
171 class_from_prefix = self._class_from_prefix
172 atombase.__init__(self, *args, **kwargs)
173 # The constructor of an abstract atom may have changed
174 # the class of `self` to something different of `NewCol`
175 # and `atombase` (that's why the prefix map is saved).
176 if self.__class__ is not NewCol:
177 colclass = class_from_prefix[self.prefix()]
178 self.__class__ = colclass
179 self._v_pos = pos
180 self._v_offset = offset
182 __eq__ = same_position(atombase.__eq__)
183 _is_equal_to_atom = same_position(atombase._is_equal_to_atom)
185 # XXX: API incompatible change for PyTables 3 line
186 # Overriding __eq__ blocks inheritance of __hash__ in 3.x
187 # def __hash__(self):
188 # return hash((self._v_pos, self.atombase))
190 if prefix == 'Enum':
191 _is_equal_to_enumatom = same_position(
192 atombase._is_equal_to_enumatom)
194 NewCol.__name__ = cname
196 class_from_prefix[prefix] = NewCol
197 return NewCol
199 def __repr__(self):
200 # Reuse the atom representation.
201 atomrepr = super().__repr__()
202 lpar = atomrepr.index('(')
203 rpar = atomrepr.rindex(')')
204 atomargs = atomrepr[lpar + 1:rpar]
205 classname = self.__class__.__name__
206 return f'{classname}({atomargs}, pos={self._v_pos})'
208 def _get_init_args(self):
209 """Get a dictionary of instance constructor arguments."""
211 kwargs = {arg: getattr(self, arg) for arg in ('shape', 'dflt')}
212 kwargs['pos'] = getattr(self, '_v_pos', None)
213 return kwargs
216def _generate_col_classes():
217 """Generate all column classes."""
219 # Abstract classes are not in the class map.
220 cprefixes = ['Int', 'UInt', 'Float', 'Time']
221 for (kind, kdata) in atom.atom_map.items():
222 if hasattr(kdata, 'kind'): # atom class: non-fixed item size
223 atomclass = kdata
224 cprefixes.append(atomclass.prefix())
225 else: # dictionary: fixed item size
226 for atomclass in kdata.values():
227 cprefixes.append(atomclass.prefix())
229 # Bottom-level complex classes are not in the type map, of course.
230 # We still want the user to get the compatibility warning, though.
231 cprefixes.extend(['Complex32', 'Complex64', 'Complex128'])
232 if hasattr(atom, 'Complex192Atom'):
233 cprefixes.append('Complex192')
234 if hasattr(atom, 'Complex256Atom'):
235 cprefixes.append('Complex256')
237 for cprefix in cprefixes:
238 newclass = Col._subclass_from_prefix(cprefix)
239 yield newclass
242# Create all column classes.
243# for _newclass in _generate_col_classes():
244# exec('%s = _newclass' % _newclass.__name__)
245# del _newclass
247StringCol = Col._subclass_from_prefix('String')
248BoolCol = Col._subclass_from_prefix('Bool')
249EnumCol = Col._subclass_from_prefix('Enum')
250IntCol = Col._subclass_from_prefix('Int')
251Int8Col = Col._subclass_from_prefix('Int8')
252Int16Col = Col._subclass_from_prefix('Int16')
253Int32Col = Col._subclass_from_prefix('Int32')
254Int64Col = Col._subclass_from_prefix('Int64')
255UIntCol = Col._subclass_from_prefix('UInt')
256UInt8Col = Col._subclass_from_prefix('UInt8')
257UInt16Col = Col._subclass_from_prefix('UInt16')
258UInt32Col = Col._subclass_from_prefix('UInt32')
259UInt64Col = Col._subclass_from_prefix('UInt64')
261FloatCol = Col._subclass_from_prefix('Float')
262if hasattr(atom, 'Float16Atom'):
263 Float16Col = Col._subclass_from_prefix('Float16')
264Float32Col = Col._subclass_from_prefix('Float32')
265Float64Col = Col._subclass_from_prefix('Float64')
266if hasattr(atom, 'Float96Atom'):
267 Float96Col = Col._subclass_from_prefix('Float96')
268if hasattr(atom, 'Float128Atom'):
269 Float128Col = Col._subclass_from_prefix('Float128')
271ComplexCol = Col._subclass_from_prefix('Complex')
272Complex32Col = Col._subclass_from_prefix('Complex32')
273Complex64Col = Col._subclass_from_prefix('Complex64')
274Complex128Col = Col._subclass_from_prefix('Complex128')
275if hasattr(atom, 'Complex192Atom'):
276 Complex192Col = Col._subclass_from_prefix('Complex192')
277if hasattr(atom, 'Complex256Atom'):
278 Complex256Col = Col._subclass_from_prefix('Complex256')
280TimeCol = Col._subclass_from_prefix('Time')
281Time32Col = Col._subclass_from_prefix('Time32')
282Time64Col = Col._subclass_from_prefix('Time64')
285# Table description classes
286# =========================
287class Description:
288 """This class represents descriptions of the structure of tables.
290 An instance of this class is automatically bound to Table (see
291 :ref:`TableClassDescr`) objects when they are created. It provides a
292 browseable representation of the structure of the table, made of non-nested
293 (Col - see :ref:`ColClassDescr`) and nested (Description) columns.
295 Column definitions under a description can be accessed as attributes of it
296 (*natural naming*). For instance, if table.description is a Description
297 instance with a column named col1 under it, the later can be accessed as
298 table.description.col1. If col1 is nested and contains a col2 column, this
299 can be accessed as table.description.col1.col2. Because of natural naming,
300 the names of members start with special prefixes, like in the Group class
301 (see :ref:`GroupClassDescr`).
304 .. rubric:: Description attributes
306 .. attribute:: _v_colobjects
308 A dictionary mapping the names of the columns hanging
309 directly from the associated table or nested column to their
310 respective descriptions (Col - see :ref:`ColClassDescr` or
311 Description - see :ref:`DescriptionClassDescr` instances).
313 .. versionchanged:: 3.0
314 The *_v_colObjects* attribute has been renamed into
315 *_v_colobjects*.
317 .. attribute:: _v_dflts
319 A dictionary mapping the names of non-nested columns
320 hanging directly from the associated table or nested column
321 to their respective default values.
323 .. attribute:: _v_dtype
325 The NumPy type which reflects the structure of this
326 table or nested column. You can use this as the
327 dtype argument of NumPy array factories.
329 .. attribute:: _v_dtypes
331 A dictionary mapping the names of non-nested columns
332 hanging directly from the associated table or nested column
333 to their respective NumPy types.
335 .. attribute:: _v_is_nested
337 Whether the associated table or nested column contains
338 further nested columns or not.
340 .. attribute:: _v_itemsize
342 The size in bytes of an item in this table or nested column.
344 .. attribute:: _v_name
346 The name of this description group. The name of the
347 root group is '/'.
349 .. attribute:: _v_names
351 A list of the names of the columns hanging directly
352 from the associated table or nested column. The order of the
353 names matches the order of their respective columns in the
354 containing table.
356 .. attribute:: _v_nested_descr
358 A nested list of pairs of (name, format) tuples for all the columns
359 under this table or nested column. You can use this as the dtype and
360 descr arguments of NumPy array factories.
362 .. versionchanged:: 3.0
363 The *_v_nestedDescr* attribute has been renamed into
364 *_v_nested_descr*.
366 .. attribute:: _v_nested_formats
368 A nested list of the NumPy string formats (and shapes) of all the
369 columns under this table or nested column. You can use this as the
370 formats argument of NumPy array factories.
372 .. versionchanged:: 3.0
373 The *_v_nestedFormats* attribute has been renamed into
374 *_v_nested_formats*.
376 .. attribute:: _v_nestedlvl
378 The level of the associated table or nested column in the nested
379 datatype.
381 .. attribute:: _v_nested_names
383 A nested list of the names of all the columns under this table or
384 nested column. You can use this as the names argument of NumPy array
385 factories.
387 .. versionchanged:: 3.0
388 The *_v_nestedNames* attribute has been renamed into
389 *_v_nested_names*.
391 .. attribute:: _v_pathname
393 Pathname of the table or nested column.
395 .. attribute:: _v_pathnames
397 A list of the pathnames of all the columns under this table or nested
398 column (in preorder). If it does not contain nested columns, this is
399 exactly the same as the :attr:`Description._v_names` attribute.
401 .. attribute:: _v_types
403 A dictionary mapping the names of non-nested columns hanging directly
404 from the associated table or nested column to their respective PyTables
405 types.
407 .. attribute:: _v_offsets
409 A list of offsets for all the columns. If the list is empty, means
410 that there are no padding in the data structure. However, the support
411 for offsets is currently limited to flat tables; for nested tables, the
412 potential padding is always removed (exactly the same as in pre-3.5
413 versions), and this variable is set to empty.
415 .. versionadded:: 3.5
416 Previous to this version all the compound types were converted
417 internally to 'packed' types, i.e. with no padding between the
418 component types. Starting with 3.5, the holes in native HDF5
419 types (non-nested) are honored and replicated during dataset
420 and attribute copies.
421 """
423 def __init__(self, classdict, nestedlvl=-1, validate=True, ptparams=None):
425 if not classdict:
426 raise ValueError("cannot create an empty data type")
428 # Do a shallow copy of classdict just in case this is going to
429 # be shared by other instances
430 newdict = self.__dict__
431 newdict["_v_name"] = "/" # The name for root descriptor
432 newdict["_v_names"] = []
433 newdict["_v_dtypes"] = {}
434 newdict["_v_types"] = {}
435 newdict["_v_dflts"] = {}
436 newdict["_v_colobjects"] = {}
437 newdict["_v_is_nested"] = False
438 nestedFormats = []
439 nestedDType = []
441 if not hasattr(newdict, "_v_nestedlvl"):
442 newdict["_v_nestedlvl"] = nestedlvl + 1
444 cols_with_pos = [] # colum (position, name) pairs
445 cols_no_pos = [] # just column names
446 cols_offsets = [] # the offsets of the columns
447 valid_offsets = False # by default there a no valid offsets
449 # Check for special variables and convert column descriptions
450 for (name, descr) in classdict.items():
451 if name.startswith('_v_'):
452 if name in newdict:
453 # print("Warning!")
454 # special methods &c: copy to newdict, warn about conflicts
455 warnings.warn("Can't set attr %r in description class %r"
456 % (name, self))
457 else:
458 # print("Special variable!-->", name, classdict[name])
459 newdict[name] = descr
460 continue # This variable is not needed anymore
462 columns = None
463 if (type(descr) == type(IsDescription) and
464 issubclass(descr, IsDescription)):
465 # print("Nested object (type I)-->", name)
466 columns = descr().columns
467 elif (type(descr.__class__) == type(IsDescription) and
468 issubclass(descr.__class__, IsDescription)):
469 # print("Nested object (type II)-->", name)
470 columns = descr.columns
471 elif isinstance(descr, dict):
472 # print("Nested object (type III)-->", name)
473 columns = descr
474 else:
475 # print("Nested object (type IV)-->", name)
476 descr = copy.copy(descr)
477 # The copies above and below ensure that the structures
478 # provided by the user will remain unchanged even if we
479 # tamper with the values of ``_v_pos`` here.
480 if columns is not None:
481 descr = Description(copy.copy(columns), self._v_nestedlvl,
482 ptparams=ptparams)
483 classdict[name] = descr
485 pos = getattr(descr, '_v_pos', None)
486 if pos is None:
487 cols_no_pos.append(name)
488 else:
489 cols_with_pos.append((pos, name))
490 offset = getattr(descr, '_v_offset', None)
491 if offset is not None:
492 cols_offsets.append(offset)
494 # Sort field names:
495 #
496 # 1. Fields with explicit positions, according to their
497 # positions (and their names if coincident).
498 # 2. Fields with no position, in alphabetical order.
499 cols_with_pos.sort()
500 cols_no_pos.sort()
501 keys = [name for (pos, name) in cols_with_pos] + cols_no_pos
503 pos = 0
504 nested = False
505 # Get properties for compound types
506 for k in keys:
507 if validate:
508 # Check for key name validity
509 check_name_validity(k)
510 # Class variables
511 object = classdict[k]
512 newdict[k] = object # To allow natural naming
513 if not isinstance(object, (Col, Description)):
514 raise TypeError('Passing an incorrect value to a table column.'
515 ' Expected a Col (or subclass) instance and '
516 'got: "%s". Please make use of the Col(), or '
517 'descendant, constructor to properly '
518 'initialize columns.' % object)
519 object._v_pos = pos # Set the position of this object
520 object._v_parent = self # The parent description
521 pos += 1
522 newdict['_v_colobjects'][k] = object
523 newdict['_v_names'].append(k)
524 object.__dict__['_v_name'] = k
526 if not isinstance(k, str):
527 # numpy only accepts "str" for field names
528 # Python 3.x: bytes --> str (unicode)
529 kk = k.decode()
530 else:
531 kk = k
533 if isinstance(object, Col):
534 dtype = object.dtype
535 newdict['_v_dtypes'][k] = dtype
536 newdict['_v_types'][k] = object.type
537 newdict['_v_dflts'][k] = object.dflt
538 nestedFormats.append(object.recarrtype)
539 baserecarrtype = dtype.base.str[1:]
540 nestedDType.append((kk, baserecarrtype, dtype.shape))
541 else: # A description
542 nestedFormats.append(object._v_nested_formats)
543 nestedDType.append((kk, object._v_dtype))
544 nested = True
546 # Useful for debugging purposes
547 # import traceback
548 # if ptparams is None:
549 # print("*** print_stack:")
550 # traceback.print_stack()
552 # Check whether we are gonna use padding or not. Two possibilities:
553 # 1) Make padding True by default (except if ALLOW_PADDING is set
554 # to False)
555 # 2) Make padding False by default (except if ALLOW_PADDING is set
556 # to True)
557 # Currently we choose 1) because it favours honoring padding even on
558 # unhandled situations (should be very few).
559 # However, for development, option 2) is recommended as it catches
560 # most of the unhandled situations.
561 allow_padding = ptparams is None or ptparams['ALLOW_PADDING']
562 # allow_padding = ptparams is not None and ptparams['ALLOW_PADDING']
563 if (allow_padding and
564 len(cols_offsets) > 1 and
565 len(keys) == len(cols_with_pos) and
566 len(keys) == len(cols_offsets) and
567 not nested): # TODO: support offsets with nested types
568 # We have to sort the offsets too, as they must follow the column
569 # order. As the offsets and the pos should be place in the same
570 # order, a single sort is enough here.
571 cols_offsets.sort()
572 valid_offsets = True
573 else:
574 newdict['_v_offsets'] = []
576 # Assign the format list to _v_nested_formats
577 newdict['_v_nested_formats'] = nestedFormats
579 if self._v_nestedlvl == 0:
580 # Get recursively nested _v_nested_names and _v_nested_descr attrs
581 self._g_set_nested_names_descr()
582 # Get pathnames for nested groups
583 self._g_set_path_names()
584 # Check the _v_byteorder has been used an issue an Error
585 if hasattr(self, "_v_byteorder"):
586 raise ValueError(
587 "Using a ``_v_byteorder`` in the description is obsolete. "
588 "Use the byteorder parameter in the constructor instead.")
590 # Compute the dtype with offsets or without
591 # print("offsets ->", cols_offsets, nestedDType, nested, valid_offsets)
592 if valid_offsets:
593 # TODO: support offsets within nested types
594 dtype_fields = {
595 'names': newdict['_v_names'], 'formats': nestedFormats,
596 'offsets': cols_offsets}
597 itemsize = newdict.get('_v_itemsize', None)
598 if itemsize is not None:
599 dtype_fields['itemsize'] = itemsize
600 dtype = np.dtype(dtype_fields)
601 else:
602 dtype = np.dtype(nestedDType)
603 newdict['_v_dtype'] = dtype
604 newdict['_v_itemsize'] = dtype.itemsize
605 newdict['_v_offsets'] = [dtype.fields[name][1] for name in dtype.names]
607 def _g_set_nested_names_descr(self):
608 """Computes the nested names and descriptions for nested datatypes."""
610 names = self._v_names
611 fmts = self._v_nested_formats
612 self._v_nested_names = names[:] # Important to do a copy!
613 self._v_nested_descr = list(zip(names, fmts))
614 for i, name in enumerate(names):
615 new_object = self._v_colobjects[name]
616 if isinstance(new_object, Description):
617 new_object._g_set_nested_names_descr()
618 # replace the column nested name by a correct tuple
619 self._v_nested_names[i] = (name, new_object._v_nested_names)
620 self._v_nested_descr[i] = (name, new_object._v_nested_descr)
621 # set the _v_is_nested flag
622 self._v_is_nested = True
624 def _g_set_path_names(self):
625 """Compute the pathnames for arbitrary nested descriptions.
627 This method sets the ``_v_pathname`` and ``_v_pathnames``
628 attributes of all the elements (both descriptions and columns)
629 in this nested description.
631 """
633 def get_cols_in_order(description):
634 return [description._v_colobjects[colname]
635 for colname in description._v_names]
637 def join_paths(path1, path2):
638 if not path1:
639 return path2
640 return f'{path1}/{path2}'
642 # The top of the stack always has a nested description
643 # and a list of its child columns
644 # (be they nested ``Description`` or non-nested ``Col`` objects).
645 # In the end, the list contains only a list of column paths
646 # under this one.
647 #
648 # For instance, given this top of the stack::
649 #
650 # (<Description X>, [<Column A>, <Column B>])
651 #
652 # After computing the rest of the stack, the top is::
653 #
654 # (<Description X>, ['a', 'a/m', 'a/n', ... , 'b', ...])
656 stack = []
658 # We start by pushing the top-level description
659 # and its child columns.
660 self._v_pathname = ''
661 stack.append((self, get_cols_in_order(self)))
663 while stack:
664 desc, cols = stack.pop()
665 head = cols[0]
667 # What's the first child in the list?
668 if isinstance(head, Description):
669 # A nested description. We remove it from the list and
670 # push it with its child columns. This will be the next
671 # handled description.
672 head._v_pathname = join_paths(desc._v_pathname, head._v_name)
673 stack.append((desc, cols[1:])) # alter the top
674 stack.append((head, get_cols_in_order(head))) # new top
675 elif isinstance(head, Col):
676 # A non-nested column. We simply remove it from the
677 # list and append its name to it.
678 head._v_pathname = join_paths(desc._v_pathname, head._v_name)
679 cols.append(head._v_name) # alter the top
680 stack.append((desc, cols[1:])) # alter the top
681 else:
682 # Since paths and names are appended *to the end* of
683 # children lists, a string signals that no more children
684 # remain to be processed, so we are done with the
685 # description at the top of the stack.
686 assert isinstance(head, str)
687 # Assign the computed set of descendent column paths.
688 desc._v_pathnames = cols
689 if len(stack) > 0:
690 # Compute the paths with respect to the parent node
691 # (including the path of the current description)
692 # and append them to its list.
693 descName = desc._v_name
694 colPaths = [join_paths(descName, path) for path in cols]
695 colPaths.insert(0, descName)
696 parentCols = stack[-1][1]
697 parentCols.extend(colPaths)
698 # (Nothing is pushed, we are done with this description.)
700 def _f_walk(self, type='All'):
701 """Iterate over nested columns.
703 If type is 'All' (the default), all column description objects (Col and
704 Description instances) are yielded in top-to-bottom order (preorder).
706 If type is 'Col' or 'Description', only column descriptions of that
707 type are yielded.
709 """
711 if type not in ["All", "Col", "Description"]:
712 raise ValueError("""\
713type can only take the parameters 'All', 'Col' or 'Description'.""")
715 stack = [self]
716 while stack:
717 object = stack.pop(0) # pop at the front so as to ensure the order
718 if type in ["All", "Description"]:
719 yield object # yield description
720 for name in object._v_names:
721 new_object = object._v_colobjects[name]
722 if isinstance(new_object, Description):
723 stack.append(new_object)
724 else:
725 if type in ["All", "Col"]:
726 yield new_object # yield column
728 def __repr__(self):
729 """Gives a detailed Description column representation."""
731 rep = ['%s\"%s\": %r' %
732 (" " * self._v_nestedlvl, k, self._v_colobjects[k])
733 for k in self._v_names]
734 return '{\n %s}' % (',\n '.join(rep))
736 def __str__(self):
737 """Gives a brief Description representation."""
739 return f'Description({self._v_nested_descr})'
742class MetaIsDescription(type):
743 """Helper metaclass to return the class variables as a dictionary."""
745 def __new__(mcs, classname, bases, classdict):
746 """Return a new class with a "columns" attribute filled."""
748 newdict = {"columns": {}, }
749 if '__doc__' in classdict:
750 newdict['__doc__'] = classdict['__doc__']
751 for b in bases:
752 if "columns" in b.__dict__:
753 newdict["columns"].update(b.__dict__["columns"])
754 for k in classdict:
755 # if not (k.startswith('__') or k.startswith('_v_')):
756 # We let pass _v_ variables to configure class behaviour
757 if not (k.startswith('__')):
758 newdict["columns"][k] = classdict[k]
760 # Return a new class with the "columns" attribute filled
761 return type.__new__(mcs, classname, bases, newdict)
764class IsDescription(metaclass=MetaIsDescription):
765 """Description of the structure of a table or nested column.
767 This class is designed to be used as an easy, yet meaningful way to
768 describe the structure of new Table (see :ref:`TableClassDescr`) datasets
769 or nested columns through the definition of *derived classes*. In order to
770 define such a class, you must declare it as descendant of IsDescription,
771 with as many attributes as columns you want in your table. The name of each
772 attribute will become the name of a column, and its value will hold a
773 description of it.
775 Ordinary columns can be described using instances of the Col class (see
776 :ref:`ColClassDescr`). Nested columns can be described by using classes
777 derived from IsDescription, instances of it, or name-description
778 dictionaries. Derived classes can be declared in place (in which case the
779 column takes the name of the class) or referenced by name.
781 Nested columns can have a _v_pos special attribute which sets the
782 *relative* position of the column among sibling columns *also having
783 explicit positions*. The pos constructor argument of Col instances is used
784 for the same purpose. Columns with no explicit position will be placed
785 afterwards in alphanumeric order.
787 Once you have created a description object, you can pass it to the Table
788 constructor, where all the information it contains will be used to define
789 the table structure.
791 .. rubric:: IsDescription attributes
793 .. attribute:: _v_pos
795 Sets the position of a possible nested column description among its
796 sibling columns. This attribute can be specified *when declaring*
797 an IsDescription subclass to complement its *metadata*.
799 .. attribute:: columns
801 Maps the name of each column in the description to its own descriptive
802 object. This attribute is *automatically created* when an IsDescription
803 subclass is declared. Please note that declared columns can no longer
804 be accessed as normal class variables after its creation.
806 """
809def descr_from_dtype(dtype_, ptparams=None):
810 """Get a description instance and byteorder from a (nested) NumPy dtype."""
812 fields = {}
813 fbyteorder = '|'
814 for name in dtype_.names:
815 dtype, offset = dtype_.fields[name][:2]
816 kind = dtype.base.kind
817 byteorder = dtype.base.byteorder
818 if byteorder in '><=':
819 if fbyteorder not in ['|', byteorder]:
820 raise NotImplementedError(
821 "structured arrays with mixed byteorders "
822 "are not supported yet, sorry")
823 fbyteorder = byteorder
824 # Non-nested column
825 if kind in 'biufSUc':
826 col = Col.from_dtype(dtype, pos=offset, _offset=offset)
827 # Nested column
828 elif kind == 'V' and dtype.shape in [(), (1,)]:
829 if dtype.shape != ():
830 warnings.warn(
831 "nested descriptions will be converted to scalar")
832 col, _ = descr_from_dtype(dtype.base, ptparams=ptparams)
833 col._v_pos = offset
834 col._v_offset = offset
835 else:
836 raise NotImplementedError(
837 "structured arrays with columns with type description ``%s`` "
838 "are not supported yet, sorry" % dtype)
839 fields[name] = col
841 return Description(fields, ptparams=ptparams), fbyteorder
844def dtype_from_descr(descr, byteorder=None, ptparams=None):
845 """Get a (nested) NumPy dtype from a description instance and byteorder.
847 The descr parameter can be a Description or IsDescription
848 instance, sub-class of IsDescription or a dictionary.
850 """
852 if isinstance(descr, dict):
853 descr = Description(descr, ptparams=ptparams)
854 elif (type(descr) == type(IsDescription)
855 and issubclass(descr, IsDescription)):
856 descr = Description(descr().columns, ptparams=ptparams)
857 elif isinstance(descr, IsDescription):
858 descr = Description(descr.columns, ptparams=ptparams)
859 elif not isinstance(descr, Description):
860 raise ValueError('invalid description: %r' % descr)
862 dtype_ = descr._v_dtype
864 if byteorder and byteorder != '|':
865 dtype_ = dtype_.newbyteorder(byteorder)
867 return dtype_
870if __name__ == "__main__":
871 """Test code."""
873 class Info(IsDescription):
874 _v_pos = 2
875 Name = UInt32Col()
876 Value = Float64Col()
878 class Test(IsDescription):
879 """A description that has several columns."""
881 x = Col.from_type("int32", 2, 0, pos=0)
882 y = Col.from_kind('float', dflt=1, shape=(2, 3))
883 z = UInt8Col(dflt=1)
884 color = StringCol(2, dflt=" ")
885 # color = UInt32Col(2)
886 Info = Info()
888 class info(IsDescription):
889 _v_pos = 1
890 name = UInt32Col()
891 value = Float64Col(pos=0)
892 y2 = Col.from_kind('float', dflt=1, shape=(2, 3), pos=1)
893 z2 = UInt8Col(dflt=1)
895 class info2(IsDescription):
896 y3 = Col.from_kind('float', dflt=1, shape=(2, 3))
897 z3 = UInt8Col(dflt=1)
898 name = UInt32Col()
899 value = Float64Col()
901 class info3(IsDescription):
902 name = UInt32Col()
903 value = Float64Col()
904 y4 = Col.from_kind('float', dflt=1, shape=(2, 3))
905 z4 = UInt8Col(dflt=1)
907# class Info(IsDescription):
908# _v_pos = 2
909# Name = StringCol(itemsize=2)
910# Value = ComplexCol(itemsize=16)
912# class Test(IsDescription):
913# """A description that has several columns"""
914# x = Col.from_type("int32", 2, 0, pos=0)
915# y = Col.from_kind('float', dflt=1, shape=(2,3))
916# z = UInt8Col(dflt=1)
917# color = StringCol(2, dflt=" ")
918# Info = Info()
919# class info(IsDescription):
920# _v_pos = 1
921# name = StringCol(itemsize=2)
922# value = ComplexCol(itemsize=16, pos=0)
923# y2 = Col.from_kind('float', dflt=1, shape=(2,3), pos=1)
924# z2 = UInt8Col(dflt=1)
925# class info2(IsDescription):
926# y3 = Col.from_kind('float', dflt=1, shape=(2,3))
927# z3 = UInt8Col(dflt=1)
928# name = StringCol(itemsize=2)
929# value = ComplexCol(itemsize=16)
930# class info3(IsDescription):
931# name = StringCol(itemsize=2)
932# value = ComplexCol(itemsize=16)
933# y4 = Col.from_kind('float', dflt=1, shape=(2,3))
934# z4 = UInt8Col(dflt=1)
936 # example cases of class Test
937 klass = Test()
938 # klass = Info()
939 desc = Description(klass.columns)
940 print("Description representation (short) ==>", desc)
941 print("Description representation (long) ==>", repr(desc))
942 print("Column names ==>", desc._v_names)
943 print("Column x ==>", desc.x)
944 print("Column Info ==>", desc.Info)
945 print("Column Info.value ==>", desc.Info.Value)
946 print("Nested column names ==>", desc._v_nested_names)
947 print("Defaults ==>", desc._v_dflts)
948 print("Nested Formats ==>", desc._v_nested_formats)
949 print("Nested Descriptions ==>", desc._v_nested_descr)
950 print("Nested Descriptions (info) ==>", desc.info._v_nested_descr)
951 print("Total size ==>", desc._v_dtype.itemsize)
953 # check _f_walk
954 for object in desc._f_walk():
955 if isinstance(object, Description):
956 print("******begin object*************", end=' ')
957 print("name -->", object._v_name)
958 # print("name -->", object._v_dtype.name)
959 # print("object childs-->", object._v_names)
960 # print("object nested childs-->", object._v_nested_names)
961 print("totalsize-->", object._v_dtype.itemsize)
962 else:
963 # pass
964 print("leaf -->", object._v_name, object.dtype)
966 class testDescParent(IsDescription):
967 c = Int32Col()
969 class testDesc(testDescParent):
970 pass
972 assert 'c' in testDesc.columns