Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2IO related functions.
3"""
4import contextlib
5import functools
6import itertools
7import operator
8import os
9import pickle
10import re
11import warnings
12import weakref
13from collections.abc import Mapping
14from operator import itemgetter
16import numpy as np
17from numpy._core import overrides
18from numpy._core._multiarray_umath import _load_from_filelike
19from numpy._core.multiarray import packbits, unpackbits
20from numpy._core.overrides import finalize_array_function_like, set_module
21from numpy._utils import asbytes, asunicode
23from . import format
24from ._datasource import DataSource # noqa: F401
25from ._format_impl import _MAX_HEADER_SIZE
26from ._iotools import (
27 ConversionWarning,
28 ConverterError,
29 ConverterLockError,
30 LineSplitter,
31 NameValidator,
32 StringConverter,
33 _decode_line,
34 _is_string_like,
35 easy_dtype,
36 flatten_dtype,
37 has_nested_fields,
38)
40__all__ = [
41 'savetxt', 'loadtxt', 'genfromtxt', 'load', 'save', 'savez',
42 'savez_compressed', 'packbits', 'unpackbits', 'fromregex'
43 ]
46array_function_dispatch = functools.partial(
47 overrides.array_function_dispatch, module='numpy')
50class BagObj:
51 """
52 BagObj(obj)
54 Convert attribute look-ups to getitems on the object passed in.
56 Parameters
57 ----------
58 obj : class instance
59 Object on which attribute look-up is performed.
61 Examples
62 --------
63 >>> import numpy as np
64 >>> from numpy.lib._npyio_impl import BagObj as BO
65 >>> class BagDemo:
66 ... def __getitem__(self, key): # An instance of BagObj(BagDemo)
67 ... # will call this method when any
68 ... # attribute look-up is required
69 ... result = "Doesn't matter what you want, "
70 ... return result + "you're gonna get this"
71 ...
72 >>> demo_obj = BagDemo()
73 >>> bagobj = BO(demo_obj)
74 >>> bagobj.hello_there
75 "Doesn't matter what you want, you're gonna get this"
76 >>> bagobj.I_can_be_anything
77 "Doesn't matter what you want, you're gonna get this"
79 """
81 def __init__(self, obj):
82 # Use weakref to make NpzFile objects collectable by refcount
83 self._obj = weakref.proxy(obj)
85 def __getattribute__(self, key):
86 try:
87 return object.__getattribute__(self, '_obj')[key]
88 except KeyError:
89 raise AttributeError(key) from None
91 def __dir__(self):
92 """
93 Enables dir(bagobj) to list the files in an NpzFile.
95 This also enables tab-completion in an interpreter or IPython.
96 """
97 return list(object.__getattribute__(self, '_obj').keys())
100def zipfile_factory(file, *args, **kwargs):
101 """
102 Create a ZipFile.
104 Allows for Zip64, and the `file` argument can accept file, str, or
105 pathlib.Path objects. `args` and `kwargs` are passed to the zipfile.ZipFile
106 constructor.
107 """
108 if not hasattr(file, 'read'):
109 file = os.fspath(file)
110 import zipfile
111 kwargs['allowZip64'] = True
112 return zipfile.ZipFile(file, *args, **kwargs)
115@set_module('numpy.lib.npyio')
116class NpzFile(Mapping):
117 """
118 NpzFile(fid)
120 A dictionary-like object with lazy-loading of files in the zipped
121 archive provided on construction.
123 `NpzFile` is used to load files in the NumPy ``.npz`` data archive
124 format. It assumes that files in the archive have a ``.npy`` extension,
125 other files are ignored.
127 The arrays and file strings are lazily loaded on either
128 getitem access using ``obj['key']`` or attribute lookup using
129 ``obj.f.key``. A list of all files (without ``.npy`` extensions) can
130 be obtained with ``obj.files`` and the ZipFile object itself using
131 ``obj.zip``.
133 Attributes
134 ----------
135 files : list of str
136 List of all files in the archive with a ``.npy`` extension.
137 zip : ZipFile instance
138 The ZipFile object initialized with the zipped archive.
139 f : BagObj instance
140 An object on which attribute can be performed as an alternative
141 to getitem access on the `NpzFile` instance itself.
142 allow_pickle : bool, optional
143 Allow loading pickled data. Default: False
144 pickle_kwargs : dict, optional
145 Additional keyword arguments to pass on to pickle.load.
146 These are only useful when loading object arrays saved on
147 Python 2.
148 max_header_size : int, optional
149 Maximum allowed size of the header. Large headers may not be safe
150 to load securely and thus require explicitly passing a larger value.
151 See :py:func:`ast.literal_eval()` for details.
152 This option is ignored when `allow_pickle` is passed. In that case
153 the file is by definition trusted and the limit is unnecessary.
155 Parameters
156 ----------
157 fid : file, str, or pathlib.Path
158 The zipped archive to open. This is either a file-like object
159 or a string containing the path to the archive.
160 own_fid : bool, optional
161 Whether NpzFile should close the file handle.
162 Requires that `fid` is a file-like object.
164 Examples
165 --------
166 >>> import numpy as np
167 >>> from tempfile import TemporaryFile
168 >>> outfile = TemporaryFile()
169 >>> x = np.arange(10)
170 >>> y = np.sin(x)
171 >>> np.savez(outfile, x=x, y=y)
172 >>> _ = outfile.seek(0)
174 >>> npz = np.load(outfile)
175 >>> isinstance(npz, np.lib.npyio.NpzFile)
176 True
177 >>> npz
178 NpzFile 'object' with keys: x, y
179 >>> sorted(npz.files)
180 ['x', 'y']
181 >>> npz['x'] # getitem access
182 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
183 >>> npz.f.x # attribute lookup
184 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
186 """
187 # Make __exit__ safe if zipfile_factory raises an exception
188 zip = None
189 fid = None
190 _MAX_REPR_ARRAY_COUNT = 5
192 def __init__(self, fid, own_fid=False, allow_pickle=False,
193 pickle_kwargs=None, *,
194 max_header_size=_MAX_HEADER_SIZE):
195 # Import is postponed to here since zipfile depends on gzip, an
196 # optional component of the so-called standard library.
197 _zip = zipfile_factory(fid)
198 _files = _zip.namelist()
199 self.files = [name.removesuffix(".npy") for name in _files]
200 self._files = dict(zip(self.files, _files))
201 self._files.update(zip(_files, _files))
202 self.allow_pickle = allow_pickle
203 self.max_header_size = max_header_size
204 self.pickle_kwargs = pickle_kwargs
205 self.zip = _zip
206 self.f = BagObj(self)
207 if own_fid:
208 self.fid = fid
210 def __enter__(self):
211 return self
213 def __exit__(self, exc_type, exc_value, traceback):
214 self.close()
216 def close(self):
217 """
218 Close the file.
220 """
221 if self.zip is not None:
222 self.zip.close()
223 self.zip = None
224 if self.fid is not None:
225 self.fid.close()
226 self.fid = None
227 self.f = None # break reference cycle
229 def __del__(self):
230 self.close()
232 # Implement the Mapping ABC
233 def __iter__(self):
234 return iter(self.files)
236 def __len__(self):
237 return len(self.files)
239 def __getitem__(self, key):
240 try:
241 key = self._files[key]
242 except KeyError:
243 raise KeyError(f"{key} is not a file in the archive") from None
244 else:
245 with self.zip.open(key) as bytes:
246 magic = bytes.read(len(format.MAGIC_PREFIX))
247 bytes.seek(0)
248 if magic == format.MAGIC_PREFIX:
249 # FIXME: This seems like it will copy strings around
250 # more than is strictly necessary. The zipfile
251 # will read the string and then
252 # the format.read_array will copy the string
253 # to another place in memory.
254 # It would be better if the zipfile could read
255 # (or at least uncompress) the data
256 # directly into the array memory.
257 return format.read_array(
258 bytes,
259 allow_pickle=self.allow_pickle,
260 pickle_kwargs=self.pickle_kwargs,
261 max_header_size=self.max_header_size
262 )
263 else:
264 return bytes.read()
266 def __contains__(self, key):
267 return (key in self._files)
269 def __repr__(self):
270 # Get filename or default to `object`
271 if isinstance(self.fid, str):
272 filename = self.fid
273 else:
274 filename = getattr(self.fid, "name", "object")
276 # Get the name of arrays
277 array_names = ', '.join(self.files[:self._MAX_REPR_ARRAY_COUNT])
278 if len(self.files) > self._MAX_REPR_ARRAY_COUNT:
279 array_names += "..."
280 return f"NpzFile {filename!r} with keys: {array_names}"
282 # Work around problems with the docstrings in the Mapping methods
283 # They contain a `->`, which confuses the type annotation interpretations
284 # of sphinx-docs. See gh-25964
286 def get(self, key, default=None, /):
287 """
288 D.get(k,[,d]) returns D[k] if k in D, else d. d defaults to None.
289 """
290 return Mapping.get(self, key, default)
292 def items(self):
293 """
294 D.items() returns a set-like object providing a view on the items
295 """
296 return Mapping.items(self)
298 def keys(self):
299 """
300 D.keys() returns a set-like object providing a view on the keys
301 """
302 return Mapping.keys(self)
304 def values(self):
305 """
306 D.values() returns a set-like object providing a view on the values
307 """
308 return Mapping.values(self)
311@set_module('numpy')
312def load(file, mmap_mode=None, allow_pickle=False, fix_imports=True,
313 encoding='ASCII', *, max_header_size=_MAX_HEADER_SIZE):
314 """
315 Load arrays or pickled objects from ``.npy``, ``.npz`` or pickled files.
317 .. warning:: Loading files that contain object arrays uses the ``pickle``
318 module, which is not secure against erroneous or maliciously
319 constructed data. Consider passing ``allow_pickle=False`` to
320 load data that is known not to contain object arrays for the
321 safer handling of untrusted sources.
323 Parameters
324 ----------
325 file : file-like object, string, or pathlib.Path
326 The file to read. File-like objects must support the
327 ``seek()`` and ``read()`` methods and must always
328 be opened in binary mode. Pickled files require that the
329 file-like object support the ``readline()`` method as well.
330 mmap_mode : {None, 'r+', 'r', 'w+', 'c'}, optional
331 If not None, then memory-map the file, using the given mode (see
332 `numpy.memmap` for a detailed description of the modes). A
333 memory-mapped array is kept on disk. However, it can be accessed
334 and sliced like any ndarray. Memory mapping is especially useful
335 for accessing small fragments of large files without reading the
336 entire file into memory.
337 allow_pickle : bool, optional
338 Allow loading pickled object arrays stored in npy files. Reasons for
339 disallowing pickles include security, as loading pickled data can
340 execute arbitrary code. If pickles are disallowed, loading object
341 arrays will fail. Default: False
342 fix_imports : bool, optional
343 Only useful when loading Python 2 generated pickled files,
344 which includes npy/npz files containing object arrays. If `fix_imports`
345 is True, pickle will try to map the old Python 2 names to the new names
346 used in Python 3.
347 encoding : str, optional
348 What encoding to use when reading Python 2 strings. Only useful when
349 loading Python 2 generated pickled files, which includes
350 npy/npz files containing object arrays. Values other than 'latin1',
351 'ASCII', and 'bytes' are not allowed, as they can corrupt numerical
352 data. Default: 'ASCII'
353 max_header_size : int, optional
354 Maximum allowed size of the header. Large headers may not be safe
355 to load securely and thus require explicitly passing a larger value.
356 See :py:func:`ast.literal_eval()` for details.
357 This option is ignored when `allow_pickle` is passed. In that case
358 the file is by definition trusted and the limit is unnecessary.
360 Returns
361 -------
362 result : array, tuple, dict, etc.
363 Data stored in the file. For ``.npz`` files, the returned instance
364 of NpzFile class must be closed to avoid leaking file descriptors.
366 Raises
367 ------
368 OSError
369 If the input file does not exist or cannot be read.
370 UnpicklingError
371 If ``allow_pickle=True``, but the file cannot be loaded as a pickle.
372 ValueError
373 The file contains an object array, but ``allow_pickle=False`` given.
374 EOFError
375 When calling ``np.load`` multiple times on the same file handle,
376 if all data has already been read
378 See Also
379 --------
380 save, savez, savez_compressed, loadtxt
381 memmap : Create a memory-map to an array stored in a file on disk.
382 lib.format.open_memmap : Create or load a memory-mapped ``.npy`` file.
384 Notes
385 -----
386 - If the file contains pickle data, then whatever object is stored
387 in the pickle is returned.
388 - If the file is a ``.npy`` file, then a single array is returned.
389 - If the file is a ``.npz`` file, then a dictionary-like object is
390 returned, containing ``{filename: array}`` key-value pairs, one for
391 each file in the archive.
392 - If the file is a ``.npz`` file, the returned value supports the
393 context manager protocol in a similar fashion to the open function::
395 with load('foo.npz') as data:
396 a = data['a']
398 The underlying file descriptor is closed when exiting the 'with'
399 block.
401 Examples
402 --------
403 >>> import numpy as np
405 Store data to disk, and load it again:
407 >>> np.save('/tmp/123', np.array([[1, 2, 3], [4, 5, 6]]))
408 >>> np.load('/tmp/123.npy')
409 array([[1, 2, 3],
410 [4, 5, 6]])
412 Store compressed data to disk, and load it again:
414 >>> a=np.array([[1, 2, 3], [4, 5, 6]])
415 >>> b=np.array([1, 2])
416 >>> np.savez('/tmp/123.npz', a=a, b=b)
417 >>> data = np.load('/tmp/123.npz')
418 >>> data['a']
419 array([[1, 2, 3],
420 [4, 5, 6]])
421 >>> data['b']
422 array([1, 2])
423 >>> data.close()
425 Mem-map the stored array, and then access the second row
426 directly from disk:
428 >>> X = np.load('/tmp/123.npy', mmap_mode='r')
429 >>> X[1, :]
430 memmap([4, 5, 6])
432 """
433 if encoding not in ('ASCII', 'latin1', 'bytes'):
434 # The 'encoding' value for pickle also affects what encoding
435 # the serialized binary data of NumPy arrays is loaded
436 # in. Pickle does not pass on the encoding information to
437 # NumPy. The unpickling code in numpy._core.multiarray is
438 # written to assume that unicode data appearing where binary
439 # should be is in 'latin1'. 'bytes' is also safe, as is 'ASCII'.
440 #
441 # Other encoding values can corrupt binary data, and we
442 # purposefully disallow them. For the same reason, the errors=
443 # argument is not exposed, as values other than 'strict'
444 # result can similarly silently corrupt numerical data.
445 raise ValueError("encoding must be 'ASCII', 'latin1', or 'bytes'")
447 pickle_kwargs = {'encoding': encoding, 'fix_imports': fix_imports}
449 with contextlib.ExitStack() as stack:
450 if hasattr(file, 'read'):
451 fid = file
452 own_fid = False
453 else:
454 fid = stack.enter_context(open(os.fspath(file), "rb"))
455 own_fid = True
457 # Code to distinguish from NumPy binary files and pickles.
458 _ZIP_PREFIX = b'PK\x03\x04'
459 _ZIP_SUFFIX = b'PK\x05\x06' # empty zip files start with this
460 N = len(format.MAGIC_PREFIX)
461 magic = fid.read(N)
462 if not magic:
463 raise EOFError("No data left in file")
464 # If the file size is less than N, we need to make sure not
465 # to seek past the beginning of the file
466 fid.seek(-min(N, len(magic)), 1) # back-up
467 if magic.startswith((_ZIP_PREFIX, _ZIP_SUFFIX)):
468 # zip-file (assume .npz)
469 # Potentially transfer file ownership to NpzFile
470 stack.pop_all()
471 ret = NpzFile(fid, own_fid=own_fid, allow_pickle=allow_pickle,
472 pickle_kwargs=pickle_kwargs,
473 max_header_size=max_header_size)
474 return ret
475 elif magic == format.MAGIC_PREFIX:
476 # .npy file
477 if mmap_mode:
478 if allow_pickle:
479 max_header_size = 2**64
480 return format.open_memmap(file, mode=mmap_mode,
481 max_header_size=max_header_size)
482 else:
483 return format.read_array(fid, allow_pickle=allow_pickle,
484 pickle_kwargs=pickle_kwargs,
485 max_header_size=max_header_size)
486 else:
487 # Try a pickle
488 if not allow_pickle:
489 raise ValueError(
490 "This file contains pickled (object) data. If you trust "
491 "the file you can load it unsafely using the "
492 "`allow_pickle=` keyword argument or `pickle.load()`.")
493 try:
494 return pickle.load(fid, **pickle_kwargs)
495 except Exception as e:
496 raise pickle.UnpicklingError(
497 f"Failed to interpret file {file!r} as a pickle") from e
500def _save_dispatcher(file, arr, allow_pickle=None):
501 return (arr,)
504@array_function_dispatch(_save_dispatcher)
505def save(file, arr, allow_pickle=True):
506 """
507 Save an array to a binary file in NumPy ``.npy`` format.
509 Parameters
510 ----------
511 file : file, str, or pathlib.Path
512 File or filename to which the data is saved. If file is a file-object,
513 then the filename is unchanged. If file is a string or Path,
514 a ``.npy`` extension will be appended to the filename if it does not
515 already have one.
516 arr : array_like
517 Array data to be saved.
518 allow_pickle : bool, optional
519 Allow saving object arrays using Python pickles. Reasons for
520 disallowing pickles include security (loading pickled data can execute
521 arbitrary code) and portability (pickled objects may not be loadable
522 on different Python installations, for example if the stored objects
523 require libraries that are not available, and not all pickled data is
524 compatible between different versions of Python).
525 Default: True
527 See Also
528 --------
529 savez : Save several arrays into a ``.npz`` archive
530 savetxt, load
532 Notes
533 -----
534 For a description of the ``.npy`` format, see :py:mod:`numpy.lib.format`.
536 Any data saved to the file is appended to the end of the file.
538 Examples
539 --------
540 >>> import numpy as np
542 >>> from tempfile import TemporaryFile
543 >>> outfile = TemporaryFile()
545 >>> x = np.arange(10)
546 >>> np.save(outfile, x)
548 >>> _ = outfile.seek(0) # Only needed to simulate closing & reopening file
549 >>> np.load(outfile)
550 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
553 >>> with open('test.npy', 'wb') as f:
554 ... np.save(f, np.array([1, 2]))
555 ... np.save(f, np.array([1, 3]))
556 >>> with open('test.npy', 'rb') as f:
557 ... a = np.load(f)
558 ... b = np.load(f)
559 >>> print(a, b)
560 # [1 2] [1 3]
561 """
562 if hasattr(file, 'write'):
563 file_ctx = contextlib.nullcontext(file)
564 else:
565 file = os.fspath(file)
566 if not file.endswith('.npy'):
567 file = file + '.npy'
568 file_ctx = open(file, "wb")
570 with file_ctx as fid:
571 arr = np.asanyarray(arr)
572 format.write_array(fid, arr, allow_pickle=allow_pickle)
575def _savez_dispatcher(file, *args, allow_pickle=True, **kwds):
576 yield from args
577 yield from kwds.values()
580@array_function_dispatch(_savez_dispatcher)
581def savez(file, *args, allow_pickle=True, **kwds):
582 """Save several arrays into a single file in uncompressed ``.npz`` format.
584 Provide arrays as keyword arguments to store them under the
585 corresponding name in the output file: ``savez(fn, x=x, y=y)``.
587 If arrays are specified as positional arguments, i.e., ``savez(fn,
588 x, y)``, their names will be `arr_0`, `arr_1`, etc.
590 Parameters
591 ----------
592 file : file, str, or pathlib.Path
593 Either the filename (string) or an open file (file-like object)
594 where the data will be saved. If file is a string or a Path, the
595 ``.npz`` extension will be appended to the filename if it is not
596 already there.
597 args : Arguments, optional
598 Arrays to save to the file. Please use keyword arguments (see
599 `kwds` below) to assign names to arrays. Arrays specified as
600 args will be named "arr_0", "arr_1", and so on.
601 allow_pickle : bool, optional
602 Allow saving object arrays using Python pickles. Reasons for
603 disallowing pickles include security (loading pickled data can execute
604 arbitrary code) and portability (pickled objects may not be loadable
605 on different Python installations, for example if the stored objects
606 require libraries that are not available, and not all pickled data is
607 compatible between different versions of Python).
608 Default: True
609 kwds : Keyword arguments, optional
610 Arrays to save to the file. Each array will be saved to the
611 output file with its corresponding keyword name.
613 Returns
614 -------
615 None
617 See Also
618 --------
619 save : Save a single array to a binary file in NumPy format.
620 savetxt : Save an array to a file as plain text.
621 savez_compressed : Save several arrays into a compressed ``.npz`` archive
623 Notes
624 -----
625 The ``.npz`` file format is a zipped archive of files named after the
626 variables they contain. The archive is not compressed and each file
627 in the archive contains one variable in ``.npy`` format. For a
628 description of the ``.npy`` format, see :py:mod:`numpy.lib.format`.
630 When opening the saved ``.npz`` file with `load` a `~lib.npyio.NpzFile`
631 object is returned. This is a dictionary-like object which can be queried
632 for its list of arrays (with the ``.files`` attribute), and for the arrays
633 themselves.
635 Keys passed in `kwds` are used as filenames inside the ZIP archive.
636 Therefore, keys should be valid filenames; e.g., avoid keys that begin with
637 ``/`` or contain ``.``.
639 When naming variables with keyword arguments, it is not possible to name a
640 variable ``file``, as this would cause the ``file`` argument to be defined
641 twice in the call to ``savez``.
643 Examples
644 --------
645 >>> import numpy as np
646 >>> from tempfile import TemporaryFile
647 >>> outfile = TemporaryFile()
648 >>> x = np.arange(10)
649 >>> y = np.sin(x)
651 Using `savez` with \\*args, the arrays are saved with default names.
653 >>> np.savez(outfile, x, y)
654 >>> _ = outfile.seek(0) # Only needed to simulate closing & reopening file
655 >>> npzfile = np.load(outfile)
656 >>> npzfile.files
657 ['arr_0', 'arr_1']
658 >>> npzfile['arr_0']
659 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
661 Using `savez` with \\**kwds, the arrays are saved with the keyword names.
663 >>> outfile = TemporaryFile()
664 >>> np.savez(outfile, x=x, y=y)
665 >>> _ = outfile.seek(0)
666 >>> npzfile = np.load(outfile)
667 >>> sorted(npzfile.files)
668 ['x', 'y']
669 >>> npzfile['x']
670 array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
672 """
673 _savez(file, args, kwds, False, allow_pickle=allow_pickle)
676def _savez_compressed_dispatcher(file, *args, allow_pickle=True, **kwds):
677 yield from args
678 yield from kwds.values()
681@array_function_dispatch(_savez_compressed_dispatcher)
682def savez_compressed(file, *args, allow_pickle=True, **kwds):
683 """
684 Save several arrays into a single file in compressed ``.npz`` format.
686 Provide arrays as keyword arguments to store them under the
687 corresponding name in the output file: ``savez_compressed(fn, x=x, y=y)``.
689 If arrays are specified as positional arguments, i.e.,
690 ``savez_compressed(fn, x, y)``, their names will be `arr_0`, `arr_1`, etc.
692 Parameters
693 ----------
694 file : file, str, or pathlib.Path
695 Either the filename (string) or an open file (file-like object)
696 where the data will be saved. If file is a string or a Path, the
697 ``.npz`` extension will be appended to the filename if it is not
698 already there.
699 args : Arguments, optional
700 Arrays to save to the file. Please use keyword arguments (see
701 `kwds` below) to assign names to arrays. Arrays specified as
702 args will be named "arr_0", "arr_1", and so on.
703 allow_pickle : bool, optional
704 Allow saving object arrays using Python pickles. Reasons for
705 disallowing pickles include security (loading pickled data can execute
706 arbitrary code) and portability (pickled objects may not be loadable
707 on different Python installations, for example if the stored objects
708 require libraries that are not available, and not all pickled data is
709 compatible between different versions of Python).
710 Default: True
711 kwds : Keyword arguments, optional
712 Arrays to save to the file. Each array will be saved to the
713 output file with its corresponding keyword name.
715 Returns
716 -------
717 None
719 See Also
720 --------
721 numpy.save : Save a single array to a binary file in NumPy format.
722 numpy.savetxt : Save an array to a file as plain text.
723 numpy.savez : Save several arrays into an uncompressed ``.npz`` file format
724 numpy.load : Load the files created by savez_compressed.
726 Notes
727 -----
728 The ``.npz`` file format is a zipped archive of files named after the
729 variables they contain. The archive is compressed with
730 ``zipfile.ZIP_DEFLATED`` and each file in the archive contains one variable
731 in ``.npy`` format. For a description of the ``.npy`` format, see
732 :py:mod:`numpy.lib.format`.
735 When opening the saved ``.npz`` file with `load` a `~lib.npyio.NpzFile`
736 object is returned. This is a dictionary-like object which can be queried
737 for its list of arrays (with the ``.files`` attribute), and for the arrays
738 themselves.
740 Examples
741 --------
742 >>> import numpy as np
743 >>> test_array = np.random.rand(3, 2)
744 >>> test_vector = np.random.rand(4)
745 >>> np.savez_compressed('/tmp/123', a=test_array, b=test_vector)
746 >>> loaded = np.load('/tmp/123.npz')
747 >>> print(np.array_equal(test_array, loaded['a']))
748 True
749 >>> print(np.array_equal(test_vector, loaded['b']))
750 True
752 """
753 _savez(file, args, kwds, True, allow_pickle=allow_pickle)
756def _savez(file, args, kwds, compress, allow_pickle=True, pickle_kwargs=None):
757 # Import is postponed to here since zipfile depends on gzip, an optional
758 # component of the so-called standard library.
759 import zipfile
761 if not hasattr(file, 'write'):
762 file = os.fspath(file)
763 if not file.endswith('.npz'):
764 file = file + '.npz'
766 namedict = kwds
767 for i, val in enumerate(args):
768 key = 'arr_%d' % i
769 if key in namedict.keys():
770 raise ValueError(
771 f"Cannot use un-named variables and keyword {key}")
772 namedict[key] = val
774 if compress:
775 compression = zipfile.ZIP_DEFLATED
776 else:
777 compression = zipfile.ZIP_STORED
779 zipf = zipfile_factory(file, mode="w", compression=compression)
780 try:
781 for key, val in namedict.items():
782 fname = key + '.npy'
783 val = np.asanyarray(val)
784 # always force zip64, gh-10776
785 with zipf.open(fname, 'w', force_zip64=True) as fid:
786 format.write_array(fid, val,
787 allow_pickle=allow_pickle,
788 pickle_kwargs=pickle_kwargs)
789 finally:
790 zipf.close()
793def _ensure_ndmin_ndarray_check_param(ndmin):
794 """Just checks if the param ndmin is supported on
795 _ensure_ndmin_ndarray. It is intended to be used as
796 verification before running anything expensive.
797 e.g. loadtxt, genfromtxt
798 """
799 # Check correctness of the values of `ndmin`
800 if ndmin not in [0, 1, 2]:
801 raise ValueError(f"Illegal value of ndmin keyword: {ndmin}")
803def _ensure_ndmin_ndarray(a, *, ndmin: int):
804 """This is a helper function of loadtxt and genfromtxt to ensure
805 proper minimum dimension as requested
807 ndim : int. Supported values 1, 2, 3
808 ^^ whenever this changes, keep in sync with
809 _ensure_ndmin_ndarray_check_param
810 """
811 # Verify that the array has at least dimensions `ndmin`.
812 # Tweak the size and shape of the arrays - remove extraneous dimensions
813 if a.ndim > ndmin:
814 a = np.squeeze(a)
815 # and ensure we have the minimum number of dimensions asked for
816 # - has to be in this order for the odd case ndmin=1, a.squeeze().ndim=0
817 if a.ndim < ndmin:
818 if ndmin == 1:
819 a = np.atleast_1d(a)
820 elif ndmin == 2:
821 a = np.atleast_2d(a).T
823 return a
826# amount of lines loadtxt reads in one chunk, can be overridden for testing
827_loadtxt_chunksize = 50000
830def _check_nonneg_int(value, name="argument"):
831 try:
832 operator.index(value)
833 except TypeError:
834 raise TypeError(f"{name} must be an integer") from None
835 if value < 0:
836 raise ValueError(f"{name} must be nonnegative")
839def _preprocess_comments(iterable, comments, encoding):
840 """
841 Generator that consumes a line iterated iterable and strips out the
842 multiple (or multi-character) comments from lines.
843 This is a pre-processing step to achieve feature parity with loadtxt
844 (we assume that this feature is a nieche feature).
845 """
846 for line in iterable:
847 if isinstance(line, bytes):
848 # Need to handle conversion here, or the splitting would fail
849 line = line.decode(encoding)
851 for c in comments:
852 line = line.split(c, 1)[0]
854 yield line
857# The number of rows we read in one go if confronted with a parametric dtype
858_loadtxt_chunksize = 50000
861def _read(fname, *, delimiter=',', comment='#', quote='"',
862 imaginary_unit='j', usecols=None, skiplines=0,
863 max_rows=None, converters=None, ndmin=None, unpack=False,
864 dtype=np.float64, encoding=None):
865 r"""
866 Read a NumPy array from a text file.
867 This is a helper function for loadtxt.
869 Parameters
870 ----------
871 fname : file, str, or pathlib.Path
872 The filename or the file to be read.
873 delimiter : str, optional
874 Field delimiter of the fields in line of the file.
875 Default is a comma, ','. If None any sequence of whitespace is
876 considered a delimiter.
877 comment : str or sequence of str or None, optional
878 Character that begins a comment. All text from the comment
879 character to the end of the line is ignored.
880 Multiple comments or multiple-character comment strings are supported,
881 but may be slower and `quote` must be empty if used.
882 Use None to disable all use of comments.
883 quote : str or None, optional
884 Character that is used to quote string fields. Default is '"'
885 (a double quote). Use None to disable quote support.
886 imaginary_unit : str, optional
887 Character that represent the imaginary unit `sqrt(-1)`.
888 Default is 'j'.
889 usecols : array_like, optional
890 A one-dimensional array of integer column numbers. These are the
891 columns from the file to be included in the array. If this value
892 is not given, all the columns are used.
893 skiplines : int, optional
894 Number of lines to skip before interpreting the data in the file.
895 max_rows : int, optional
896 Maximum number of rows of data to read. Default is to read the
897 entire file.
898 converters : dict or callable, optional
899 A function to parse all columns strings into the desired value, or
900 a dictionary mapping column number to a parser function.
901 E.g. if column 0 is a date string: ``converters = {0: datestr2num}``.
902 Converters can also be used to provide a default value for missing
903 data, e.g. ``converters = lambda s: float(s.strip() or 0)`` will
904 convert empty fields to 0.
905 Default: None
906 ndmin : int, optional
907 Minimum dimension of the array returned.
908 Allowed values are 0, 1 or 2. Default is 0.
909 unpack : bool, optional
910 If True, the returned array is transposed, so that arguments may be
911 unpacked using ``x, y, z = read(...)``. When used with a structured
912 data-type, arrays are returned for each field. Default is False.
913 dtype : numpy data type
914 A NumPy dtype instance, can be a structured dtype to map to the
915 columns of the file.
916 encoding : str, optional
917 Encoding used to decode the inputfile. The special value 'bytes'
918 (the default) enables backwards-compatible behavior for `converters`,
919 ensuring that inputs to the converter functions are encoded
920 bytes objects. The special value 'bytes' has no additional effect if
921 ``converters=None``. If encoding is ``'bytes'`` or ``None``, the
922 default system encoding is used.
924 Returns
925 -------
926 ndarray
927 NumPy array.
928 """
929 # Handle special 'bytes' keyword for encoding
930 byte_converters = False
931 if encoding == 'bytes':
932 encoding = None
933 byte_converters = True
935 if dtype is None:
936 raise TypeError("a dtype must be provided.")
937 dtype = np.dtype(dtype)
939 read_dtype_via_object_chunks = None
940 if dtype.kind in 'SUM' and dtype in {
941 np.dtype("S0"), np.dtype("U0"), np.dtype("M8"), np.dtype("m8")}:
942 # This is a legacy "flexible" dtype. We do not truly support
943 # parametric dtypes currently (no dtype discovery step in the core),
944 # but have to support these for backward compatibility.
945 read_dtype_via_object_chunks = dtype
946 dtype = np.dtype(object)
948 if usecols is not None:
949 # Allow usecols to be a single int or a sequence of ints, the C-code
950 # handles the rest
951 try:
952 usecols = list(usecols)
953 except TypeError:
954 usecols = [usecols]
956 _ensure_ndmin_ndarray_check_param(ndmin)
958 if comment is None:
959 comments = None
960 else:
961 # assume comments are a sequence of strings
962 if "" in comment:
963 raise ValueError(
964 "comments cannot be an empty string. Use comments=None to "
965 "disable comments."
966 )
967 comments = tuple(comment)
968 comment = None
969 if len(comments) == 0:
970 comments = None # No comments at all
971 elif len(comments) == 1:
972 # If there is only one comment, and that comment has one character,
973 # the normal parsing can deal with it just fine.
974 if isinstance(comments[0], str) and len(comments[0]) == 1:
975 comment = comments[0]
976 comments = None
977 # Input validation if there are multiple comment characters
978 elif delimiter in comments:
979 raise TypeError(
980 f"Comment characters '{comments}' cannot include the "
981 f"delimiter '{delimiter}'"
982 )
984 # comment is now either a 1 or 0 character string or a tuple:
985 if comments is not None:
986 # Note: An earlier version support two character comments (and could
987 # have been extended to multiple characters, we assume this is
988 # rare enough to not optimize for.
989 if quote is not None:
990 raise ValueError(
991 "when multiple comments or a multi-character comment is "
992 "given, quotes are not supported. In this case quotechar "
993 "must be set to None.")
995 if len(imaginary_unit) != 1:
996 raise ValueError('len(imaginary_unit) must be 1.')
998 _check_nonneg_int(skiplines)
999 if max_rows is not None:
1000 _check_nonneg_int(max_rows)
1001 else:
1002 # Passing -1 to the C code means "read the entire file".
1003 max_rows = -1
1005 fh_closing_ctx = contextlib.nullcontext()
1006 filelike = False
1007 try:
1008 if isinstance(fname, os.PathLike):
1009 fname = os.fspath(fname)
1010 if isinstance(fname, str):
1011 fh = np.lib._datasource.open(fname, 'rt', encoding=encoding)
1012 if encoding is None:
1013 encoding = getattr(fh, 'encoding', 'latin1')
1015 fh_closing_ctx = contextlib.closing(fh)
1016 data = fh
1017 filelike = True
1018 else:
1019 if encoding is None:
1020 encoding = getattr(fname, 'encoding', 'latin1')
1021 data = iter(fname)
1022 except TypeError as e:
1023 raise ValueError(
1024 f"fname must be a string, filehandle, list of strings,\n"
1025 f"or generator. Got {type(fname)} instead.") from e
1027 with fh_closing_ctx:
1028 if comments is not None:
1029 if filelike:
1030 data = iter(data)
1031 filelike = False
1032 data = _preprocess_comments(data, comments, encoding)
1034 if read_dtype_via_object_chunks is None:
1035 arr = _load_from_filelike(
1036 data, delimiter=delimiter, comment=comment, quote=quote,
1037 imaginary_unit=imaginary_unit,
1038 usecols=usecols, skiplines=skiplines, max_rows=max_rows,
1039 converters=converters, dtype=dtype,
1040 encoding=encoding, filelike=filelike,
1041 byte_converters=byte_converters)
1043 else:
1044 # This branch reads the file into chunks of object arrays and then
1045 # casts them to the desired actual dtype. This ensures correct
1046 # string-length and datetime-unit discovery (like `arr.astype()`).
1047 # Due to chunking, certain error reports are less clear, currently.
1048 if filelike:
1049 data = iter(data) # cannot chunk when reading from file
1050 filelike = False
1052 c_byte_converters = False
1053 if read_dtype_via_object_chunks == "S":
1054 c_byte_converters = True # Use latin1 rather than ascii
1056 chunks = []
1057 while max_rows != 0:
1058 if max_rows < 0:
1059 chunk_size = _loadtxt_chunksize
1060 else:
1061 chunk_size = min(_loadtxt_chunksize, max_rows)
1063 next_arr = _load_from_filelike(
1064 data, delimiter=delimiter, comment=comment, quote=quote,
1065 imaginary_unit=imaginary_unit,
1066 usecols=usecols, skiplines=skiplines, max_rows=chunk_size,
1067 converters=converters, dtype=dtype,
1068 encoding=encoding, filelike=filelike,
1069 byte_converters=byte_converters,
1070 c_byte_converters=c_byte_converters)
1071 # Cast here already. We hope that this is better even for
1072 # large files because the storage is more compact. It could
1073 # be adapted (in principle the concatenate could cast).
1074 chunks.append(next_arr.astype(read_dtype_via_object_chunks))
1076 skiplines = 0 # Only have to skip for first chunk
1077 if max_rows >= 0:
1078 max_rows -= chunk_size
1079 if len(next_arr) < chunk_size:
1080 # There was less data than requested, so we are done.
1081 break
1083 # Need at least one chunk, but if empty, the last one may have
1084 # the wrong shape.
1085 if len(chunks) > 1 and len(chunks[-1]) == 0:
1086 del chunks[-1]
1087 if len(chunks) == 1:
1088 arr = chunks[0]
1089 else:
1090 arr = np.concatenate(chunks, axis=0)
1092 # NOTE: ndmin works as advertised for structured dtypes, but normally
1093 # these would return a 1D result plus the structured dimension,
1094 # so ndmin=2 adds a third dimension even when no squeezing occurs.
1095 # A `squeeze=False` could be a better solution (pandas uses squeeze).
1096 arr = _ensure_ndmin_ndarray(arr, ndmin=ndmin)
1098 if arr.shape:
1099 if arr.shape[0] == 0:
1100 warnings.warn(
1101 f'loadtxt: input contained no data: "{fname}"',
1102 category=UserWarning,
1103 stacklevel=3
1104 )
1106 if unpack:
1107 # Unpack structured dtypes if requested:
1108 dt = arr.dtype
1109 if dt.names is not None:
1110 # For structured arrays, return an array for each field.
1111 return [arr[field] for field in dt.names]
1112 else:
1113 return arr.T
1114 else:
1115 return arr
1118@finalize_array_function_like
1119@set_module('numpy')
1120def loadtxt(fname, dtype=float, comments='#', delimiter=None,
1121 converters=None, skiprows=0, usecols=None, unpack=False,
1122 ndmin=0, encoding=None, max_rows=None, *, quotechar=None,
1123 like=None):
1124 r"""
1125 Load data from a text file.
1127 Parameters
1128 ----------
1129 fname : file, str, pathlib.Path, list of str, generator
1130 File, filename, list, or generator to read. If the filename
1131 extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note
1132 that generators must return bytes or strings. The strings
1133 in a list or produced by a generator are treated as lines.
1134 dtype : data-type, optional
1135 Data-type of the resulting array; default: float. If this is a
1136 structured data-type, the resulting array will be 1-dimensional, and
1137 each row will be interpreted as an element of the array. In this
1138 case, the number of columns used must match the number of fields in
1139 the data-type.
1140 comments : str or sequence of str or None, optional
1141 The characters or list of characters used to indicate the start of a
1142 comment. None implies no comments. For backwards compatibility, byte
1143 strings will be decoded as 'latin1'. The default is '#'.
1144 delimiter : str, optional
1145 The character used to separate the values. For backwards compatibility,
1146 byte strings will be decoded as 'latin1'. The default is whitespace.
1148 .. versionchanged:: 1.23.0
1149 Only single character delimiters are supported. Newline characters
1150 cannot be used as the delimiter.
1152 converters : dict or callable, optional
1153 Converter functions to customize value parsing. If `converters` is
1154 callable, the function is applied to all columns, else it must be a
1155 dict that maps column number to a parser function.
1156 See examples for further details.
1157 Default: None.
1159 .. versionchanged:: 1.23.0
1160 The ability to pass a single callable to be applied to all columns
1161 was added.
1163 skiprows : int, optional
1164 Skip the first `skiprows` lines, including comments; default: 0.
1165 usecols : int or sequence, optional
1166 Which columns to read, with 0 being the first. For example,
1167 ``usecols = (1,4,5)`` will extract the 2nd, 5th and 6th columns.
1168 The default, None, results in all columns being read.
1169 unpack : bool, optional
1170 If True, the returned array is transposed, so that arguments may be
1171 unpacked using ``x, y, z = loadtxt(...)``. When used with a
1172 structured data-type, arrays are returned for each field.
1173 Default is False.
1174 ndmin : int, optional
1175 The returned array will have at least `ndmin` dimensions.
1176 Otherwise mono-dimensional axes will be squeezed.
1177 Legal values: 0 (default), 1 or 2.
1178 encoding : str, optional
1179 Encoding used to decode the inputfile. Does not apply to input streams.
1180 The special value 'bytes' enables backward compatibility workarounds
1181 that ensures you receive byte arrays as results if possible and passes
1182 'latin1' encoded strings to converters. Override this value to receive
1183 unicode arrays and pass strings as input to converters. If set to None
1184 the system default is used. The default value is None.
1186 .. versionchanged:: 2.0
1187 Before NumPy 2, the default was ``'bytes'`` for Python 2
1188 compatibility. The default is now ``None``.
1190 max_rows : int, optional
1191 Read `max_rows` rows of content after `skiprows` lines. The default is
1192 to read all the rows. Note that empty rows containing no data such as
1193 empty lines and comment lines are not counted towards `max_rows`,
1194 while such lines are counted in `skiprows`.
1196 .. versionchanged:: 1.23.0
1197 Lines containing no data, including comment lines (e.g., lines
1198 starting with '#' or as specified via `comments`) are not counted
1199 towards `max_rows`.
1200 quotechar : unicode character or None, optional
1201 The character used to denote the start and end of a quoted item.
1202 Occurrences of the delimiter or comment characters are ignored within
1203 a quoted item. The default value is ``quotechar=None``, which means
1204 quoting support is disabled.
1206 If two consecutive instances of `quotechar` are found within a quoted
1207 field, the first is treated as an escape character. See examples.
1209 .. versionadded:: 1.23.0
1210 ${ARRAY_FUNCTION_LIKE}
1212 .. versionadded:: 1.20.0
1214 Returns
1215 -------
1216 out : ndarray
1217 Data read from the text file.
1219 See Also
1220 --------
1221 load, fromstring, fromregex
1222 genfromtxt : Load data with missing values handled as specified.
1223 scipy.io.loadmat : reads MATLAB data files
1225 Notes
1226 -----
1227 This function aims to be a fast reader for simply formatted files. The
1228 `genfromtxt` function provides more sophisticated handling of, e.g.,
1229 lines with missing values.
1231 Each row in the input text file must have the same number of values to be
1232 able to read all values. If all rows do not have same number of values, a
1233 subset of up to n columns (where n is the least number of values present
1234 in all rows) can be read by specifying the columns via `usecols`.
1236 The strings produced by the Python float.hex method can be used as
1237 input for floats.
1239 Examples
1240 --------
1241 >>> import numpy as np
1242 >>> from io import StringIO # StringIO behaves like a file object
1243 >>> c = StringIO("0 1\n2 3")
1244 >>> np.loadtxt(c)
1245 array([[0., 1.],
1246 [2., 3.]])
1248 >>> d = StringIO("M 21 72\nF 35 58")
1249 >>> np.loadtxt(d, dtype={'names': ('gender', 'age', 'weight'),
1250 ... 'formats': ('S1', 'i4', 'f4')})
1251 array([(b'M', 21, 72.), (b'F', 35, 58.)],
1252 dtype=[('gender', 'S1'), ('age', '<i4'), ('weight', '<f4')])
1254 >>> c = StringIO("1,0,2\n3,0,4")
1255 >>> x, y = np.loadtxt(c, delimiter=',', usecols=(0, 2), unpack=True)
1256 >>> x
1257 array([1., 3.])
1258 >>> y
1259 array([2., 4.])
1261 The `converters` argument is used to specify functions to preprocess the
1262 text prior to parsing. `converters` can be a dictionary that maps
1263 preprocessing functions to each column:
1265 >>> s = StringIO("1.618, 2.296\n3.141, 4.669\n")
1266 >>> conv = {
1267 ... 0: lambda x: np.floor(float(x)), # conversion fn for column 0
1268 ... 1: lambda x: np.ceil(float(x)), # conversion fn for column 1
1269 ... }
1270 >>> np.loadtxt(s, delimiter=",", converters=conv)
1271 array([[1., 3.],
1272 [3., 5.]])
1274 `converters` can be a callable instead of a dictionary, in which case it
1275 is applied to all columns:
1277 >>> s = StringIO("0xDE 0xAD\n0xC0 0xDE")
1278 >>> import functools
1279 >>> conv = functools.partial(int, base=16)
1280 >>> np.loadtxt(s, converters=conv)
1281 array([[222., 173.],
1282 [192., 222.]])
1284 This example shows how `converters` can be used to convert a field
1285 with a trailing minus sign into a negative number.
1287 >>> s = StringIO("10.01 31.25-\n19.22 64.31\n17.57- 63.94")
1288 >>> def conv(fld):
1289 ... return -float(fld[:-1]) if fld.endswith("-") else float(fld)
1290 ...
1291 >>> np.loadtxt(s, converters=conv)
1292 array([[ 10.01, -31.25],
1293 [ 19.22, 64.31],
1294 [-17.57, 63.94]])
1296 Using a callable as the converter can be particularly useful for handling
1297 values with different formatting, e.g. floats with underscores:
1299 >>> s = StringIO("1 2.7 100_000")
1300 >>> np.loadtxt(s, converters=float)
1301 array([1.e+00, 2.7e+00, 1.e+05])
1303 This idea can be extended to automatically handle values specified in
1304 many different formats, such as hex values:
1306 >>> def conv(val):
1307 ... try:
1308 ... return float(val)
1309 ... except ValueError:
1310 ... return float.fromhex(val)
1311 >>> s = StringIO("1, 2.5, 3_000, 0b4, 0x1.4000000000000p+2")
1312 >>> np.loadtxt(s, delimiter=",", converters=conv)
1313 array([1.0e+00, 2.5e+00, 3.0e+03, 1.8e+02, 5.0e+00])
1315 Or a format where the ``-`` sign comes after the number:
1317 >>> s = StringIO("10.01 31.25-\n19.22 64.31\n17.57- 63.94")
1318 >>> conv = lambda x: -float(x[:-1]) if x.endswith("-") else float(x)
1319 >>> np.loadtxt(s, converters=conv)
1320 array([[ 10.01, -31.25],
1321 [ 19.22, 64.31],
1322 [-17.57, 63.94]])
1324 Support for quoted fields is enabled with the `quotechar` parameter.
1325 Comment and delimiter characters are ignored when they appear within a
1326 quoted item delineated by `quotechar`:
1328 >>> s = StringIO('"alpha, #42", 10.0\n"beta, #64", 2.0\n')
1329 >>> dtype = np.dtype([("label", "U12"), ("value", float)])
1330 >>> np.loadtxt(s, dtype=dtype, delimiter=",", quotechar='"')
1331 array([('alpha, #42', 10.), ('beta, #64', 2.)],
1332 dtype=[('label', '<U12'), ('value', '<f8')])
1334 Quoted fields can be separated by multiple whitespace characters:
1336 >>> s = StringIO('"alpha, #42" 10.0\n"beta, #64" 2.0\n')
1337 >>> dtype = np.dtype([("label", "U12"), ("value", float)])
1338 >>> np.loadtxt(s, dtype=dtype, delimiter=None, quotechar='"')
1339 array([('alpha, #42', 10.), ('beta, #64', 2.)],
1340 dtype=[('label', '<U12'), ('value', '<f8')])
1342 Two consecutive quote characters within a quoted field are treated as a
1343 single escaped character:
1345 >>> s = StringIO('"Hello, my name is ""Monty""!"')
1346 >>> np.loadtxt(s, dtype="U", delimiter=",", quotechar='"')
1347 array('Hello, my name is "Monty"!', dtype='<U26')
1349 Read subset of columns when all rows do not contain equal number of values:
1351 >>> d = StringIO("1 2\n2 4\n3 9 12\n4 16 20")
1352 >>> np.loadtxt(d, usecols=(0, 1))
1353 array([[ 1., 2.],
1354 [ 2., 4.],
1355 [ 3., 9.],
1356 [ 4., 16.]])
1358 """
1360 if like is not None:
1361 return _loadtxt_with_like(
1362 like, fname, dtype=dtype, comments=comments, delimiter=delimiter,
1363 converters=converters, skiprows=skiprows, usecols=usecols,
1364 unpack=unpack, ndmin=ndmin, encoding=encoding,
1365 max_rows=max_rows
1366 )
1368 if isinstance(delimiter, bytes):
1369 delimiter.decode("latin1")
1371 if dtype is None:
1372 dtype = np.float64
1374 comment = comments
1375 # Control character type conversions for Py3 convenience
1376 if comment is not None:
1377 if isinstance(comment, (str, bytes)):
1378 comment = [comment]
1379 comment = [
1380 x.decode('latin1') if isinstance(x, bytes) else x for x in comment]
1381 if isinstance(delimiter, bytes):
1382 delimiter = delimiter.decode('latin1')
1384 arr = _read(fname, dtype=dtype, comment=comment, delimiter=delimiter,
1385 converters=converters, skiplines=skiprows, usecols=usecols,
1386 unpack=unpack, ndmin=ndmin, encoding=encoding,
1387 max_rows=max_rows, quote=quotechar)
1389 return arr
1392_loadtxt_with_like = array_function_dispatch()(loadtxt)
1395def _savetxt_dispatcher(fname, X, fmt=None, delimiter=None, newline=None,
1396 header=None, footer=None, comments=None,
1397 encoding=None):
1398 return (X,)
1401@array_function_dispatch(_savetxt_dispatcher)
1402def savetxt(fname, X, fmt='%.18e', delimiter=' ', newline='\n', header='',
1403 footer='', comments='# ', encoding=None):
1404 """
1405 Save an array to a text file.
1407 Parameters
1408 ----------
1409 fname : filename, file handle or pathlib.Path
1410 If the filename ends in ``.gz``, the file is automatically saved in
1411 compressed gzip format. `loadtxt` understands gzipped files
1412 transparently.
1413 X : 1D or 2D array_like
1414 Data to be saved to a text file.
1415 fmt : str or sequence of strs, optional
1416 A single format (%10.5f), a sequence of formats, or a
1417 multi-format string, e.g. 'Iteration %d -- %10.5f', in which
1418 case `delimiter` is ignored. For complex `X`, the legal options
1419 for `fmt` are:
1421 * a single specifier, ``fmt='%.4e'``, resulting in numbers formatted
1422 like ``' (%s+%sj)' % (fmt, fmt)``
1423 * a full string specifying every real and imaginary part, e.g.
1424 ``' %.4e %+.4ej %.4e %+.4ej %.4e %+.4ej'`` for 3 columns
1425 * a list of specifiers, one per column - in this case, the real
1426 and imaginary part must have separate specifiers,
1427 e.g. ``['%.3e + %.3ej', '(%.15e%+.15ej)']`` for 2 columns
1428 delimiter : str, optional
1429 String or character separating columns.
1430 newline : str, optional
1431 String or character separating lines.
1432 header : str, optional
1433 String that will be written at the beginning of the file.
1434 footer : str, optional
1435 String that will be written at the end of the file.
1436 comments : str, optional
1437 String that will be prepended to the ``header`` and ``footer`` strings,
1438 to mark them as comments. Default: '# ', as expected by e.g.
1439 ``numpy.loadtxt``.
1440 encoding : {None, str}, optional
1441 Encoding used to encode the outputfile. Does not apply to output
1442 streams. If the encoding is something other than 'bytes' or 'latin1'
1443 you will not be able to load the file in NumPy versions < 1.14. Default
1444 is 'latin1'.
1446 See Also
1447 --------
1448 save : Save an array to a binary file in NumPy ``.npy`` format
1449 savez : Save several arrays into an uncompressed ``.npz`` archive
1450 savez_compressed : Save several arrays into a compressed ``.npz`` archive
1452 Notes
1453 -----
1454 Further explanation of the `fmt` parameter
1455 (``%[flag]width[.precision]specifier``):
1457 flags:
1458 ``-`` : left justify
1460 ``+`` : Forces to precede result with + or -.
1462 ``0`` : Left pad the number with zeros instead of space (see width).
1464 width:
1465 Minimum number of characters to be printed. The value is not truncated
1466 if it has more characters.
1468 precision:
1469 - For integer specifiers (eg. ``d,i,o,x``), the minimum number of
1470 digits.
1471 - For ``e, E`` and ``f`` specifiers, the number of digits to print
1472 after the decimal point.
1473 - For ``g`` and ``G``, the maximum number of significant digits.
1474 - For ``s``, the maximum number of characters.
1476 specifiers:
1477 ``c`` : character
1479 ``d`` or ``i`` : signed decimal integer
1481 ``e`` or ``E`` : scientific notation with ``e`` or ``E``.
1483 ``f`` : decimal floating point
1485 ``g,G`` : use the shorter of ``e,E`` or ``f``
1487 ``o`` : signed octal
1489 ``s`` : string of characters
1491 ``u`` : unsigned decimal integer
1493 ``x,X`` : unsigned hexadecimal integer
1495 This explanation of ``fmt`` is not complete, for an exhaustive
1496 specification see [1]_.
1498 References
1499 ----------
1500 .. [1] `Format Specification Mini-Language
1501 <https://docs.python.org/library/string.html#format-specification-mini-language>`_,
1502 Python Documentation.
1504 Examples
1505 --------
1506 >>> import numpy as np
1507 >>> x = y = z = np.arange(0.0,5.0,1.0)
1508 >>> np.savetxt('test.out', x, delimiter=',') # X is an array
1509 >>> np.savetxt('test.out', (x,y,z)) # x,y,z equal sized 1D arrays
1510 >>> np.savetxt('test.out', x, fmt='%1.4e') # use exponential notation
1512 """
1514 class WriteWrap:
1515 """Convert to bytes on bytestream inputs.
1517 """
1518 def __init__(self, fh, encoding):
1519 self.fh = fh
1520 self.encoding = encoding
1521 self.do_write = self.first_write
1523 def close(self):
1524 self.fh.close()
1526 def write(self, v):
1527 self.do_write(v)
1529 def write_bytes(self, v):
1530 if isinstance(v, bytes):
1531 self.fh.write(v)
1532 else:
1533 self.fh.write(v.encode(self.encoding))
1535 def write_normal(self, v):
1536 self.fh.write(asunicode(v))
1538 def first_write(self, v):
1539 try:
1540 self.write_normal(v)
1541 self.write = self.write_normal
1542 except TypeError:
1543 # input is probably a bytestream
1544 self.write_bytes(v)
1545 self.write = self.write_bytes
1547 own_fh = False
1548 if isinstance(fname, os.PathLike):
1549 fname = os.fspath(fname)
1550 if _is_string_like(fname):
1551 # datasource doesn't support creating a new file ...
1552 open(fname, 'wt').close()
1553 fh = np.lib._datasource.open(fname, 'wt', encoding=encoding)
1554 own_fh = True
1555 elif hasattr(fname, 'write'):
1556 # wrap to handle byte output streams
1557 fh = WriteWrap(fname, encoding or 'latin1')
1558 else:
1559 raise ValueError('fname must be a string or file handle')
1561 try:
1562 X = np.asarray(X)
1564 # Handle 1-dimensional arrays
1565 if X.ndim == 0 or X.ndim > 2:
1566 raise ValueError(
1567 "Expected 1D or 2D array, got %dD array instead" % X.ndim)
1568 elif X.ndim == 1:
1569 # Common case -- 1d array of numbers
1570 if X.dtype.names is None:
1571 X = np.atleast_2d(X).T
1572 ncol = 1
1574 # Complex dtype -- each field indicates a separate column
1575 else:
1576 ncol = len(X.dtype.names)
1577 else:
1578 ncol = X.shape[1]
1580 iscomplex_X = np.iscomplexobj(X)
1581 # `fmt` can be a string with multiple insertion points or a
1582 # list of formats. E.g. '%10.5f\t%10d' or ('%10.5f', '$10d')
1583 if type(fmt) in (list, tuple):
1584 if len(fmt) != ncol:
1585 raise AttributeError(f'fmt has wrong shape. {str(fmt)}')
1586 format = delimiter.join(fmt)
1587 elif isinstance(fmt, str):
1588 n_fmt_chars = fmt.count('%')
1589 error = ValueError(f'fmt has wrong number of % formats: {fmt}')
1590 if n_fmt_chars == 1:
1591 if iscomplex_X:
1592 fmt = [f' ({fmt}+{fmt}j)', ] * ncol
1593 else:
1594 fmt = [fmt, ] * ncol
1595 format = delimiter.join(fmt)
1596 elif iscomplex_X and n_fmt_chars != (2 * ncol):
1597 raise error
1598 elif ((not iscomplex_X) and n_fmt_chars != ncol):
1599 raise error
1600 else:
1601 format = fmt
1602 else:
1603 raise ValueError(f'invalid fmt: {fmt!r}')
1605 if len(header) > 0:
1606 header = header.replace('\n', '\n' + comments)
1607 fh.write(comments + header + newline)
1608 if iscomplex_X:
1609 for row in X:
1610 row2 = []
1611 for number in row:
1612 row2.extend((number.real, number.imag))
1613 s = format % tuple(row2) + newline
1614 fh.write(s.replace('+-', '-'))
1615 else:
1616 for row in X:
1617 try:
1618 v = format % tuple(row) + newline
1619 except TypeError as e:
1620 raise TypeError("Mismatch between array dtype ('%s') and "
1621 "format specifier ('%s')"
1622 % (str(X.dtype), format)) from e
1623 fh.write(v)
1625 if len(footer) > 0:
1626 footer = footer.replace('\n', '\n' + comments)
1627 fh.write(comments + footer + newline)
1628 finally:
1629 if own_fh:
1630 fh.close()
1633@set_module('numpy')
1634def fromregex(file, regexp, dtype, encoding=None):
1635 r"""
1636 Construct an array from a text file, using regular expression parsing.
1638 The returned array is always a structured array, and is constructed from
1639 all matches of the regular expression in the file. Groups in the regular
1640 expression are converted to fields of the structured array.
1642 Parameters
1643 ----------
1644 file : file, str, or pathlib.Path
1645 Filename or file object to read.
1647 .. versionchanged:: 1.22.0
1648 Now accepts `os.PathLike` implementations.
1650 regexp : str or regexp
1651 Regular expression used to parse the file.
1652 Groups in the regular expression correspond to fields in the dtype.
1653 dtype : dtype or list of dtypes
1654 Dtype for the structured array; must be a structured datatype.
1655 encoding : str, optional
1656 Encoding used to decode the inputfile. Does not apply to input streams.
1658 Returns
1659 -------
1660 output : ndarray
1661 The output array, containing the part of the content of `file` that
1662 was matched by `regexp`. `output` is always a structured array.
1664 Raises
1665 ------
1666 TypeError
1667 When `dtype` is not a valid dtype for a structured array.
1669 See Also
1670 --------
1671 fromstring, loadtxt
1673 Notes
1674 -----
1675 Dtypes for structured arrays can be specified in several forms, but all
1676 forms specify at least the data type and field name. For details see
1677 `basics.rec`.
1679 Examples
1680 --------
1681 >>> import numpy as np
1682 >>> from io import StringIO
1683 >>> text = StringIO("1312 foo\n1534 bar\n444 qux")
1685 >>> regexp = r"(\d+)\s+(...)" # match [digits, whitespace, anything]
1686 >>> output = np.fromregex(text, regexp,
1687 ... [('num', np.int64), ('key', 'S3')])
1688 >>> output
1689 array([(1312, b'foo'), (1534, b'bar'), ( 444, b'qux')],
1690 dtype=[('num', '<i8'), ('key', 'S3')])
1691 >>> output['num']
1692 array([1312, 1534, 444])
1694 """
1695 own_fh = False
1696 if not hasattr(file, "read"):
1697 file = os.fspath(file)
1698 file = np.lib._datasource.open(file, 'rt', encoding=encoding)
1699 own_fh = True
1701 try:
1702 if not isinstance(dtype, np.dtype):
1703 dtype = np.dtype(dtype)
1704 if dtype.names is None:
1705 raise TypeError('dtype must be a structured datatype.')
1707 content = file.read()
1708 if isinstance(content, bytes) and isinstance(regexp, str):
1709 regexp = asbytes(regexp)
1711 if not hasattr(regexp, 'match'):
1712 regexp = re.compile(regexp)
1713 seq = regexp.findall(content)
1714 if seq and not isinstance(seq[0], tuple):
1715 # Only one group is in the regexp.
1716 # Create the new array as a single data-type and then
1717 # re-interpret as a single-field structured array.
1718 newdtype = np.dtype(dtype[dtype.names[0]])
1719 output = np.array(seq, dtype=newdtype)
1720 output = output.view(dtype)
1721 else:
1722 output = np.array(seq, dtype=dtype)
1724 return output
1725 finally:
1726 if own_fh:
1727 file.close()
1730#####--------------------------------------------------------------------------
1731#---- --- ASCII functions ---
1732#####--------------------------------------------------------------------------
1735@finalize_array_function_like
1736@set_module('numpy')
1737def genfromtxt(fname, dtype=float, comments='#', delimiter=None,
1738 skip_header=0, skip_footer=0, converters=None,
1739 missing_values=None, filling_values=None, usecols=None,
1740 names=None, excludelist=None,
1741 deletechars=''.join(sorted(NameValidator.defaultdeletechars)), # noqa: B008
1742 replace_space='_', autostrip=False, case_sensitive=True,
1743 defaultfmt="f%i", unpack=None, usemask=False, loose=True,
1744 invalid_raise=True, max_rows=None, encoding=None,
1745 *, ndmin=0, like=None):
1746 """
1747 Load data from a text file, with missing values handled as specified.
1749 Each line past the first `skip_header` lines is split at the `delimiter`
1750 character, and characters following the `comments` character are discarded.
1752 Parameters
1753 ----------
1754 fname : file, str, pathlib.Path, list of str, generator
1755 File, filename, list, or generator to read. If the filename
1756 extension is ``.gz`` or ``.bz2``, the file is first decompressed. Note
1757 that generators must return bytes or strings. The strings
1758 in a list or produced by a generator are treated as lines.
1759 dtype : dtype, optional
1760 Data type of the resulting array.
1761 If None, the dtypes will be determined by the contents of each
1762 column, individually.
1763 comments : str, optional
1764 The character used to indicate the start of a comment.
1765 All the characters occurring on a line after a comment are discarded.
1766 delimiter : str, int, or sequence, optional
1767 The string used to separate values. By default, any consecutive
1768 whitespaces act as delimiter. An integer or sequence of integers
1769 can also be provided as width(s) of each field.
1770 skiprows : int, optional
1771 `skiprows` was removed in numpy 1.10. Please use `skip_header` instead.
1772 skip_header : int, optional
1773 The number of lines to skip at the beginning of the file.
1774 skip_footer : int, optional
1775 The number of lines to skip at the end of the file.
1776 converters : variable, optional
1777 The set of functions that convert the data of a column to a value.
1778 The converters can also be used to provide a default value
1779 for missing data: ``converters = {3: lambda s: float(s or 0)}``.
1780 missing : variable, optional
1781 `missing` was removed in numpy 1.10. Please use `missing_values`
1782 instead.
1783 missing_values : variable, optional
1784 The set of strings corresponding to missing data.
1785 filling_values : variable, optional
1786 The set of values to be used as default when the data are missing.
1787 usecols : sequence, optional
1788 Which columns to read, with 0 being the first. For example,
1789 ``usecols = (1, 4, 5)`` will extract the 2nd, 5th and 6th columns.
1790 names : {None, True, str, sequence}, optional
1791 If `names` is True, the field names are read from the first line after
1792 the first `skip_header` lines. This line can optionally be preceded
1793 by a comment delimiter. Any content before the comment delimiter is
1794 discarded. If `names` is a sequence or a single-string of
1795 comma-separated names, the names will be used to define the field
1796 names in a structured dtype. If `names` is None, the names of the
1797 dtype fields will be used, if any.
1798 excludelist : sequence, optional
1799 A list of names to exclude. This list is appended to the default list
1800 ['return','file','print']. Excluded names are appended with an
1801 underscore: for example, `file` would become `file_`.
1802 deletechars : str, optional
1803 A string combining invalid characters that must be deleted from the
1804 names.
1805 defaultfmt : str, optional
1806 A format used to define default field names, such as "f%i" or "f_%02i".
1807 autostrip : bool, optional
1808 Whether to automatically strip white spaces from the variables.
1809 replace_space : char, optional
1810 Character(s) used in replacement of white spaces in the variable
1811 names. By default, use a '_'.
1812 case_sensitive : {True, False, 'upper', 'lower'}, optional
1813 If True, field names are case sensitive.
1814 If False or 'upper', field names are converted to upper case.
1815 If 'lower', field names are converted to lower case.
1816 unpack : bool, optional
1817 If True, the returned array is transposed, so that arguments may be
1818 unpacked using ``x, y, z = genfromtxt(...)``. When used with a
1819 structured data-type, arrays are returned for each field.
1820 Default is False.
1821 usemask : bool, optional
1822 If True, return a masked array.
1823 If False, return a regular array.
1824 loose : bool, optional
1825 If True, do not raise errors for invalid values.
1826 invalid_raise : bool, optional
1827 If True, an exception is raised if an inconsistency is detected in the
1828 number of columns.
1829 If False, a warning is emitted and the offending lines are skipped.
1830 max_rows : int, optional
1831 The maximum number of rows to read. Must not be used with skip_footer
1832 at the same time. If given, the value must be at least 1. Default is
1833 to read the entire file.
1834 encoding : str, optional
1835 Encoding used to decode the inputfile. Does not apply when `fname`
1836 is a file object. The special value 'bytes' enables backward
1837 compatibility workarounds that ensure that you receive byte arrays
1838 when possible and passes latin1 encoded strings to converters.
1839 Override this value to receive unicode arrays and pass strings
1840 as input to converters. If set to None the system default is used.
1841 The default value is 'bytes'.
1843 .. versionchanged:: 2.0
1844 Before NumPy 2, the default was ``'bytes'`` for Python 2
1845 compatibility. The default is now ``None``.
1847 ndmin : int, optional
1848 Same parameter as `loadtxt`
1850 .. versionadded:: 1.23.0
1851 ${ARRAY_FUNCTION_LIKE}
1853 .. versionadded:: 1.20.0
1855 Returns
1856 -------
1857 out : ndarray
1858 Data read from the text file. If `usemask` is True, this is a
1859 masked array.
1861 See Also
1862 --------
1863 numpy.loadtxt : equivalent function when no data is missing.
1865 Notes
1866 -----
1867 * When spaces are used as delimiters, or when no delimiter has been given
1868 as input, there should not be any missing data between two fields.
1869 * When variables are named (either by a flexible dtype or with a `names`
1870 sequence), there must not be any header in the file (else a ValueError
1871 exception is raised).
1872 * Individual values are not stripped of spaces by default.
1873 When using a custom converter, make sure the function does remove spaces.
1874 * Custom converters may receive unexpected values due to dtype
1875 discovery.
1877 References
1878 ----------
1879 .. [1] NumPy User Guide, section `I/O with NumPy
1880 <https://docs.scipy.org/doc/numpy/user/basics.io.genfromtxt.html>`_.
1882 Examples
1883 --------
1884 >>> from io import StringIO
1885 >>> import numpy as np
1887 Comma delimited file with mixed dtype
1889 >>> s = StringIO("1,1.3,abcde")
1890 >>> data = np.genfromtxt(s, dtype=[('myint','i8'),('myfloat','f8'),
1891 ... ('mystring','S5')], delimiter=",")
1892 >>> data
1893 array((1, 1.3, b'abcde'),
1894 dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])
1896 Using dtype = None
1898 >>> _ = s.seek(0) # needed for StringIO example only
1899 >>> data = np.genfromtxt(s, dtype=None,
1900 ... names = ['myint','myfloat','mystring'], delimiter=",")
1901 >>> data
1902 array((1, 1.3, 'abcde'),
1903 dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', '<U5')])
1905 Specifying dtype and names
1907 >>> _ = s.seek(0)
1908 >>> data = np.genfromtxt(s, dtype="i8,f8,S5",
1909 ... names=['myint','myfloat','mystring'], delimiter=",")
1910 >>> data
1911 array((1, 1.3, b'abcde'),
1912 dtype=[('myint', '<i8'), ('myfloat', '<f8'), ('mystring', 'S5')])
1914 An example with fixed-width columns
1916 >>> s = StringIO("11.3abcde")
1917 >>> data = np.genfromtxt(s, dtype=None, names=['intvar','fltvar','strvar'],
1918 ... delimiter=[1,3,5])
1919 >>> data
1920 array((1, 1.3, 'abcde'),
1921 dtype=[('intvar', '<i8'), ('fltvar', '<f8'), ('strvar', '<U5')])
1923 An example to show comments
1925 >>> f = StringIO('''
1926 ... text,# of chars
1927 ... hello world,11
1928 ... numpy,5''')
1929 >>> np.genfromtxt(f, dtype='S12,S12', delimiter=',')
1930 array([(b'text', b''), (b'hello world', b'11'), (b'numpy', b'5')],
1931 dtype=[('f0', 'S12'), ('f1', 'S12')])
1933 """
1935 if like is not None:
1936 return _genfromtxt_with_like(
1937 like, fname, dtype=dtype, comments=comments, delimiter=delimiter,
1938 skip_header=skip_header, skip_footer=skip_footer,
1939 converters=converters, missing_values=missing_values,
1940 filling_values=filling_values, usecols=usecols, names=names,
1941 excludelist=excludelist, deletechars=deletechars,
1942 replace_space=replace_space, autostrip=autostrip,
1943 case_sensitive=case_sensitive, defaultfmt=defaultfmt,
1944 unpack=unpack, usemask=usemask, loose=loose,
1945 invalid_raise=invalid_raise, max_rows=max_rows, encoding=encoding,
1946 ndmin=ndmin,
1947 )
1949 _ensure_ndmin_ndarray_check_param(ndmin)
1951 if max_rows is not None:
1952 if skip_footer:
1953 raise ValueError(
1954 "The keywords 'skip_footer' and 'max_rows' can not be "
1955 "specified at the same time.")
1956 if max_rows < 1:
1957 raise ValueError("'max_rows' must be at least 1.")
1959 if usemask:
1960 from numpy.ma import MaskedArray, make_mask_descr
1961 # Check the input dictionary of converters
1962 user_converters = converters or {}
1963 if not isinstance(user_converters, dict):
1964 raise TypeError(
1965 "The input argument 'converter' should be a valid dictionary "
1966 "(got '%s' instead)" % type(user_converters))
1968 if encoding == 'bytes':
1969 encoding = None
1970 byte_converters = True
1971 else:
1972 byte_converters = False
1974 # Initialize the filehandle, the LineSplitter and the NameValidator
1975 if isinstance(fname, os.PathLike):
1976 fname = os.fspath(fname)
1977 if isinstance(fname, str):
1978 fid = np.lib._datasource.open(fname, 'rt', encoding=encoding)
1979 fid_ctx = contextlib.closing(fid)
1980 else:
1981 fid = fname
1982 fid_ctx = contextlib.nullcontext(fid)
1983 try:
1984 fhd = iter(fid)
1985 except TypeError as e:
1986 raise TypeError(
1987 "fname must be a string, a filehandle, a sequence of strings,\n"
1988 f"or an iterator of strings. Got {type(fname)} instead."
1989 ) from e
1990 with fid_ctx:
1991 split_line = LineSplitter(delimiter=delimiter, comments=comments,
1992 autostrip=autostrip, encoding=encoding)
1993 validate_names = NameValidator(excludelist=excludelist,
1994 deletechars=deletechars,
1995 case_sensitive=case_sensitive,
1996 replace_space=replace_space)
1998 # Skip the first `skip_header` rows
1999 try:
2000 for i in range(skip_header):
2001 next(fhd)
2003 # Keep on until we find the first valid values
2004 first_values = None
2006 while not first_values:
2007 first_line = _decode_line(next(fhd), encoding)
2008 if (names is True) and (comments is not None):
2009 if comments in first_line:
2010 first_line = (
2011 ''.join(first_line.split(comments)[1:]))
2012 first_values = split_line(first_line)
2013 except StopIteration:
2014 # return an empty array if the datafile is empty
2015 first_line = ''
2016 first_values = []
2017 warnings.warn(
2018 f'genfromtxt: Empty input file: "{fname}"', stacklevel=2
2019 )
2021 # Should we take the first values as names ?
2022 if names is True:
2023 fval = first_values[0].strip()
2024 if comments is not None:
2025 if fval in comments:
2026 del first_values[0]
2028 # Check the columns to use: make sure `usecols` is a list
2029 if usecols is not None:
2030 try:
2031 usecols = [_.strip() for _ in usecols.split(",")]
2032 except AttributeError:
2033 try:
2034 usecols = list(usecols)
2035 except TypeError:
2036 usecols = [usecols, ]
2037 nbcols = len(usecols or first_values)
2039 # Check the names and overwrite the dtype.names if needed
2040 if names is True:
2041 names = validate_names([str(_.strip()) for _ in first_values])
2042 first_line = ''
2043 elif _is_string_like(names):
2044 names = validate_names([_.strip() for _ in names.split(',')])
2045 elif names:
2046 names = validate_names(names)
2047 # Get the dtype
2048 if dtype is not None:
2049 dtype = easy_dtype(dtype, defaultfmt=defaultfmt, names=names,
2050 excludelist=excludelist,
2051 deletechars=deletechars,
2052 case_sensitive=case_sensitive,
2053 replace_space=replace_space)
2054 # Make sure the names is a list (for 2.5)
2055 if names is not None:
2056 names = list(names)
2058 if usecols:
2059 for (i, current) in enumerate(usecols):
2060 # if usecols is a list of names, convert to a list of indices
2061 if _is_string_like(current):
2062 usecols[i] = names.index(current)
2063 elif current < 0:
2064 usecols[i] = current + len(first_values)
2065 # If the dtype is not None, make sure we update it
2066 if (dtype is not None) and (len(dtype) > nbcols):
2067 descr = dtype.descr
2068 dtype = np.dtype([descr[_] for _ in usecols])
2069 names = list(dtype.names)
2070 # If `names` is not None, update the names
2071 elif (names is not None) and (len(names) > nbcols):
2072 names = [names[_] for _ in usecols]
2073 elif (names is not None) and (dtype is not None):
2074 names = list(dtype.names)
2076 # Process the missing values ...............................
2077 # Rename missing_values for convenience
2078 user_missing_values = missing_values or ()
2079 if isinstance(user_missing_values, bytes):
2080 user_missing_values = user_missing_values.decode('latin1')
2082 # Define the list of missing_values (one column: one list)
2083 missing_values = [[''] for _ in range(nbcols)]
2085 # We have a dictionary: process it field by field
2086 if isinstance(user_missing_values, dict):
2087 # Loop on the items
2088 for (key, val) in user_missing_values.items():
2089 # Is the key a string ?
2090 if _is_string_like(key):
2091 try:
2092 # Transform it into an integer
2093 key = names.index(key)
2094 except ValueError:
2095 # We couldn't find it: the name must have been dropped
2096 continue
2097 # Redefine the key as needed if it's a column number
2098 if usecols:
2099 try:
2100 key = usecols.index(key)
2101 except ValueError:
2102 pass
2103 # Transform the value as a list of string
2104 if isinstance(val, (list, tuple)):
2105 val = [str(_) for _ in val]
2106 else:
2107 val = [str(val), ]
2108 # Add the value(s) to the current list of missing
2109 if key is None:
2110 # None acts as default
2111 for miss in missing_values:
2112 miss.extend(val)
2113 else:
2114 missing_values[key].extend(val)
2115 # We have a sequence : each item matches a column
2116 elif isinstance(user_missing_values, (list, tuple)):
2117 for (value, entry) in zip(user_missing_values, missing_values):
2118 value = str(value)
2119 if value not in entry:
2120 entry.append(value)
2121 # We have a string : apply it to all entries
2122 elif isinstance(user_missing_values, str):
2123 user_value = user_missing_values.split(",")
2124 for entry in missing_values:
2125 entry.extend(user_value)
2126 # We have something else: apply it to all entries
2127 else:
2128 for entry in missing_values:
2129 entry.extend([str(user_missing_values)])
2131 # Process the filling_values ...............................
2132 # Rename the input for convenience
2133 user_filling_values = filling_values
2134 if user_filling_values is None:
2135 user_filling_values = []
2136 # Define the default
2137 filling_values = [None] * nbcols
2138 # We have a dictionary : update each entry individually
2139 if isinstance(user_filling_values, dict):
2140 for (key, val) in user_filling_values.items():
2141 if _is_string_like(key):
2142 try:
2143 # Transform it into an integer
2144 key = names.index(key)
2145 except ValueError:
2146 # We couldn't find it: the name must have been dropped
2147 continue
2148 # Redefine the key if it's a column number
2149 # and usecols is defined
2150 if usecols:
2151 try:
2152 key = usecols.index(key)
2153 except ValueError:
2154 pass
2155 # Add the value to the list
2156 filling_values[key] = val
2157 # We have a sequence : update on a one-to-one basis
2158 elif isinstance(user_filling_values, (list, tuple)):
2159 n = len(user_filling_values)
2160 if (n <= nbcols):
2161 filling_values[:n] = user_filling_values
2162 else:
2163 filling_values = user_filling_values[:nbcols]
2164 # We have something else : use it for all entries
2165 else:
2166 filling_values = [user_filling_values] * nbcols
2168 # Initialize the converters ................................
2169 if dtype is None:
2170 # Note: we can't use a [...]*nbcols, as we would have 3 times
2171 # the same converter, instead of 3 different converters.
2172 converters = [
2173 StringConverter(None, missing_values=miss, default=fill)
2174 for (miss, fill) in zip(missing_values, filling_values)
2175 ]
2176 else:
2177 dtype_flat = flatten_dtype(dtype, flatten_base=True)
2178 # Initialize the converters
2179 if len(dtype_flat) > 1:
2180 # Flexible type : get a converter from each dtype
2181 zipit = zip(dtype_flat, missing_values, filling_values)
2182 converters = [StringConverter(dt,
2183 locked=True,
2184 missing_values=miss,
2185 default=fill)
2186 for (dt, miss, fill) in zipit]
2187 else:
2188 # Set to a default converter (but w/ different missing values)
2189 zipit = zip(missing_values, filling_values)
2190 converters = [StringConverter(dtype,
2191 locked=True,
2192 missing_values=miss,
2193 default=fill)
2194 for (miss, fill) in zipit]
2195 # Update the converters to use the user-defined ones
2196 uc_update = []
2197 for (j, conv) in user_converters.items():
2198 # If the converter is specified by column names,
2199 # use the index instead
2200 if _is_string_like(j):
2201 try:
2202 j = names.index(j)
2203 i = j
2204 except ValueError:
2205 continue
2206 elif usecols:
2207 try:
2208 i = usecols.index(j)
2209 except ValueError:
2210 # Unused converter specified
2211 continue
2212 else:
2213 i = j
2214 # Find the value to test - first_line is not filtered by usecols:
2215 if len(first_line):
2216 testing_value = first_values[j]
2217 else:
2218 testing_value = None
2219 if conv is bytes:
2220 user_conv = asbytes
2221 elif byte_converters:
2222 # Converters may use decode to workaround numpy's old
2223 # behavior, so encode the string again before passing
2224 # to the user converter.
2225 def tobytes_first(x, conv):
2226 if type(x) is bytes:
2227 return conv(x)
2228 return conv(x.encode("latin1"))
2229 user_conv = functools.partial(tobytes_first, conv=conv)
2230 else:
2231 user_conv = conv
2232 converters[i].update(user_conv, locked=True,
2233 testing_value=testing_value,
2234 default=filling_values[i],
2235 missing_values=missing_values[i],)
2236 uc_update.append((i, user_conv))
2237 # Make sure we have the corrected keys in user_converters...
2238 user_converters.update(uc_update)
2240 # Fixme: possible error as following variable never used.
2241 # miss_chars = [_.missing_values for _ in converters]
2243 # Initialize the output lists ...
2244 # ... rows
2245 rows = []
2246 append_to_rows = rows.append
2247 # ... masks
2248 if usemask:
2249 masks = []
2250 append_to_masks = masks.append
2251 # ... invalid
2252 invalid = []
2253 append_to_invalid = invalid.append
2255 # Parse each line
2256 for (i, line) in enumerate(itertools.chain([first_line, ], fhd)):
2257 values = split_line(line)
2258 nbvalues = len(values)
2259 # Skip an empty line
2260 if nbvalues == 0:
2261 continue
2262 if usecols:
2263 # Select only the columns we need
2264 try:
2265 values = [values[_] for _ in usecols]
2266 except IndexError:
2267 append_to_invalid((i + skip_header + 1, nbvalues))
2268 continue
2269 elif nbvalues != nbcols:
2270 append_to_invalid((i + skip_header + 1, nbvalues))
2271 continue
2272 # Store the values
2273 append_to_rows(tuple(values))
2274 if usemask:
2275 append_to_masks(tuple(v.strip() in m
2276 for (v, m) in zip(values,
2277 missing_values)))
2278 if len(rows) == max_rows:
2279 break
2281 # Upgrade the converters (if needed)
2282 if dtype is None:
2283 for (i, converter) in enumerate(converters):
2284 current_column = [itemgetter(i)(_m) for _m in rows]
2285 try:
2286 converter.iterupgrade(current_column)
2287 except ConverterLockError:
2288 errmsg = f"Converter #{i} is locked and cannot be upgraded: "
2289 current_column = map(itemgetter(i), rows)
2290 for (j, value) in enumerate(current_column):
2291 try:
2292 converter.upgrade(value)
2293 except (ConverterError, ValueError):
2294 line_number = j + 1 + skip_header
2295 errmsg += f"(occurred line #{line_number} for value '{value}')"
2296 raise ConverterError(errmsg)
2298 # Check that we don't have invalid values
2299 nbinvalid = len(invalid)
2300 if nbinvalid > 0:
2301 nbrows = len(rows) + nbinvalid - skip_footer
2302 # Construct the error message
2303 template = f" Line #%i (got %i columns instead of {nbcols})"
2304 if skip_footer > 0:
2305 nbinvalid_skipped = len([_ for _ in invalid
2306 if _[0] > nbrows + skip_header])
2307 invalid = invalid[:nbinvalid - nbinvalid_skipped]
2308 skip_footer -= nbinvalid_skipped
2309#
2310# nbrows -= skip_footer
2311# errmsg = [template % (i, nb)
2312# for (i, nb) in invalid if i < nbrows]
2313# else:
2314 errmsg = [template % (i, nb)
2315 for (i, nb) in invalid]
2316 if len(errmsg):
2317 errmsg.insert(0, "Some errors were detected !")
2318 errmsg = "\n".join(errmsg)
2319 # Raise an exception ?
2320 if invalid_raise:
2321 raise ValueError(errmsg)
2322 # Issue a warning ?
2323 else:
2324 warnings.warn(errmsg, ConversionWarning, stacklevel=2)
2326 # Strip the last skip_footer data
2327 if skip_footer > 0:
2328 rows = rows[:-skip_footer]
2329 if usemask:
2330 masks = masks[:-skip_footer]
2332 # Convert each value according to the converter:
2333 # We want to modify the list in place to avoid creating a new one...
2334 if loose:
2335 rows = list(
2336 zip(*[[conv._loose_call(_r) for _r in map(itemgetter(i), rows)]
2337 for (i, conv) in enumerate(converters)]))
2338 else:
2339 rows = list(
2340 zip(*[[conv._strict_call(_r) for _r in map(itemgetter(i), rows)]
2341 for (i, conv) in enumerate(converters)]))
2343 # Reset the dtype
2344 data = rows
2345 if dtype is None:
2346 # Get the dtypes from the types of the converters
2347 column_types = [conv.type for conv in converters]
2348 # Find the columns with strings...
2349 strcolidx = [i for (i, v) in enumerate(column_types)
2350 if v == np.str_]
2352 if byte_converters and strcolidx:
2353 # convert strings back to bytes for backward compatibility
2354 warnings.warn(
2355 "Reading unicode strings without specifying the encoding "
2356 "argument is deprecated. Set the encoding, use None for the "
2357 "system default.",
2358 np.exceptions.VisibleDeprecationWarning, stacklevel=2)
2360 def encode_unicode_cols(row_tup):
2361 row = list(row_tup)
2362 for i in strcolidx:
2363 row[i] = row[i].encode('latin1')
2364 return tuple(row)
2366 try:
2367 data = [encode_unicode_cols(r) for r in data]
2368 except UnicodeEncodeError:
2369 pass
2370 else:
2371 for i in strcolidx:
2372 column_types[i] = np.bytes_
2374 # Update string types to be the right length
2375 sized_column_types = column_types.copy()
2376 for i, col_type in enumerate(column_types):
2377 if np.issubdtype(col_type, np.character):
2378 n_chars = max(len(row[i]) for row in data)
2379 sized_column_types[i] = (col_type, n_chars)
2381 if names is None:
2382 # If the dtype is uniform (before sizing strings)
2383 base = {
2384 c_type
2385 for c, c_type in zip(converters, column_types)
2386 if c._checked}
2387 if len(base) == 1:
2388 uniform_type, = base
2389 (ddtype, mdtype) = (uniform_type, bool)
2390 else:
2391 ddtype = [(defaultfmt % i, dt)
2392 for (i, dt) in enumerate(sized_column_types)]
2393 if usemask:
2394 mdtype = [(defaultfmt % i, bool)
2395 for (i, dt) in enumerate(sized_column_types)]
2396 else:
2397 ddtype = list(zip(names, sized_column_types))
2398 mdtype = list(zip(names, [bool] * len(sized_column_types)))
2399 output = np.array(data, dtype=ddtype)
2400 if usemask:
2401 outputmask = np.array(masks, dtype=mdtype)
2402 else:
2403 # Overwrite the initial dtype names if needed
2404 if names and dtype.names is not None:
2405 dtype.names = names
2406 # Case 1. We have a structured type
2407 if len(dtype_flat) > 1:
2408 # Nested dtype, eg [('a', int), ('b', [('b0', int), ('b1', 'f4')])]
2409 # First, create the array using a flattened dtype:
2410 # [('a', int), ('b1', int), ('b2', float)]
2411 # Then, view the array using the specified dtype.
2412 if 'O' in (_.char for _ in dtype_flat):
2413 if has_nested_fields(dtype):
2414 raise NotImplementedError(
2415 "Nested fields involving objects are not supported...")
2416 else:
2417 output = np.array(data, dtype=dtype)
2418 else:
2419 rows = np.array(data, dtype=[('', _) for _ in dtype_flat])
2420 output = rows.view(dtype)
2421 # Now, process the rowmasks the same way
2422 if usemask:
2423 rowmasks = np.array(
2424 masks, dtype=np.dtype([('', bool) for t in dtype_flat]))
2425 # Construct the new dtype
2426 mdtype = make_mask_descr(dtype)
2427 outputmask = rowmasks.view(mdtype)
2428 # Case #2. We have a basic dtype
2429 else:
2430 # We used some user-defined converters
2431 if user_converters:
2432 ishomogeneous = True
2433 descr = []
2434 for i, ttype in enumerate([conv.type for conv in converters]):
2435 # Keep the dtype of the current converter
2436 if i in user_converters:
2437 ishomogeneous &= (ttype == dtype.type)
2438 if np.issubdtype(ttype, np.character):
2439 ttype = (ttype, max(len(row[i]) for row in data))
2440 descr.append(('', ttype))
2441 else:
2442 descr.append(('', dtype))
2443 # So we changed the dtype ?
2444 if not ishomogeneous:
2445 # We have more than one field
2446 if len(descr) > 1:
2447 dtype = np.dtype(descr)
2448 # We have only one field: drop the name if not needed.
2449 else:
2450 dtype = np.dtype(ttype)
2451 #
2452 output = np.array(data, dtype)
2453 if usemask:
2454 if dtype.names is not None:
2455 mdtype = [(_, bool) for _ in dtype.names]
2456 else:
2457 mdtype = bool
2458 outputmask = np.array(masks, dtype=mdtype)
2459 # Try to take care of the missing data we missed
2460 names = output.dtype.names
2461 if usemask and names:
2462 for (name, conv) in zip(names, converters):
2463 missing_values = [conv(_) for _ in conv.missing_values
2464 if _ != '']
2465 for mval in missing_values:
2466 outputmask[name] |= (output[name] == mval)
2467 # Construct the final array
2468 if usemask:
2469 output = output.view(MaskedArray)
2470 output._mask = outputmask
2472 output = _ensure_ndmin_ndarray(output, ndmin=ndmin)
2474 if unpack:
2475 if names is None:
2476 return output.T
2477 elif len(names) == 1:
2478 # squeeze single-name dtypes too
2479 return output[names[0]]
2480 else:
2481 # For structured arrays with multiple fields,
2482 # return an array for each field.
2483 return [output[field] for field in names]
2484 return output
2487_genfromtxt_with_like = array_function_dispatch()(genfromtxt)
2490def recfromtxt(fname, **kwargs):
2491 """
2492 Load ASCII data from a file and return it in a record array.
2494 If ``usemask=False`` a standard `recarray` is returned,
2495 if ``usemask=True`` a MaskedRecords array is returned.
2497 .. deprecated:: 2.0
2498 Use `numpy.genfromtxt` instead.
2500 Parameters
2501 ----------
2502 fname, kwargs : For a description of input parameters, see `genfromtxt`.
2504 See Also
2505 --------
2506 numpy.genfromtxt : generic function
2508 Notes
2509 -----
2510 By default, `dtype` is None, which means that the data-type of the output
2511 array will be determined from the data.
2513 """
2515 # Deprecated in NumPy 2.0, 2023-07-11
2516 warnings.warn(
2517 "`recfromtxt` is deprecated, "
2518 "use `numpy.genfromtxt` instead."
2519 "(deprecated in NumPy 2.0)",
2520 DeprecationWarning,
2521 stacklevel=2
2522 )
2524 kwargs.setdefault("dtype", None)
2525 usemask = kwargs.get('usemask', False)
2526 output = genfromtxt(fname, **kwargs)
2527 if usemask:
2528 from numpy.ma.mrecords import MaskedRecords
2529 output = output.view(MaskedRecords)
2530 else:
2531 output = output.view(np.recarray)
2532 return output
2535def recfromcsv(fname, **kwargs):
2536 """
2537 Load ASCII data stored in a comma-separated file.
2539 The returned array is a record array (if ``usemask=False``, see
2540 `recarray`) or a masked record array (if ``usemask=True``,
2541 see `ma.mrecords.MaskedRecords`).
2543 .. deprecated:: 2.0
2544 Use `numpy.genfromtxt` with comma as `delimiter` instead.
2546 Parameters
2547 ----------
2548 fname, kwargs : For a description of input parameters, see `genfromtxt`.
2550 See Also
2551 --------
2552 numpy.genfromtxt : generic function to load ASCII data.
2554 Notes
2555 -----
2556 By default, `dtype` is None, which means that the data-type of the output
2557 array will be determined from the data.
2559 """
2561 # Deprecated in NumPy 2.0, 2023-07-11
2562 warnings.warn(
2563 "`recfromcsv` is deprecated, "
2564 "use `numpy.genfromtxt` with comma as `delimiter` instead. "
2565 "(deprecated in NumPy 2.0)",
2566 DeprecationWarning,
2567 stacklevel=2
2568 )
2570 # Set default kwargs for genfromtxt as relevant to csv import.
2571 kwargs.setdefault("case_sensitive", "lower")
2572 kwargs.setdefault("names", True)
2573 kwargs.setdefault("delimiter", ",")
2574 kwargs.setdefault("dtype", None)
2575 output = genfromtxt(fname, **kwargs)
2577 usemask = kwargs.get("usemask", False)
2578 if usemask:
2579 from numpy.ma.mrecords import MaskedRecords
2580 output = output.view(MaskedRecords)
2581 else:
2582 output = output.view(np.recarray)
2583 return output