Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Module contains tools for processing files into DataFrames or other objects
4GH#48849 provides a convenient way of deprecating keyword arguments
5"""
7from __future__ import annotations
9from collections import (
10 abc,
11 defaultdict,
12)
13import csv
14import sys
15from typing import (
16 IO,
17 TYPE_CHECKING,
18 Any,
19 Generic,
20 Literal,
21 Self,
22 TypedDict,
23 Unpack,
24 cast,
25 overload,
26)
27import warnings
29import numpy as np
31from pandas._libs import lib
32from pandas._libs.parsers import STR_NA_VALUES
33from pandas.errors import (
34 AbstractMethodError,
35 ParserWarning,
36)
37from pandas.util._decorators import (
38 set_module,
39)
40from pandas.util._exceptions import find_stack_level
41from pandas.util._validators import check_dtype_backend
43from pandas.core.dtypes.common import (
44 is_file_like,
45 is_float,
46 is_integer,
47 is_list_like,
48 pandas_dtype,
49)
51from pandas import Series
52from pandas.core.frame import DataFrame
53from pandas.core.indexes.api import RangeIndex
55from pandas.io.common import (
56 IOHandles,
57 get_handle,
58 stringify_path,
59 validate_header_arg,
60)
61from pandas.io.parsers.arrow_parser_wrapper import ArrowParserWrapper
62from pandas.io.parsers.base_parser import (
63 ParserBase,
64 is_index_col,
65 parser_defaults,
66)
67from pandas.io.parsers.c_parser_wrapper import CParserWrapper
68from pandas.io.parsers.python_parser import (
69 FixedWidthFieldParser,
70 PythonParser,
71)
73if TYPE_CHECKING:
74 from collections.abc import (
75 Callable,
76 Hashable,
77 Iterable,
78 Mapping,
79 Sequence,
80 )
81 from types import TracebackType
83 from pandas._typing import (
84 CompressionOptions,
85 CSVEngine,
86 DtypeArg,
87 DtypeBackend,
88 FilePath,
89 HashableT,
90 IndexLabel,
91 ReadCsvBuffer,
92 StorageOptions,
93 UsecolsArgType,
94 )
96 class _read_shared(TypedDict, Generic[HashableT], total=False):
97 # annotations shared between read_csv/fwf/table's overloads
98 # NOTE: Keep in sync with the annotations of the implementation
99 sep: str | None | lib.NoDefault
100 delimiter: str | None | lib.NoDefault
101 header: int | Sequence[int] | None | Literal["infer"]
102 names: Sequence[Hashable] | None | lib.NoDefault
103 index_col: IndexLabel | Literal[False] | None
104 usecols: UsecolsArgType
105 dtype: DtypeArg | None
106 engine: CSVEngine | None
107 converters: Mapping[HashableT, Callable] | None
108 true_values: list | None
109 false_values: list | None
110 skipinitialspace: bool
111 skiprows: list[int] | int | Callable[[Hashable], bool] | None
112 skipfooter: int
113 nrows: int | None
114 na_values: (
115 Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None
116 )
117 keep_default_na: bool
118 na_filter: bool
119 skip_blank_lines: bool
120 parse_dates: bool | Sequence[Hashable] | None
121 date_format: str | dict[Hashable, str] | None
122 dayfirst: bool
123 cache_dates: bool
124 compression: CompressionOptions
125 thousands: str | None
126 decimal: str
127 lineterminator: str | None
128 quotechar: str
129 quoting: int
130 doublequote: bool
131 escapechar: str | None
132 comment: str | None
133 encoding: str | None
134 encoding_errors: str | None
135 dialect: str | csv.Dialect | None
136 on_bad_lines: str
137 low_memory: bool
138 memory_map: bool
139 float_precision: Literal["high", "legacy", "round_trip"] | None
140 storage_options: StorageOptions | None
141 dtype_backend: DtypeBackend | lib.NoDefault
143else:
144 _read_shared = dict
147class _C_Parser_Defaults(TypedDict):
148 na_filter: Literal[True]
149 low_memory: Literal[True]
150 memory_map: Literal[False]
151 float_precision: None
154_c_parser_defaults: _C_Parser_Defaults = {
155 "na_filter": True,
156 "low_memory": True,
157 "memory_map": False,
158 "float_precision": None,
159}
162class _Fwf_Defaults(TypedDict):
163 colspecs: Literal["infer"]
164 infer_nrows: Literal[100]
165 widths: None
168_fwf_defaults: _Fwf_Defaults = {"colspecs": "infer", "infer_nrows": 100, "widths": None}
169_c_unsupported = {"skipfooter"}
170_python_unsupported = {"low_memory", "float_precision"}
171_pyarrow_unsupported = {
172 "skipfooter",
173 "float_precision",
174 "chunksize",
175 "comment",
176 "nrows",
177 "thousands",
178 "memory_map",
179 "dialect",
180 "quoting",
181 "lineterminator",
182 "converters",
183 "iterator",
184 "dayfirst",
185 "skipinitialspace",
186 "low_memory",
187}
190@overload
191def validate_integer(name: str, val: None, min_val: int = ...) -> None: ...
194@overload
195def validate_integer(name: str, val: float, min_val: int = ...) -> int: ...
198@overload
199def validate_integer(name: str, val: int | None, min_val: int = ...) -> int | None: ...
202def validate_integer(
203 name: str, val: int | float | None, min_val: int = 0
204) -> int | None:
205 """
206 Checks whether the 'name' parameter for parsing is either
207 an integer OR float that can SAFELY be cast to an integer
208 without losing accuracy. Raises a ValueError if that is
209 not the case.
211 Parameters
212 ----------
213 name : str
214 Parameter name (used for error reporting)
215 val : int or float
216 The value to check
217 min_val : int
218 Minimum allowed value (val < min_val will result in a ValueError)
219 """
220 if val is None:
221 return val
223 msg = f"'{name:s}' must be an integer >={min_val:d}"
224 if is_float(val):
225 if int(val) != val:
226 raise ValueError(msg)
227 val = int(val)
228 elif not (is_integer(val) and val >= min_val):
229 raise ValueError(msg)
231 return int(val)
234def _validate_names(names: Sequence[Hashable] | None) -> None:
235 """
236 Raise ValueError if the `names` parameter contains duplicates or has an
237 invalid data type.
239 Parameters
240 ----------
241 names : array-like or None
242 An array containing a list of the names used for the output DataFrame.
244 Raises
245 ------
246 ValueError
247 If names are not unique or are not ordered (e.g. set).
248 """
249 if names is not None:
250 if len(names) != len(set(names)):
251 raise ValueError("Duplicate names are not allowed.")
252 if not (
253 is_list_like(names, allow_sets=False) or isinstance(names, abc.KeysView)
254 ):
255 raise ValueError("Names should be an ordered collection.")
258def _read(
259 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], kwds
260) -> DataFrame | TextFileReader:
261 """Generic reader of line files."""
262 # if we pass a date_format and parse_dates=False, we should not parse the
263 # dates GH#44366
264 if kwds.get("parse_dates", None) is None:
265 if kwds.get("date_format", None) is None:
266 kwds["parse_dates"] = False
267 else:
268 kwds["parse_dates"] = True
270 # Extract some of the arguments (pass chunksize on).
271 iterator = kwds.get("iterator", False)
272 chunksize = kwds.get("chunksize", None)
274 # Check type of encoding_errors
275 errors = kwds.get("encoding_errors", "strict")
276 if not isinstance(errors, str):
277 raise ValueError(
278 f"encoding_errors must be a string, got {type(errors).__name__}"
279 )
281 if kwds.get("engine") == "pyarrow":
282 if iterator:
283 raise ValueError(
284 "The 'iterator' option is not supported with the 'pyarrow' engine"
285 )
287 if chunksize is not None:
288 raise ValueError(
289 "The 'chunksize' option is not supported with the 'pyarrow' engine"
290 )
291 else:
292 chunksize = validate_integer("chunksize", chunksize, 1)
294 nrows = kwds.get("nrows", None)
296 # Check for duplicates in names.
297 _validate_names(kwds.get("names", None))
299 # Create the parser.
300 parser = TextFileReader(filepath_or_buffer, **kwds)
302 if chunksize or iterator:
303 return parser
305 with parser:
306 return parser.read(nrows)
309@overload
310def read_csv(
311 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
312 *,
313 iterator: Literal[True],
314 chunksize: int | None = ...,
315 **kwds: Unpack[_read_shared[HashableT]],
316) -> TextFileReader: ...
319@overload
320def read_csv(
321 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
322 *,
323 iterator: bool = ...,
324 chunksize: int,
325 **kwds: Unpack[_read_shared[HashableT]],
326) -> TextFileReader: ...
329@overload
330def read_csv(
331 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
332 *,
333 iterator: Literal[False] = ...,
334 chunksize: None = ...,
335 **kwds: Unpack[_read_shared[HashableT]],
336) -> DataFrame: ...
339@overload
340def read_csv(
341 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
342 *,
343 iterator: bool = ...,
344 chunksize: int | None = ...,
345 **kwds: Unpack[_read_shared[HashableT]],
346) -> DataFrame | TextFileReader: ...
349@set_module("pandas")
350def read_csv(
351 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
352 *,
353 sep: str | None | lib.NoDefault = lib.no_default,
354 delimiter: str | None | lib.NoDefault = None,
355 # Column and Index Locations and Names
356 header: int | Sequence[int] | None | Literal["infer"] = "infer",
357 names: Sequence[Hashable] | None | lib.NoDefault = lib.no_default,
358 index_col: IndexLabel | Literal[False] | None = None,
359 usecols: UsecolsArgType = None,
360 # General Parsing Configuration
361 dtype: DtypeArg | None = None,
362 engine: CSVEngine | None = None,
363 converters: Mapping[HashableT, Callable] | None = None,
364 true_values: list | None = None,
365 false_values: list | None = None,
366 skipinitialspace: bool = False,
367 skiprows: list[int] | int | Callable[[Hashable], bool] | None = None,
368 skipfooter: int = 0,
369 nrows: int | None = None,
370 # NA and Missing Data Handling
371 na_values: (
372 Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None
373 ) = None,
374 keep_default_na: bool = True,
375 na_filter: bool = True,
376 skip_blank_lines: bool = True,
377 # Datetime Handling
378 parse_dates: bool | Sequence[Hashable] | None = None,
379 date_format: str | dict[Hashable, str] | None = None,
380 dayfirst: bool = False,
381 cache_dates: bool = True,
382 # Iteration
383 iterator: bool = False,
384 chunksize: int | None = None,
385 # Quoting, Compression, and File Format
386 compression: CompressionOptions = "infer",
387 thousands: str | None = None,
388 decimal: str = ".",
389 lineterminator: str | None = None,
390 quotechar: str = '"',
391 quoting: int = csv.QUOTE_MINIMAL,
392 doublequote: bool = True,
393 escapechar: str | None = None,
394 comment: str | None = None,
395 encoding: str | None = None,
396 encoding_errors: str | None = "strict",
397 dialect: str | csv.Dialect | None = None,
398 # Error Handling
399 on_bad_lines: str = "error",
400 # Internal
401 low_memory: bool = _c_parser_defaults["low_memory"],
402 memory_map: bool = False,
403 float_precision: Literal["high", "legacy", "round_trip"] | None = None,
404 storage_options: StorageOptions | None = None,
405 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
406) -> DataFrame | TextFileReader:
407 """
408 Read a comma-separated values (csv) file into DataFrame.
410 Also supports optionally iterating or breaking of the file
411 into chunks.
413 Additional help can be found in the online docs for
414 `IO Tools <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html>`_.
416 Parameters
417 ----------
418 filepath_or_buffer : str, path object or file-like object
419 Any valid string path is acceptable. The string could be a URL. Valid
420 URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is
421 expected. A local file could be: file://localhost/path/to/table.csv.
423 If you want to pass in a path object, pandas accepts any ``os.PathLike``.
425 By file-like object, we refer to objects with a ``read()`` method, such as
426 a file handle (e.g. via builtin ``open`` function) or ``StringIO``.
427 sep : str, default ','
428 Character or regex pattern to treat as the delimiter. If ``sep=None``, the
429 C engine cannot automatically detect
430 the separator, but the Python parsing engine can, meaning the latter will
431 be used and automatically detect the separator from only the first valid
432 row of the file by Python's builtin sniffer tool, ``csv.Sniffer``.
433 In addition, separators longer than 1 character and different from
434 ``'\\s+'`` will be interpreted as regular expressions and will also force
435 the use of the Python parsing engine. Note that regex delimiters are prone
436 to ignoring quoted data. Regex example: ``'\\r\\t'``.
437 delimiter : str, optional
438 Alias for ``sep``.
439 header : int, Sequence of int, 'infer' or None, default 'infer'
440 Row number(s) containing column labels and marking the start of the
441 data (zero-indexed). Default behavior is to infer the column names:
442 if no ``names``
443 are passed the behavior is identical to ``header=0`` and column
444 names are inferred from the first line of the file, if column
445 names are passed explicitly to ``names`` then the behavior is identical to
446 ``header=None``. Explicitly pass ``header=0`` to be able to
447 replace existing names. The header can be a list of integers that
448 specify row locations for a :class:`~pandas.MultiIndex` on the columns
449 e.g. ``[0, 1, 3]``. Intervening rows that are not specified will be
450 skipped (e.g. 2 in this example is skipped). Note that this
451 parameter ignores commented lines and empty lines if
452 ``skip_blank_lines=True``, so ``header=0`` denotes the first line of
453 data rather than the first line of the file.
455 When inferred from the file contents, headers are kept distinct from
456 each other by renaming duplicate names with a numeric suffix of the form
457 ``".{count}"`` starting from 1, e.g. ``"foo"`` and ``"foo.1"``.
458 Empty headers are named ``"Unnamed: {i}"`` or ``
459 "Unnamed: {i}_level_{level}"``
460 in the case of MultiIndex columns.
461 names : Sequence of Hashable, optional
462 Sequence of column labels to apply. If the file contains a header row,
463 then you should explicitly pass ``header=0`` to override the column names.
464 Duplicates in this list are not allowed.
465 index_col : Hashable, Sequence of Hashable or False, optional
466 Column(s) to use as row label(s), denoted either by column labels or column
467 indices. If a sequence of labels or indices is given,
468 :class:`~pandas.MultiIndex`
469 will be formed for the row labels.
471 Note: ``index_col=False`` can be used to force pandas to *not* use the first
472 column as the index, e.g., when you have a malformed file with delimiters at
473 the end of each line.
474 usecols : Sequence of Hashable or Callable, optional
475 Subset of columns to select, denoted either
476 by column labels or column indices.
477 If list-like, all elements must either
478 be positional (i.e. integer indices into the document columns) or strings
479 that correspond to column names provided either by the user in ``names`` or
480 inferred from the document header row(s).
481 If ``names`` are given, the document
482 header row(s) are not taken into account. For example, a valid list-like
483 ``usecols`` parameter would be ``[0, 1, 2]`` or ``['foo', 'bar', 'baz']``.
484 Element order is ignored, so ``usecols=[0, 1]`` is the same as ``[1, 0]``.
485 To instantiate a :class:`~pandas.DataFrame` from ``data`` with element order
486 preserved use ``pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]``
487 for columns in ``['foo', 'bar']`` order or
488 ``pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]``
489 for ``['bar', 'foo']`` order.
491 If callable, the callable function will be evaluated against the column
492 names, returning names where the callable function evaluates to ``True``. An
493 example of a valid callable argument would be ``lambda x: x.upper() in
494 ['AAA', 'BBB', 'DDD']``. Using this parameter results in much faster
495 parsing time and lower memory usage.
496 dtype : dtype or dict of {Hashable : dtype}, optional
497 Data type(s) to apply to either the whole dataset or individual columns.
498 E.g., ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}``
499 Use ``str`` or ``object`` together with suitable ``na_values`` settings
500 to preserve and not interpret ``dtype``.
501 If ``converters`` are specified, they will be applied INSTEAD
502 of ``dtype`` conversion. Specify a ``defaultdict`` as input where
503 the default determines the ``dtype``
504 of the columns which are not explicitly
505 listed.
506 engine : {'c', 'python', 'pyarrow'}, optional
507 Parser engine to use. The C and pyarrow engines are faster,
508 while the python engine
509 is currently more feature-complete. Multithreading
510 is currently only supported by
511 the pyarrow engine. Some features of the "pyarrow" engine
512 are unsupported or may not work correctly.
513 converters : dict of {Hashable : Callable}, optional
514 Functions for converting values in specified columns. Keys can either
515 be column labels or column indices.
516 true_values : list, optional
517 Values to consider as ``True`` in addition
518 to case-insensitive variants of 'True'.
519 false_values : list, optional
520 Values to consider as ``False`` in addition to case-insensitive
521 variants of 'False'.
522 skipinitialspace : bool, default False
523 Skip spaces after delimiter.
524 skiprows : int, list of int or Callable, optional
525 Line numbers to skip (0-indexed) or number of lines to skip (``int``)
526 at the start of the file.
528 If callable, the callable function will be evaluated against the row
529 indices, returning ``True`` if the row should be skipped and ``False``
530 otherwise.
531 An example of a valid callable argument would be ``lambda x: x in [0, 2]``.
532 skipfooter : int, default 0
533 Number of lines at bottom of file to skip (Unsupported with ``engine='c'``).
534 nrows : int, optional
535 Number of rows of file to read. Useful for reading pieces of large files.
536 Refers to the number of data rows in the returned DataFrame, excluding:
538 * The header row containing column names.
539 * Rows before the header row, if ``header=1`` or larger.
541 Example usage:
543 * To read the first 999,999 (non-header) rows:
544 ``read_csv(..., nrows=999999)``
546 * To read rows 1,000,000 through 1,999,999:
547 ``read_csv(..., skiprows=1000000, nrows=999999)``
549 na_values : Hashable, Iterable of Hashable or dict of {Hashable : Iterable},
550 optional
551 Additional strings to recognize as ``NA``/``NaN``. If ``dict``
552 passed, specific
553 per-column ``NA`` values. By default the following values
554 are interpreted as
555 ``NaN``: empty string, "NaN", "N/A", "NULL", and other common
556 representations of missing data.
557 keep_default_na : bool, default True
558 Whether or not to include the default ``NaN`` values when parsing the data.
559 Depending on whether ``na_values`` is passed in, the behavior is as follows:
561 * If ``keep_default_na`` is ``True``, and ``na_values``
562 are specified, ``na_values``
563 is appended to the default ``NaN`` values used for parsing.
564 * If ``keep_default_na`` is ``True``, and ``na_values`` are not specified, only
565 the default ``NaN`` values are used for parsing.
566 * If ``keep_default_na`` is ``False``, and ``na_values`` are specified, only
567 the ``NaN`` values specified ``na_values`` are used for parsing.
568 * If ``keep_default_na`` is ``False``, and ``na_values`` are not specified, no
569 strings will be parsed as ``NaN``.
571 Note that if ``na_filter`` is passed in as ``False``,
572 the ``keep_default_na`` and
573 ``na_values`` parameters will be ignored.
574 na_filter : bool, default True
575 Detect missing value markers (empty strings and the value of ``na_values``). In
576 data without any ``NA`` values, passing ``na_filter=False`` can improve the
577 performance of reading a large file.
578 skip_blank_lines : bool, default True
579 If ``True``, skip over blank lines rather than interpreting as ``NaN`` values.
580 parse_dates : bool, None, list of Hashable, default None
581 The behavior is as follows:
583 * ``bool``. If ``True`` -> try parsing the index.
584 * ``None``. Behaves like ``True`` if ``date_format`` is specified.
585 * ``list`` of ``int`` or names.
586 e.g. If ``[1, 2, 3]`` -> try parsing columns 1, 2, 3
587 each as a separate date column.
589 If a column or index cannot be represented as an array of ``datetime``,
590 say because of an unparsable value or a mixture of timezones, the column
591 or index will be returned unaltered as an ``object`` data type. For
592 non-standard ``datetime`` parsing, use :func:`~pandas.to_datetime` after
593 :func:`~pandas.read_csv`.
595 Note: A fast-path exists for iso8601-formatted dates.
596 date_format : str or dict of column -> format, optional
597 Format to use for parsing dates and/or times when
598 used in conjunction with ``parse_dates``.
599 The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See
600 `strftime documentation
601 <https://docs.python.org/3/library/datetime.html
602 #strftime-and-strptime-behavior>`_ for more information on choices, though
603 note that :const:`"%f"`` will parse all the way up to nanoseconds.
604 You can also pass:
606 - "ISO8601", to parse any `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_
607 time string (not necessarily in exactly the same format);
608 - "mixed", to infer the format for each element individually. This is risky,
609 and you should probably use it along with `dayfirst`.
611 .. versionadded:: 2.0.0
612 dayfirst : bool, default False
613 DD/MM format dates, international and European format.
614 cache_dates : bool, default True
615 If ``True``, use a cache of unique, converted dates to apply the ``datetime``
616 conversion. May produce significant speed-up when parsing duplicate
617 date strings, especially ones with timezone offsets.
619 iterator : bool, default False
620 Return ``TextFileReader`` object for iteration or getting chunks with
621 ``get_chunk()``.
622 chunksize : int, optional
623 Number of lines to read from the file per chunk. Passing a value will cause the
624 function to return a ``TextFileReader`` object for iteration.
625 See the `IO Tools docs
626 <https://pandas.pydata.org/pandas-docs/stable/io.html#io-chunking>`_
627 for more information on ``iterator`` and ``chunksize``.
629 compression : str or dict, default 'infer'
630 For on-the-fly decompression of on-disk data.
631 If 'infer' and 'filepath_or_buffer' is
632 path-like, then detect compression from the following extensions: '.gz',
633 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
634 (otherwise no compression).
635 If using 'zip' or 'tar', the ZIP file must contain only
636 one data file to be read in.
637 Set to ``None`` for no decompression.
638 Can also be a dict with key ``'method'`` set
639 to one of {``'zip'``, ``'gzip'``, ``'bz2'``,
640 ``'zstd'``, ``'xz'``, ``'tar'``} and
641 other key-value pairs are forwarded to
642 ``zipfile.ZipFile``, ``gzip.GzipFile``,
643 ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
644 ``tarfile.TarFile``, respectively.
645 As an example, the following could be passed for
646 Zstandard decompression using a
647 custom compression dictionary:
648 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
650 thousands : str (length 1), optional
651 Character acting as the thousands separator in numerical values.
652 decimal : str (length 1), default '.'
653 Character to recognize as decimal point (e.g., use ',' for European data).
654 lineterminator : str (length 1), optional
655 Character used to denote a line break. Only valid with C parser.
656 quotechar : str (length 1), optional
657 Character used to denote the start and end of a quoted item. Quoted
658 items can include the ``delimiter`` and it will be ignored.
659 quoting : {0 or csv.QUOTE_MINIMAL, 1 or csv.QUOTE_ALL,
660 2 or csv.QUOTE_NONNUMERIC, 3 or csv.QUOTE_NONE}, default csv.QUOTE_MINIMAL
661 Control field quoting behavior per ``csv.QUOTE_*`` constants. Default is
662 ``csv.QUOTE_MINIMAL`` (i.e., 0) which implies that
663 only fields containing special
664 characters are quoted (e.g., characters defined
665 in ``quotechar``, ``delimiter``,
666 or ``lineterminator``.
667 doublequote : bool, default True
668 When ``quotechar`` is specified and ``quoting`` is not ``QUOTE_NONE``, indicate
669 whether or not to interpret two consecutive ``quotechar`` elements INSIDE a
670 field as a single ``quotechar`` element.
671 escapechar : str (length 1), optional
672 Character used to escape other characters.
673 comment : str (length 1), optional
674 Character indicating that the remainder of line should not be parsed.
675 If found at the beginning
676 of a line, the line will be ignored altogether. This parameter must be a
677 single character. Like empty lines (as long as ``skip_blank_lines=True``),
678 fully commented lines are ignored by the parameter ``header`` but not by
679 ``skiprows``. For example, if ``comment='#'``, parsing
680 ``#empty\\na,b,c\\n1,2,3`` with ``header=0`` will result in ``'a,b,c'`` being
681 treated as the header.
682 encoding : str, optional, default 'utf-8'
683 Encoding to use for UTF when reading/writing (ex. ``'utf-8'``). `List of Python
684 standard encodings
685 <https://docs.python.org/3/library/codecs.html#standard-encodings>`_ .
687 encoding_errors : str, optional, default 'strict'
688 How encoding errors are treated. `List of possible values
689 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
691 dialect : str or csv.Dialect, optional
692 If provided, this parameter will override values (default or not) for the
693 following parameters: ``delimiter``, ``doublequote``, ``escapechar``,
694 ``skipinitialspace``, ``quotechar``, and ``quoting``. If it is necessary to
695 override values, a ``ParserWarning`` will be issued. See ``csv.Dialect``
696 documentation for more details.
697 on_bad_lines : {'error', 'warn', 'skip'} or Callable, default 'error'
698 Specifies what to do upon encountering a bad line (a line with too many fields).
699 Allowed values are:
701 - ``'error'``, raise an Exception when a bad line is encountered.
702 - ``'warn'``, raise a warning when a bad line is
703 encountered and skip that line.
704 - ``'skip'``, skip bad lines without raising or warning when
705 they are encountered.
706 - Callable, function that will process a single bad line.
707 - With ``engine='python'``, function with signature
708 ``(bad_line: list[str]) -> list[str] | None``.
709 ``bad_line`` is a list of strings split by the ``sep``.
710 If the function returns ``None``, the bad line will be ignored.
711 If the function returns a new ``list`` of strings with
712 more elements than
713 expected, a ``ParserWarning`` will be emitted while
714 dropping extra elements.
715 - With ``engine='pyarrow'``, function with signature
716 as described in pyarrow documentation: `invalid_row_handler
717 <https://arrow.apache.org/docs/python
718 /generated/pyarrow.csv.ParseOptions.html
719 #pyarrow.csv.ParseOptions.invalid_row_handler>`_.
721 .. versionchanged:: 2.2.0
723 Callable for ``engine='pyarrow'``
725 low_memory : bool, default True
726 Internally process the file in chunks, resulting in lower memory use
727 while parsing, but possibly mixed type inference. To ensure no mixed
728 types either set ``False``, or specify the type with the ``dtype`` parameter.
729 Note that the entire file is read into a single :class:`~pandas.DataFrame`
730 regardless, use the ``chunksize`` or ``iterator``
731 parameter to return the data in
732 chunks. (Only valid with C parser).
733 memory_map : bool, default False
734 If a filepath is provided for ``filepath_or_buffer``, map the file object
735 directly onto memory and access the data directly from there. Using this
736 option can improve performance because there is no longer any I/O overhead.
737 float_precision : {'high', 'legacy', 'round_trip'}, optional
738 Specifies which converter the C engine should use for floating-point
739 values. The options are ``None`` or ``'high'`` for the ordinary converter,
740 ``'legacy'`` for the original lower precision pandas converter, and
741 ``'round_trip'`` for the round-trip converter.
743 storage_options : dict, optional
744 Extra options that make sense for a particular storage connection, e.g.
745 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
746 are forwarded to ``urllib.request.Request`` as header options. For other
747 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
748 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
749 details, and for more examples on storage options refer `here
750 <https://pandas.pydata.org/docs/user_guide/io.html?
751 highlight=storage_options#reading-writing-remote-files>`_.
753 dtype_backend : {'numpy_nullable', 'pyarrow'}
754 Back-end data type applied to the resultant :class:`DataFrame`
755 (still experimental). If not specified, the default behavior
756 is to not use nullable data types. If specified, the behavior
757 is as follows:
759 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
760 * ``"pyarrow"``: returns
761 pyarrow-backed nullable :class:`ArrowDtype` :class:`DataFrame`
763 .. versionadded:: 2.0
765 Returns
766 -------
767 DataFrame or TextFileReader
768 A comma-separated values (csv) file is returned as two-dimensional
769 data structure with labeled axes.
771 See Also
772 --------
773 DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
774 read_table : Read general delimited file into DataFrame.
775 read_fwf : Read a table of fixed-width formatted lines into DataFrame.
777 Examples
778 --------
779 >>> pd.read_csv("data.csv") # doctest: +SKIP
780 Name Value
781 0 foo 1
782 1 bar 2
783 2 #baz 3
785 Index and header can be specified via the `index_col` and `header` arguments.
787 >>> pd.read_csv("data.csv", header=None) # doctest: +SKIP
788 0 1
789 0 Name Value
790 1 foo 1
791 2 bar 2
792 3 #baz 3
794 >>> pd.read_csv("data.csv", index_col="Value") # doctest: +SKIP
795 Name
796 Value
797 1 foo
798 2 bar
799 3 #baz
801 Column types are inferred but can be explicitly specified using the dtype argument.
803 >>> pd.read_csv("data.csv", dtype={"Value": float}) # doctest: +SKIP
804 Name Value
805 0 foo 1.0
806 1 bar 2.0
807 2 #baz 3.0
809 True, False, and NA values, and thousands separators have defaults,
810 but can be explicitly specified, too. Supply the values you would like
811 as strings or lists of strings!
813 >>> pd.read_csv("data.csv", na_values=["foo", "bar"]) # doctest: +SKIP
814 Name Value
815 0 NaN 1
816 1 NaN 2
817 2 #baz 3
819 Comment lines in the input file can be skipped using the `comment` argument.
821 >>> pd.read_csv("data.csv", comment="#") # doctest: +SKIP
822 Name Value
823 0 foo 1
824 1 bar 2
826 By default, columns with dates will be read as ``object`` rather than ``datetime``.
828 >>> df = pd.read_csv("tmp.csv") # doctest: +SKIP
830 >>> df # doctest: +SKIP
831 col 1 col 2 col 3
832 0 10 10/04/2018 Sun 15 Jan 2023
833 1 20 15/04/2018 Fri 12 May 2023
835 >>> df.dtypes # doctest: +SKIP
836 col 1 int64
837 col 2 object
838 col 3 object
839 dtype: object
841 Specific columns can be parsed as dates by using the `parse_dates` and
842 `date_format` arguments.
844 >>> df = pd.read_csv(
845 ... "tmp.csv",
846 ... parse_dates=[1, 2],
847 ... date_format={"col 2": "%d/%m/%Y", "col 3": "%a %d %b %Y"},
848 ... ) # doctest: +SKIP
850 >>> df.dtypes # doctest: +SKIP
851 col 1 int64
852 col 2 datetime64[ns]
853 col 3 datetime64[ns]
854 dtype: object
855 """
856 # locals() should never be modified
857 kwds = locals().copy()
858 del kwds["filepath_or_buffer"]
859 del kwds["sep"]
861 kwds_defaults = _refine_defaults_read(
862 dialect,
863 delimiter,
864 engine,
865 sep,
866 on_bad_lines,
867 names,
868 defaults={"delimiter": ","},
869 dtype_backend=dtype_backend,
870 )
871 kwds.update(kwds_defaults)
873 return _read(filepath_or_buffer, kwds)
876@overload
877def read_table(
878 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
879 *,
880 iterator: Literal[True],
881 chunksize: int | None = ...,
882 **kwds: Unpack[_read_shared[HashableT]],
883) -> TextFileReader: ...
886@overload
887def read_table(
888 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
889 *,
890 iterator: bool = ...,
891 chunksize: int,
892 **kwds: Unpack[_read_shared[HashableT]],
893) -> TextFileReader: ...
896@overload
897def read_table(
898 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
899 *,
900 iterator: Literal[False] = ...,
901 chunksize: None = ...,
902 **kwds: Unpack[_read_shared[HashableT]],
903) -> DataFrame: ...
906@overload
907def read_table(
908 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
909 *,
910 iterator: bool = ...,
911 chunksize: int | None = ...,
912 **kwds: Unpack[_read_shared[HashableT]],
913) -> DataFrame | TextFileReader: ...
916@set_module("pandas")
917def read_table(
918 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
919 *,
920 sep: str | None | lib.NoDefault = lib.no_default,
921 delimiter: str | None | lib.NoDefault = None,
922 # Column and Index Locations and Names
923 header: int | Sequence[int] | None | Literal["infer"] = "infer",
924 names: Sequence[Hashable] | None | lib.NoDefault = lib.no_default,
925 index_col: IndexLabel | Literal[False] | None = None,
926 usecols: UsecolsArgType = None,
927 # General Parsing Configuration
928 dtype: DtypeArg | None = None,
929 engine: CSVEngine | None = None,
930 converters: Mapping[HashableT, Callable] | None = None,
931 true_values: list | None = None,
932 false_values: list | None = None,
933 skipinitialspace: bool = False,
934 skiprows: list[int] | int | Callable[[Hashable], bool] | None = None,
935 skipfooter: int = 0,
936 nrows: int | None = None,
937 # NA and Missing Data Handling
938 na_values: (
939 Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None
940 ) = None,
941 keep_default_na: bool = True,
942 na_filter: bool = True,
943 skip_blank_lines: bool = True,
944 # Datetime Handling
945 parse_dates: bool | Sequence[Hashable] | None = None,
946 date_format: str | dict[Hashable, str] | None = None,
947 dayfirst: bool = False,
948 cache_dates: bool = True,
949 # Iteration
950 iterator: bool = False,
951 chunksize: int | None = None,
952 # Quoting, Compression, and File Format
953 compression: CompressionOptions = "infer",
954 thousands: str | None = None,
955 decimal: str = ".",
956 lineterminator: str | None = None,
957 quotechar: str = '"',
958 quoting: int = csv.QUOTE_MINIMAL,
959 doublequote: bool = True,
960 escapechar: str | None = None,
961 comment: str | None = None,
962 encoding: str | None = None,
963 encoding_errors: str | None = "strict",
964 dialect: str | csv.Dialect | None = None,
965 # Error Handling
966 on_bad_lines: str = "error",
967 # Internal
968 low_memory: bool = _c_parser_defaults["low_memory"],
969 memory_map: bool = False,
970 float_precision: Literal["high", "legacy", "round_trip"] | None = None,
971 storage_options: StorageOptions | None = None,
972 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
973) -> DataFrame | TextFileReader:
974 """
975 Read general delimited file into DataFrame.
977 Also supports optionally iterating or breaking of the file
978 into chunks.
980 Additional help can be found in the online docs for
981 `IO Tools <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html>`_.
983 Parameters
984 ----------
985 filepath_or_buffer : str, path object or file-like object
986 Any valid string path is acceptable. The string could be a URL. Valid
987 URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is
988 expected. A local file could be: file://localhost/path/to/table.csv.
990 If you want to pass in a path object, pandas accepts any ``os.PathLike``.
992 By file-like object, we refer to objects with a ``read()`` method, such as
993 a file handle (e.g. via builtin ``open`` function) or ``StringIO``.
994 sep : str, default '\\t' (tab-stop)
995 Character or regex pattern to treat as the delimiter. If ``sep=None``, the
996 C engine cannot automatically detect
997 the separator, but the Python parsing engine can, meaning the latter will
998 be used and automatically detect the separator from only the first valid
999 row of the file by Python's builtin sniffer tool, ``csv.Sniffer``.
1000 In addition, separators longer than 1 character and different from
1001 ``'\\s+'`` will be interpreted as regular expressions and will also force
1002 the use of the Python parsing engine. Note that regex delimiters are prone
1003 to ignoring quoted data. Regex example: ``'\\r\\t'``.
1004 delimiter : str, optional
1005 Alias for ``sep``.
1006 header : int, Sequence of int, 'infer' or None, default 'infer'
1007 Row number(s) containing column labels and marking the start of the
1008 data (zero-indexed). Default behavior
1009 is to infer the column names: if no ``names``
1010 are passed the behavior is identical to ``header=0`` and column
1011 names are inferred from the first line of the file, if column
1012 names are passed explicitly to ``names`` then the behavior is identical to
1013 ``header=None``. Explicitly pass ``header=0`` to be able to
1014 replace existing names. The header can be a list of integers that
1015 specify row locations for a :class:`~pandas.MultiIndex` on the columns
1016 e.g. ``[0, 1, 3]``. Intervening rows that are not specified will be
1017 skipped (e.g. 2 in this example is skipped). Note that this
1018 parameter ignores commented lines and empty lines if
1019 ``skip_blank_lines=True``, so ``header=0`` denotes the first line of
1020 data rather than the first line of the file.
1022 When inferred from the file contents, headers are kept distinct from
1023 each other by renaming duplicate names with a numeric suffix of the form
1024 ``".{count}"`` starting from 1, e.g. ``"foo"`` and ``"foo.1"``.
1025 Empty headers are named
1026 ``"Unnamed: {i}"`` or ``"Unnamed: {i}_level_{level}"``
1027 in the case of MultiIndex columns.
1028 names : Sequence of Hashable, optional
1029 Sequence of column labels to apply. If the file contains a header row,
1030 then you should explicitly pass ``header=0`` to override the column names.
1031 Duplicates in this list are not allowed.
1032 index_col : Hashable, Sequence of Hashable or False, optional
1033 Column(s) to use as row label(s), denoted either by column labels or column
1034 indices. If a sequence of labels or indices is given,
1035 :class:`~pandas.MultiIndex`
1036 will be formed for the row labels.
1038 Note: ``index_col=False`` can be used to force pandas to *not* use the first
1039 column as the index, e.g., when you have a malformed file with delimiters at
1040 the end of each line.
1041 usecols : Sequence of Hashable or Callable, optional
1042 Subset of columns to select, denoted either by column labels or column indices.
1043 If list-like, all elements must either
1044 be positional (i.e. integer indices into the document columns) or strings
1045 that correspond to column names provided either by the user in ``names`` or
1046 inferred from the document header row(s). If ``names`` are given, the document
1047 header row(s) are not taken into account. For example, a valid list-like
1048 ``usecols`` parameter would be ``[0, 1, 2]`` or ``['foo', 'bar', 'baz']``.
1049 Element order is ignored, so ``usecols=[0, 1]`` is the same as ``[1, 0]``.
1050 To instantiate a :class:`~pandas.DataFrame` from ``data`` with element order
1051 preserved use ``pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]``
1052 for columns in ``['foo', 'bar']`` order or
1053 ``pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]``
1054 for ``['bar', 'foo']`` order.
1056 If callable, the callable function will be evaluated against the column
1057 names, returning names where the callable function evaluates to ``True``. An
1058 example of a valid callable argument would be ``lambda x: x.upper() in
1059 ['AAA', 'BBB', 'DDD']``. Using this parameter results in much faster
1060 parsing time and lower memory usage.
1061 dtype : dtype or dict of {Hashable : dtype}, optional
1062 Data type(s) to apply to either the whole dataset or individual columns.
1063 E.g., ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}``
1064 Use ``str`` or ``object`` together with suitable ``na_values`` settings
1065 to preserve and not interpret ``dtype``.
1066 If ``converters`` are specified, they will be applied INSTEAD
1067 of ``dtype`` conversion. Specify a ``defaultdict`` as input where
1068 the default determines the ``dtype`` of the columns which
1069 are not explicitly listed.
1070 engine : {'c', 'python', 'pyarrow'}, optional
1071 Parser engine to use. The C and pyarrow engines are faster,
1072 while the python engine
1073 is currently more feature-complete. Multithreading is
1074 currently only supported by
1075 the pyarrow engine. The 'pyarrow' engine is an *experimental* engine,
1076 and some features are unsupported, or may not work correctly, with this engine.
1077 converters : dict of {Hashable : Callable}, optional
1078 Functions for converting values in specified columns. Keys can either
1079 be column labels or column indices.
1080 true_values : list, optional
1081 Values to consider as ``True`` in addition to
1082 case-insensitive variants of 'True'.
1083 false_values : list, optional
1084 Values to consider as ``False`` in addition
1085 to case-insensitive variants of 'False'.
1086 skipinitialspace : bool, default False
1087 Skip spaces after delimiter.
1088 skiprows : int, list of int or Callable, optional
1089 Line numbers to skip (0-indexed) or number of lines to skip (``int``)
1090 at the start of the file.
1092 If callable, the callable function will be evaluated against the row
1093 indices, returning ``True`` if the row
1094 should be skipped and ``False`` otherwise.
1095 An example of a valid callable argument would be ``lambda x: x in [0, 2]``.
1096 skipfooter : int, default 0
1097 Number of lines at bottom of file to skip (Unsupported with ``engine='c'``).
1098 nrows : int, optional
1099 Number of rows of file to read. Useful for reading pieces of large files.
1100 Refers to the number of data rows in the returned DataFrame, excluding:
1102 * The header row containing column names.
1103 * Rows before the header row, if ``header=1`` or larger.
1105 Example usage:
1107 * To read the first 999,999 (non-header) rows:
1108 ``read_csv(..., nrows=999999)``
1110 * To read rows 1,000,000 through 1,999,999:
1111 ``read_csv(..., skiprows=1000000, nrows=999999)``
1113 na_values : Hashable, Iterable of Hashable or dict of {Hashable : Iterable},
1114 optional
1115 Additional strings to recognize as ``NA``/``NaN``.
1116 If ``dict`` passed, specific
1117 per-column ``NA`` values. By default the following values are interpreted as
1118 ``NaN``: empty string, "NaN", "N/A", "NULL", and other
1119 common representations of missing data.
1120 keep_default_na : bool, default True
1121 Whether or not to include the default ``NaN`` values when parsing the data.
1122 Depending on whether ``na_values`` is passed in, the behavior is as follows:
1124 * If ``keep_default_na`` is ``True``,
1125 and ``na_values`` are specified, ``na_values``
1126 is appended to the default ``NaN`` values used for parsing.
1127 * If ``keep_default_na`` is ``True``, and ``na_values`` are not specified, only
1128 the default ``NaN`` values are used for parsing.
1129 * If ``keep_default_na`` is ``False``, and ``na_values`` are specified, only
1130 the ``NaN`` values specified ``na_values`` are used for parsing.
1131 * If ``keep_default_na`` is ``False``, and ``na_values`` are not specified, no
1132 strings will be parsed as ``NaN``.
1134 Note that if ``na_filter`` is passed in as
1135 ``False``, the ``keep_default_na`` and
1136 ``na_values`` parameters will be ignored.
1137 na_filter : bool, default True
1138 Detect missing value markers (empty strings and the value of ``na_values``). In
1139 data without any ``NA`` values, passing ``na_filter=False`` can improve the
1140 performance of reading a large file.
1141 skip_blank_lines : bool, default True
1142 If ``True``, skip over blank lines rather than interpreting as ``NaN`` values.
1143 parse_dates : bool, None, list of Hashable, default None
1144 The behavior is as follows:
1146 * ``bool``. If ``True`` -> try parsing the index.
1147 * ``None``. Behaves like ``True`` if ``date_format`` is specified.
1148 * ``list`` of ``int`` or names.
1149 e.g. If ``[1, 2, 3]`` -> try parsing columns 1, 2, 3
1150 each as a separate date column.
1152 If a column or index cannot be represented as an array of ``datetime``,
1153 say because of an unparsable value or a mixture of timezones, the column
1154 or index will be returned unaltered as an ``object`` data type. For
1155 non-standard ``datetime`` parsing, use :func:`~pandas.to_datetime` after
1156 :func:`~pandas.read_csv`.
1158 Note: A fast-path exists for iso8601-formatted dates.
1159 date_format : str or dict of column -> format, optional
1160 Format to use for parsing dates and/or times when used
1161 in conjunction with ``parse_dates``.
1162 The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See
1163 `strftime documentation
1164 <https://docs.python.org/3/library/datetime.html
1165 #strftime-and-strptime-behavior>`_ for more information on choices, though
1166 note that :const:`"%f"`` will parse all the way up to nanoseconds.
1167 You can also pass:
1169 - "ISO8601", to parse any `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_
1170 time string (not necessarily in exactly the same format);
1171 - "mixed", to infer the format for each element individually. This is risky,
1172 and you should probably use it along with `dayfirst`.
1174 .. versionadded:: 2.0.0
1175 dayfirst : bool, default False
1176 DD/MM format dates, international and European format.
1177 cache_dates : bool, default True
1178 If ``True``, use a cache of unique, converted dates to apply the ``datetime``
1179 conversion. May produce significant speed-up when parsing duplicate
1180 date strings, especially ones with timezone offsets.
1182 iterator : bool, default False
1183 Return ``TextFileReader`` object for iteration or getting chunks with
1184 ``get_chunk()``.
1185 chunksize : int, optional
1186 Number of lines to read from the file per chunk. Passing a value will cause the
1187 function to return a ``TextFileReader`` object for iteration.
1188 See the `IO Tools docs
1189 <https://pandas.pydata.org/pandas-docs/stable/io.html#io-chunking>`_
1190 for more information on ``iterator`` and ``chunksize``.
1192 compression : str or dict, default 'infer'
1193 For on-the-fly decompression of on-disk data. If 'infer'
1194 and 'filepath_or_buffer' is
1195 path-like, then detect compression from the following extensions: '.gz',
1196 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
1197 (otherwise no compression).
1198 If using 'zip' or 'tar', the ZIP file must contain
1199 only one data file to be read in.
1200 Set to ``None`` for no decompression.
1201 Can also be a dict with key ``'method'`` set
1202 to one of {``'zip'``, ``'gzip'``, ``'bz2'``,
1203 ``'zstd'``, ``'xz'``, ``'tar'``} and
1204 other key-value pairs are forwarded to
1205 ``zipfile.ZipFile``, ``gzip.GzipFile``,
1206 ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
1207 ``tarfile.TarFile``, respectively.
1208 As an example, the following could be passed for
1209 Zstandard decompression using a
1210 custom compression dictionary:
1211 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
1213 thousands : str (length 1), optional
1214 Character acting as the thousands separator in numerical values.
1215 decimal : str (length 1), default '.'
1216 Character to recognize as decimal point (e.g., use ',' for European data).
1217 lineterminator : str (length 1), optional
1218 Character used to denote a line break. Only valid with C parser.
1219 quotechar : str (length 1), optional
1220 Character used to denote the start and end of a quoted item. Quoted
1221 items can include the ``delimiter`` and it will be ignored.
1222 quoting : {0 or csv.QUOTE_MINIMAL, 1 or csv.QUOTE_ALL, 2 or
1223 csv.QUOTE_NONNUMERIC, 3 or csv.QUOTE_NONE}, default csv.QUOTE_MINIMAL
1224 Control field quoting behavior per ``csv.QUOTE_*`` constants. Default is
1225 ``csv.QUOTE_MINIMAL`` (i.e., 0) which
1226 implies that only fields containing special
1227 characters are quoted (e.g., characters defined
1228 in ``quotechar``, ``delimiter``,
1229 or ``lineterminator``.
1230 doublequote : bool, default True
1231 When ``quotechar`` is specified and ``quoting`` is not ``QUOTE_NONE``, indicate
1232 whether or not to interpret two consecutive ``quotechar`` elements INSIDE a
1233 field as a single ``quotechar`` element.
1234 escapechar : str (length 1), optional
1235 Character used to escape other characters.
1236 comment : str (length 1), optional
1237 Character indicating that the remainder of line should not be parsed.
1238 If found at the beginning
1239 of a line, the line will be ignored altogether. This parameter must be a
1240 single character. Like empty lines (as long as ``skip_blank_lines=True``),
1241 fully commented lines are ignored by the parameter ``header`` but not by
1242 ``skiprows``. For example, if ``comment='#'``, parsing
1243 ``#empty\\na,b,c\\n1,2,3`` with ``header=0`` will result in ``'a,b,c'`` being
1244 treated as the header.
1245 encoding : str, optional, default 'utf-8'
1246 Encoding to use for UTF when reading/writing (ex. ``'utf-8'``). `List of Python
1247 standard encodings
1248 <https://docs.python.org/3/library/codecs.html#standard-encodings>`_ .
1250 encoding_errors : str, optional, default 'strict'
1251 How encoding errors are treated. `List of possible values
1252 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
1254 dialect : str or csv.Dialect, optional
1255 If provided, this parameter will override values (default or not) for the
1256 following parameters: ``delimiter``, ``doublequote``, ``escapechar``,
1257 ``skipinitialspace``, ``quotechar``, and ``quoting``. If it is necessary to
1258 override values, a ``ParserWarning`` will be issued. See ``csv.Dialect``
1259 documentation for more details.
1260 on_bad_lines : {'error', 'warn', 'skip'} or Callable, default 'error'
1261 Specifies what to do upon encountering a bad
1262 line (a line with too many fields).
1263 Allowed values are:
1265 - ``'error'``, raise an Exception when a bad line is encountered.
1266 - ``'warn'``, raise a warning when a bad line is encountered and
1267 skip that line.
1268 - ``'skip'``, skip bad lines without raising or warning when they
1269 are encountered.
1270 - Callable, function that will process a single bad line.
1271 - With ``engine='python'``, function with signature
1272 ``(bad_line: list[str]) -> list[str] | None``.
1273 ``bad_line`` is a list of strings split by the ``sep``.
1274 If the function returns ``None``, the bad line will be ignored.
1275 If the function returns a new ``list`` of strings with more elements than
1276 expected, a ``ParserWarning`` will be emitted while
1277 dropping extra elements.
1278 - With ``engine='pyarrow'``, function with signature
1279 as described in pyarrow documentation: `invalid_row_handler
1280 <https://arrow.apache.org/docs/
1281 python/generated/pyarrow.csv.ParseOptions.html
1282 #pyarrow.csv.ParseOptions.invalid_row_handler>`_.
1284 .. versionadded:: 2.2.0
1286 Callable for ``engine='pyarrow'``
1288 low_memory : bool, default True
1289 Internally process the file in chunks, resulting in lower memory use
1290 while parsing, but possibly mixed type inference. To ensure no mixed
1291 types either set ``False``, or specify the type with the ``dtype`` parameter.
1292 Note that the entire file is read into a single :class:`~pandas.DataFrame`
1293 regardless, use the ``chunksize`` or ``iterator`` parameter
1294 to return the data in
1295 chunks. (Only valid with C parser).
1296 memory_map : bool, default False
1297 If a filepath is provided for ``filepath_or_buffer``, map the file object
1298 directly onto memory and access the data directly from there. Using this
1299 option can improve performance because there is no longer any I/O overhead.
1300 float_precision : {'high', 'legacy', 'round_trip'}, optional
1301 Specifies which converter the C engine should use for floating-point
1302 values. The options are ``None`` or ``'high'`` for the ordinary converter,
1303 ``'legacy'`` for the original lower precision pandas converter, and
1304 ``'round_trip'`` for the round-trip converter.
1306 storage_options : dict, optional
1307 Extra options that make sense for a particular storage connection, e.g.
1308 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
1309 are forwarded to ``urllib.request.Request`` as header options. For other
1310 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
1311 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
1312 details, and for more examples on storage options refer `here
1313 <https://pandas.pydata.org/docs/user_guide/io.html?
1314 highlight=storage_options#reading-writing-remote-files>`_.
1316 dtype_backend : {'numpy_nullable', 'pyarrow'}
1317 Back-end data type applied to the resultant :class:`DataFrame`
1318 (still experimental). If not specified, the default behavior
1319 is to not use nullable data types. If specified, the behavior
1320 is as follows:
1322 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
1323 * ``"pyarrow"``: returns pyarrow-backed nullable
1324 :class:`ArrowDtype` :class:`DataFrame`
1326 .. versionadded:: 2.0
1328 Returns
1329 -------
1330 DataFrame or TextFileReader
1331 A comma-separated values (csv) file is returned as two-dimensional
1332 data structure with labeled axes.
1334 See Also
1335 --------
1336 DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
1337 read_csv : Read a comma-separated values (csv) file into DataFrame.
1338 read_fwf : Read a table of fixed-width formatted lines into DataFrame.
1340 Examples
1341 --------
1342 >>> pd.read_table("data.csv") # doctest: +SKIP
1343 Name Value
1344 0 foo 1
1345 1 bar 2
1346 2 #baz 3
1348 Index and header can be specified via the `index_col` and `header` arguments.
1350 >>> pd.read_table("data.csv", header=None) # doctest: +SKIP
1351 0 1
1352 0 Name Value
1353 1 foo 1
1354 2 bar 2
1355 3 #baz 3
1357 >>> pd.read_table("data.csv", index_col="Value") # doctest: +SKIP
1358 Name
1359 Value
1360 1 foo
1361 2 bar
1362 3 #baz
1364 Column types are inferred but can be explicitly specified using the dtype argument.
1366 >>> pd.read_table("data.csv", dtype={"Value": float}) # doctest: +SKIP
1367 Name Value
1368 0 foo 1.0
1369 1 bar 2.0
1370 2 #baz 3.0
1372 True, False, and NA values, and thousands separators have defaults,
1373 but can be explicitly specified, too. Supply the values you would like
1374 as strings or lists of strings!
1376 >>> pd.read_table("data.csv", na_values=["foo", "bar"]) # doctest: +SKIP
1377 Name Value
1378 0 NaN 1
1379 1 NaN 2
1380 2 #baz 3
1382 Comment lines in the input file can be skipped using the `comment` argument.
1384 >>> pd.read_table("data.csv", comment="#") # doctest: +SKIP
1385 Name Value
1386 0 foo 1
1387 1 bar 2
1389 By default, columns with dates will be read as ``object`` rather than ``datetime``.
1391 >>> df = pd.read_table("tmp.csv") # doctest: +SKIP
1393 >>> df # doctest: +SKIP
1394 col 1 col 2 col 3
1395 0 10 10/04/2018 Sun 15 Jan 2023
1396 1 20 15/04/2018 Fri 12 May 2023
1398 >>> df.dtypes # doctest: +SKIP
1399 col 1 int64
1400 col 2 object
1401 col 3 object
1402 dtype: object
1404 Specific columns can be parsed as dates by using the `parse_dates` and
1405 `date_format` arguments.
1407 >>> df = pd.read_table(
1408 ... "tmp.csv",
1409 ... parse_dates=[1, 2],
1410 ... date_format={"col 2": "%d/%m/%Y", "col 3": "%a %d %b %Y"},
1411 ... ) # doctest: +SKIP
1413 >>> df.dtypes # doctest: +SKIP
1414 col 1 int64
1415 col 2 datetime64[ns]
1416 col 3 datetime64[ns]
1417 dtype: object
1418 """
1419 # locals() should never be modified
1420 kwds = locals().copy()
1421 del kwds["filepath_or_buffer"]
1422 del kwds["sep"]
1424 kwds_defaults = _refine_defaults_read(
1425 dialect,
1426 delimiter,
1427 engine,
1428 sep,
1429 on_bad_lines,
1430 names,
1431 defaults={"delimiter": "\t"},
1432 dtype_backend=dtype_backend,
1433 )
1434 kwds.update(kwds_defaults)
1436 return _read(filepath_or_buffer, kwds)
1439@overload
1440def read_fwf(
1441 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1442 *,
1443 colspecs: Sequence[tuple[int, int]] | str | None = ...,
1444 widths: Sequence[int] | None = ...,
1445 infer_nrows: int = ...,
1446 iterator: Literal[True],
1447 chunksize: int | None = ...,
1448 **kwds: Unpack[_read_shared[HashableT]],
1449) -> TextFileReader: ...
1452@overload
1453def read_fwf(
1454 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1455 *,
1456 colspecs: Sequence[tuple[int, int]] | str | None = ...,
1457 widths: Sequence[int] | None = ...,
1458 infer_nrows: int = ...,
1459 iterator: bool = ...,
1460 chunksize: int,
1461 **kwds: Unpack[_read_shared[HashableT]],
1462) -> TextFileReader: ...
1465@overload
1466def read_fwf(
1467 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1468 *,
1469 colspecs: Sequence[tuple[int, int]] | str | None = ...,
1470 widths: Sequence[int] | None = ...,
1471 infer_nrows: int = ...,
1472 iterator: Literal[False] = ...,
1473 chunksize: None = ...,
1474 **kwds: Unpack[_read_shared[HashableT]],
1475) -> DataFrame: ...
1478@set_module("pandas")
1479def read_fwf(
1480 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1481 *,
1482 colspecs: Sequence[tuple[int, int]] | str | None = "infer",
1483 widths: Sequence[int] | None = None,
1484 infer_nrows: int = 100,
1485 iterator: bool = False,
1486 chunksize: int | None = None,
1487 **kwds: Unpack[_read_shared[HashableT]],
1488) -> DataFrame | TextFileReader:
1489 r"""
1490 Read a table of fixed-width formatted lines into DataFrame.
1492 Also supports optionally iterating or breaking of the file
1493 into chunks.
1495 Additional help can be found in the `online docs for IO Tools
1496 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html>`_.
1498 Parameters
1499 ----------
1500 filepath_or_buffer : str, path object, or file-like object
1501 String, path object (implementing ``os.PathLike[str]``), or file-like
1502 object implementing a text ``read()`` function.The string could be a URL.
1503 Valid URL schemes include http, ftp, s3, and file. For file URLs, a host is
1504 expected. A local file could be:
1505 ``file://localhost/path/to/table.csv``.
1506 colspecs : list of tuple (int, int) or 'infer'. optional
1507 A list of tuples giving the extents of the fixed-width
1508 fields of each line as half-open intervals (i.e., [from, to] ).
1509 String value 'infer' can be used to instruct the parser to try
1510 detecting the column specifications from the first 100 rows of
1511 the data which are not being skipped via skiprows (default='infer').
1512 widths : list of int, optional
1513 A list of field widths which can be used instead of 'colspecs' if
1514 the intervals are contiguous.
1515 infer_nrows : int, default 100
1516 The number of rows to consider when letting the parser determine the
1517 `colspecs`.
1518 iterator : bool, default False
1519 Return ``TextFileReader`` object for iteration or getting chunks with
1520 ``get_chunk()``.
1521 chunksize : int, optional
1522 Number of lines to read from the file per chunk.
1523 **kwds : optional
1524 Optional keyword arguments can be passed to ``TextFileReader``.
1526 Returns
1527 -------
1528 DataFrame or TextFileReader
1529 A comma-separated values (csv) file is returned as two-dimensional
1530 data structure with labeled axes.
1532 See Also
1533 --------
1534 DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
1535 read_csv : Read a comma-separated values (csv) file into DataFrame.
1537 Examples
1538 --------
1539 >>> pd.read_fwf("data.csv") # doctest: +SKIP
1540 """
1541 # Check input arguments.
1542 if colspecs is None and widths is None:
1543 raise ValueError("Must specify either colspecs or widths")
1544 if colspecs not in (None, "infer") and widths is not None:
1545 raise ValueError("You must specify only one of 'widths' and 'colspecs'")
1547 # Compute 'colspecs' from 'widths', if specified.
1548 if widths is not None:
1549 colspecs, col = [], 0
1550 for w in widths:
1551 colspecs.append((col, col + w))
1552 col += w
1554 # for mypy
1555 assert colspecs is not None
1557 # GH#40830
1558 # Ensure length of `colspecs` matches length of `names`
1559 names = kwds.get("names")
1560 if names is not None and names is not lib.no_default:
1561 if len(names) != len(colspecs) and colspecs != "infer":
1562 # need to check len(index_col) as it might contain
1563 # unnamed indices, in which case it's name is not required
1564 len_index = 0
1565 if kwds.get("index_col") is not None:
1566 index_col: Any = kwds.get("index_col")
1567 if index_col is not False:
1568 if not is_list_like(index_col):
1569 len_index = 1
1570 else:
1571 # for mypy: handled in the if-branch
1572 assert index_col is not lib.no_default
1574 len_index = len(index_col)
1575 if kwds.get("usecols") is None and len(names) + len_index != len(colspecs):
1576 # If usecols is used colspec may be longer than names
1577 raise ValueError("Length of colspecs must match length of names")
1579 check_dtype_backend(kwds.setdefault("dtype_backend", lib.no_default))
1580 return _read(
1581 filepath_or_buffer,
1582 kwds
1583 | {
1584 "colspecs": colspecs,
1585 "infer_nrows": infer_nrows,
1586 "engine": "python-fwf",
1587 "iterator": iterator,
1588 "chunksize": chunksize,
1589 },
1590 )
1593class TextFileReader(abc.Iterator):
1594 """
1596 Passed dialect overrides any of the related parser options
1598 """
1600 def __init__(
1601 self,
1602 f: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str] | list,
1603 engine: CSVEngine | None = None,
1604 **kwds,
1605 ) -> None:
1606 if engine is not None:
1607 engine_specified = True
1608 else:
1609 engine = "python"
1610 engine_specified = False
1611 self.engine = engine
1612 self._engine_specified = kwds.get("engine_specified", engine_specified)
1614 _validate_skipfooter(kwds)
1616 dialect = _extract_dialect(kwds)
1617 if dialect is not None:
1618 if engine == "pyarrow":
1619 raise ValueError(
1620 "The 'dialect' option is not supported with the 'pyarrow' engine"
1621 )
1622 kwds = _merge_with_dialect_properties(dialect, kwds)
1624 if kwds.get("header", "infer") == "infer":
1625 kwds["header"] = 0 if kwds.get("names") is None else None
1627 self.orig_options = kwds
1629 # miscellanea
1630 self._currow = 0
1632 options = self._get_options_with_defaults(engine)
1633 options["storage_options"] = kwds.get("storage_options", None)
1635 self.chunksize = options.pop("chunksize", None)
1636 self.nrows = options.pop("nrows", None)
1638 self._check_file_or_buffer(f, engine)
1639 self.options, self.engine = self._clean_options(options, engine)
1641 if "has_index_names" in kwds:
1642 self.options["has_index_names"] = kwds["has_index_names"]
1644 self.handles: IOHandles | None = None
1645 self._engine = self._make_engine(f, self.engine)
1647 def close(self) -> None:
1648 if self.handles is not None:
1649 self.handles.close()
1650 self._engine.close()
1652 def _get_options_with_defaults(self, engine: CSVEngine) -> dict[str, Any]:
1653 kwds = self.orig_options
1655 options = {}
1656 default: object | None
1658 for argname, default in parser_defaults.items():
1659 value = kwds.get(argname, default)
1661 # see gh-12935
1662 if (
1663 engine == "pyarrow"
1664 and argname in _pyarrow_unsupported
1665 and value != default
1666 and value != getattr(value, "value", default)
1667 ):
1668 raise ValueError(
1669 f"The {argname!r} option is not supported with the 'pyarrow' engine"
1670 )
1671 options[argname] = value
1673 for argname, default in _c_parser_defaults.items():
1674 if argname in kwds:
1675 value = kwds[argname]
1677 if engine != "c" and value != default:
1678 # TODO: Refactor this logic, its pretty convoluted
1679 if "python" in engine and argname not in _python_unsupported:
1680 pass
1681 elif "pyarrow" in engine and argname not in _pyarrow_unsupported:
1682 pass
1683 else:
1684 raise ValueError(
1685 f"The {argname!r} option is not supported with the "
1686 f"{engine!r} engine"
1687 )
1688 else:
1689 value = default
1690 options[argname] = value
1692 if engine == "python-fwf":
1693 for argname, default in _fwf_defaults.items():
1694 options[argname] = kwds.get(argname, default)
1696 return options
1698 def _check_file_or_buffer(self, f, engine: CSVEngine) -> None:
1699 # see gh-16530
1700 if is_file_like(f) and engine != "c" and not hasattr(f, "__iter__"):
1701 # The C engine doesn't need the file-like to have the "__iter__"
1702 # attribute. However, the Python engine needs "__iter__(...)"
1703 # when iterating through such an object, meaning it
1704 # needs to have that attribute
1705 raise ValueError(
1706 "The 'python' engine cannot iterate through this file buffer."
1707 )
1708 if hasattr(f, "encoding"):
1709 file_encoding = f.encoding
1710 orig_reader_enc = self.orig_options.get("encoding", None)
1711 any_none = file_encoding is None or orig_reader_enc is None
1712 if file_encoding != orig_reader_enc and not any_none:
1713 file_path = getattr(f, "name", None)
1714 raise ValueError(
1715 f"The specified reader encoding {orig_reader_enc} is different "
1716 f"from the encoding {file_encoding} of file {file_path}."
1717 )
1719 def _clean_options(
1720 self, options: dict[str, Any], engine: CSVEngine
1721 ) -> tuple[dict[str, Any], CSVEngine]:
1722 result = options.copy()
1724 fallback_reason = None
1726 # C engine not supported yet
1727 if engine == "c":
1728 if options["skipfooter"] > 0:
1729 fallback_reason = "the 'c' engine does not support skipfooter"
1730 engine = "python"
1732 sep = options["delimiter"]
1734 if sep is not None and len(sep) > 1:
1735 if engine == "c" and sep == r"\s+":
1736 # delim_whitespace passed on to pandas._libs.parsers.TextReader
1737 result["delim_whitespace"] = True
1738 del result["delimiter"]
1739 elif engine not in ("python", "python-fwf"):
1740 # wait until regex engine integrated
1741 fallback_reason = (
1742 f"the '{engine}' engine does not support "
1743 "regex separators (separators > 1 char and "
1744 r"different from '\s+' are interpreted as regex)"
1745 )
1746 engine = "python"
1747 elif sep is not None:
1748 encodeable = True
1749 encoding = sys.getfilesystemencoding() or "utf-8"
1750 try:
1751 if len(sep.encode(encoding)) > 1:
1752 encodeable = False
1753 except UnicodeDecodeError:
1754 encodeable = False
1755 if not encodeable and engine not in ("python", "python-fwf"):
1756 fallback_reason = (
1757 f"the separator encoded in {encoding} "
1758 f"is > 1 char long, and the '{engine}' engine "
1759 "does not support such separators"
1760 )
1761 engine = "python"
1763 quotechar = options["quotechar"]
1764 if quotechar is not None and isinstance(quotechar, (str, bytes)):
1765 if (
1766 len(quotechar) == 1
1767 and ord(quotechar) > 127
1768 and engine not in ("python", "python-fwf")
1769 ):
1770 fallback_reason = (
1771 "ord(quotechar) > 127, meaning the "
1772 "quotechar is larger than one byte, "
1773 f"and the '{engine}' engine does not support such quotechars"
1774 )
1775 engine = "python"
1777 if fallback_reason and self._engine_specified:
1778 raise ValueError(fallback_reason)
1780 if engine == "c":
1781 for arg in _c_unsupported:
1782 del result[arg]
1784 if "python" in engine:
1785 for arg in _python_unsupported:
1786 if fallback_reason and result[arg] != _c_parser_defaults.get(arg):
1787 raise ValueError(
1788 "Falling back to the 'python' engine because "
1789 f"{fallback_reason}, but this causes {arg!r} to be "
1790 "ignored as it is not supported by the 'python' engine."
1791 )
1792 del result[arg]
1794 if fallback_reason:
1795 warnings.warn(
1796 (
1797 "Falling back to the 'python' engine because "
1798 f"{fallback_reason}; you can avoid this warning by specifying "
1799 "engine='python'."
1800 ),
1801 ParserWarning,
1802 stacklevel=find_stack_level(),
1803 )
1805 index_col = options["index_col"]
1806 names = options["names"]
1807 converters = options["converters"]
1808 na_values = options["na_values"]
1809 skiprows = options["skiprows"]
1811 validate_header_arg(options["header"])
1813 if index_col is True:
1814 raise ValueError("The value of index_col couldn't be 'True'")
1815 if is_index_col(index_col):
1816 if not isinstance(index_col, (list, tuple, np.ndarray)):
1817 index_col = [index_col]
1818 result["index_col"] = index_col
1820 names = list(names) if names is not None else names
1822 # type conversion-related
1823 if converters is not None:
1824 if not isinstance(converters, dict):
1825 raise TypeError(
1826 "Type converters must be a dict or subclass, "
1827 f"input was a {type(converters).__name__}"
1828 )
1829 else:
1830 converters = {}
1832 # Converting values to NA
1833 keep_default_na = options["keep_default_na"]
1834 floatify = engine != "pyarrow"
1835 na_values, na_fvalues = _clean_na_values(
1836 na_values, keep_default_na, floatify=floatify
1837 )
1839 # handle skiprows; this is internally handled by the
1840 # c-engine, so only need for python and pyarrow parsers
1841 if engine == "pyarrow":
1842 if not is_integer(skiprows) and skiprows is not None:
1843 # pyarrow expects skiprows to be passed as an integer
1844 raise ValueError(
1845 "skiprows argument must be an integer when using engine='pyarrow'"
1846 )
1847 else:
1848 if is_integer(skiprows):
1849 skiprows = range(skiprows)
1850 if skiprows is None:
1851 skiprows = set()
1852 elif not callable(skiprows):
1853 skiprows = set(skiprows)
1855 # put stuff back
1856 result["names"] = names
1857 result["converters"] = converters
1858 result["na_values"] = na_values
1859 result["na_fvalues"] = na_fvalues
1860 result["skiprows"] = skiprows
1862 return result, engine
1864 def __next__(self) -> DataFrame:
1865 try:
1866 return self.get_chunk()
1867 except StopIteration:
1868 self.close()
1869 raise
1871 def _make_engine(
1872 self,
1873 f: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str] | list | IO,
1874 engine: CSVEngine = "c",
1875 ) -> ParserBase:
1876 mapping: dict[str, type[ParserBase]] = {
1877 "c": CParserWrapper,
1878 "python": PythonParser,
1879 "pyarrow": ArrowParserWrapper,
1880 "python-fwf": FixedWidthFieldParser,
1881 }
1883 if engine not in mapping:
1884 raise ValueError(
1885 f"Unknown engine: {engine} (valid options are {mapping.keys()})"
1886 )
1887 if not isinstance(f, list):
1888 # open file here
1889 is_text = True
1890 mode = "r"
1891 if engine == "pyarrow":
1892 is_text = False
1893 mode = "rb"
1894 elif (
1895 engine == "c"
1896 and self.options.get("encoding", "utf-8") == "utf-8"
1897 and isinstance(stringify_path(f), str)
1898 ):
1899 # c engine can decode utf-8 bytes, adding TextIOWrapper makes
1900 # the c-engine especially for memory_map=True far slower
1901 is_text = False
1902 if "b" not in mode:
1903 mode += "b"
1904 self.handles = get_handle(
1905 f,
1906 mode,
1907 encoding=self.options.get("encoding", None),
1908 compression=self.options.get("compression", None),
1909 memory_map=self.options.get("memory_map", False),
1910 is_text=is_text,
1911 errors=self.options.get("encoding_errors", "strict"),
1912 storage_options=self.options.get("storage_options", None),
1913 )
1914 assert self.handles is not None
1915 f = self.handles.handle
1917 elif engine != "python":
1918 msg = f"Invalid file path or buffer object type: {type(f)}"
1919 raise ValueError(msg)
1921 try:
1922 return mapping[engine](f, **self.options)
1923 except Exception:
1924 if self.handles is not None:
1925 self.handles.close()
1926 raise
1928 def _failover_to_python(self) -> None:
1929 raise AbstractMethodError(self)
1931 def read(self, nrows: int | None = None) -> DataFrame:
1932 if self.engine == "pyarrow":
1933 try:
1934 # error: "ParserBase" has no attribute "read"
1935 df = self._engine.read() # type: ignore[attr-defined]
1936 except Exception:
1937 self.close()
1938 raise
1939 else:
1940 nrows = validate_integer("nrows", nrows)
1941 try:
1942 # error: "ParserBase" has no attribute "read"
1943 (
1944 index,
1945 columns,
1946 col_dict,
1947 ) = self._engine.read( # type: ignore[attr-defined]
1948 nrows
1949 )
1950 except Exception:
1951 self.close()
1952 raise
1954 if index is None:
1955 if col_dict:
1956 # Any column is actually fine:
1957 new_rows = len(next(iter(col_dict.values())))
1958 index = RangeIndex(self._currow, self._currow + new_rows)
1959 else:
1960 new_rows = 0
1961 else:
1962 new_rows = len(index)
1964 if hasattr(self, "orig_options"):
1965 dtype_arg = self.orig_options.get("dtype", None)
1966 else:
1967 dtype_arg = None
1969 if isinstance(dtype_arg, dict):
1970 dtype = defaultdict(lambda: None) # type: ignore[var-annotated]
1971 dtype.update(dtype_arg)
1972 elif dtype_arg is not None and pandas_dtype(dtype_arg) in (
1973 np.str_,
1974 np.object_,
1975 ):
1976 dtype = defaultdict(lambda: dtype_arg)
1977 else:
1978 dtype = None
1980 if dtype is not None:
1981 new_col_dict = {}
1982 for k, v in col_dict.items():
1983 d = (
1984 dtype[k]
1985 if pandas_dtype(dtype[k]) in (np.str_, np.object_)
1986 else None
1987 )
1988 new_col_dict[k] = Series(v, index=index, dtype=d, copy=False)
1989 else:
1990 new_col_dict = col_dict
1992 df = DataFrame(
1993 new_col_dict,
1994 columns=columns,
1995 index=index,
1996 copy=False,
1997 )
1999 self._currow += new_rows
2000 return df
2002 def get_chunk(self, size: int | None = None) -> DataFrame:
2003 if size is None:
2004 size = self.chunksize
2005 if self.nrows is not None:
2006 if self._currow >= self.nrows:
2007 raise StopIteration
2008 if size is None:
2009 size = self.nrows - self._currow
2010 else:
2011 size = min(size, self.nrows - self._currow)
2012 return self.read(nrows=size)
2014 def __enter__(self) -> Self:
2015 return self
2017 def __exit__(
2018 self,
2019 exc_type: type[BaseException] | None,
2020 exc_value: BaseException | None,
2021 traceback: TracebackType | None,
2022 ) -> None:
2023 self.close()
2026def TextParser(*args, **kwds) -> TextFileReader:
2027 """
2028 Converts lists of lists/tuples into DataFrames with proper type inference
2029 and optional (e.g. string to datetime) conversion. Also enables iterating
2030 lazily over chunks of large files
2032 Parameters
2033 ----------
2034 data : file-like object or list
2035 delimiter : separator character to use
2036 dialect : str or csv.Dialect instance, optional
2037 Ignored if delimiter is longer than 1 character
2038 names : sequence, default
2039 header : int, default 0
2040 Row to use to parse column labels. Defaults to the first row. Prior
2041 rows will be discarded
2042 index_col : int or list, optional
2043 Column or columns to use as the (possibly hierarchical) index
2044 has_index_names: bool, default False
2045 True if the cols defined in index_col have an index name and are
2046 not in the header.
2047 na_values : scalar, str, list-like, or dict, optional
2048 Additional strings to recognize as NA/NaN.
2049 keep_default_na : bool, default True
2050 thousands : str, optional
2051 Thousands separator
2052 comment : str, optional
2053 Comment out remainder of line
2054 parse_dates : bool, default False
2055 date_format : str or dict of column -> format, default ``None``
2057 .. versionadded:: 2.0.0
2058 skiprows : list of integers
2059 Row numbers to skip
2060 skipfooter : int
2061 Number of line at bottom of file to skip
2062 converters : dict, optional
2063 Dict of functions for converting values in certain columns. Keys can
2064 either be integers or column labels, values are functions that take one
2065 input argument, the cell (not column) content, and return the
2066 transformed content.
2067 encoding : str, optional
2068 Encoding to use for UTF when reading/writing (ex. 'utf-8')
2069 float_precision : str, optional
2070 Specifies which converter the C engine should use for floating-point
2071 values. The options are `None` or `high` for the ordinary converter,
2072 `legacy` for the original lower precision pandas converter, and
2073 `round_trip` for the round-trip converter.
2074 """
2075 kwds["engine"] = "python"
2076 return TextFileReader(*args, **kwds)
2079def _clean_na_values(na_values, keep_default_na: bool = True, floatify: bool = True):
2080 na_fvalues: set | dict
2081 if na_values is None:
2082 if keep_default_na:
2083 na_values = STR_NA_VALUES
2084 else:
2085 na_values = set()
2086 na_fvalues = set()
2087 elif isinstance(na_values, dict):
2088 old_na_values = na_values.copy()
2089 na_values = {} # Prevent aliasing.
2091 # Convert the values in the na_values dictionary
2092 # into array-likes for further use. This is also
2093 # where we append the default NaN values, provided
2094 # that `keep_default_na=True`.
2095 for k, v in old_na_values.items():
2096 if not is_list_like(v):
2097 v = [v]
2099 if keep_default_na:
2100 v = set(v) | STR_NA_VALUES
2102 na_values[k] = _stringify_na_values(v, floatify)
2103 na_fvalues = {k: _floatify_na_values(v) for k, v in na_values.items()}
2104 else:
2105 if not is_list_like(na_values):
2106 na_values = [na_values]
2107 na_values = _stringify_na_values(na_values, floatify)
2108 if keep_default_na:
2109 na_values = na_values | STR_NA_VALUES
2111 na_fvalues = _floatify_na_values(na_values)
2113 return na_values, na_fvalues
2116def _floatify_na_values(na_values) -> set[float]:
2117 # create float versions of the na_values
2118 result = set()
2119 for v in na_values:
2120 try:
2121 v = float(v)
2122 if not np.isnan(v):
2123 result.add(v)
2124 except (TypeError, ValueError, OverflowError):
2125 pass
2126 return result
2129def _stringify_na_values(na_values, floatify: bool) -> set[str | float]:
2130 """return a stringified and numeric for these values"""
2131 result: list[str | float] = []
2132 for x in na_values:
2133 result.append(str(x))
2134 result.append(x)
2135 try:
2136 v = float(x)
2138 # we are like 999 here
2139 if v == int(v):
2140 v = int(v)
2141 result.append(f"{v}.0")
2142 result.append(str(v))
2144 if floatify:
2145 result.append(v)
2146 except (TypeError, ValueError, OverflowError):
2147 pass
2148 if floatify:
2149 try:
2150 result.append(int(x))
2151 except (TypeError, ValueError, OverflowError):
2152 pass
2153 return set(result)
2156def _refine_defaults_read(
2157 dialect: str | csv.Dialect | None,
2158 delimiter: str | None | lib.NoDefault,
2159 engine: CSVEngine | None,
2160 sep: str | None | lib.NoDefault,
2161 on_bad_lines: str | Callable,
2162 names: Sequence[Hashable] | None | lib.NoDefault,
2163 defaults: dict[str, Any],
2164 dtype_backend: DtypeBackend | lib.NoDefault,
2165):
2166 """Validate/refine default values of input parameters of read_csv, read_table.
2168 Parameters
2169 ----------
2170 dialect : str or csv.Dialect
2171 If provided, this parameter will override values (default or not) for the
2172 following parameters: `delimiter`, `doublequote`, `escapechar`,
2173 `skipinitialspace`, `quotechar`, and `quoting`. If it is necessary to
2174 override values, a ParserWarning will be issued. See csv.Dialect
2175 documentation for more details.
2176 delimiter : str or object
2177 Alias for sep.
2178 engine : {'c', 'python'}
2179 Parser engine to use. The C engine is faster while the python engine is
2180 currently more feature-complete.
2181 sep : str or object
2182 A delimiter provided by the user (str) or a sentinel value, i.e.
2183 pandas._libs.lib.no_default.
2184 on_bad_lines : str, callable
2185 An option for handling bad lines or a sentinel value(None).
2186 names : array-like, optional
2187 List of column names to use. If the file contains a header row,
2188 then you should explicitly pass ``header=0`` to override the column names.
2189 Duplicates in this list are not allowed.
2190 defaults: dict
2191 Default values of input parameters.
2193 Returns
2194 -------
2195 kwds : dict
2196 Input parameters with correct values.
2197 """
2198 # fix types for sep, delimiter to Union(str, Any)
2199 delim_default = defaults["delimiter"]
2200 kwds: dict[str, Any] = {}
2201 # gh-23761
2202 #
2203 # When a dialect is passed, it overrides any of the overlapping
2204 # parameters passed in directly. We don't want to warn if the
2205 # default parameters were passed in (since it probably means
2206 # that the user didn't pass them in explicitly in the first place).
2207 #
2208 # "delimiter" is the annoying corner case because we alias it to
2209 # "sep" before doing comparison to the dialect values later on.
2210 # Thus, we need a flag to indicate that we need to "override"
2211 # the comparison to dialect values by checking if default values
2212 # for BOTH "delimiter" and "sep" were provided.
2213 if dialect is not None:
2214 kwds["sep_override"] = delimiter is None and (
2215 sep is lib.no_default or sep == delim_default
2216 )
2218 if delimiter and (sep is not lib.no_default):
2219 raise ValueError("Specified a sep and a delimiter; you can only specify one.")
2221 kwds["names"] = None if names is lib.no_default else names
2223 # Alias sep -> delimiter.
2224 if delimiter is None:
2225 delimiter = sep
2227 if delimiter == "\n":
2228 raise ValueError(
2229 r"Specified \n as separator or delimiter. This forces the python engine "
2230 "which does not accept a line terminator. Hence it is not allowed to use "
2231 "the line terminator as separator.",
2232 )
2234 if delimiter is lib.no_default:
2235 # assign default separator value
2236 kwds["delimiter"] = delim_default
2237 else:
2238 kwds["delimiter"] = delimiter
2240 if engine is not None:
2241 kwds["engine_specified"] = True
2242 else:
2243 kwds["engine"] = "c"
2244 kwds["engine_specified"] = False
2246 if on_bad_lines == "error":
2247 kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.ERROR
2248 elif on_bad_lines == "warn":
2249 kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.WARN
2250 elif on_bad_lines == "skip":
2251 kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.SKIP
2252 elif callable(on_bad_lines):
2253 if engine not in ["python", "pyarrow"]:
2254 raise ValueError(
2255 "on_bad_line can only be a callable function "
2256 "if engine='python' or 'pyarrow'"
2257 )
2258 kwds["on_bad_lines"] = on_bad_lines
2259 else:
2260 raise ValueError(f"Argument {on_bad_lines} is invalid for on_bad_lines")
2262 check_dtype_backend(dtype_backend)
2264 kwds["dtype_backend"] = dtype_backend
2266 return kwds
2269def _extract_dialect(kwds: dict[str, str | csv.Dialect]) -> csv.Dialect | None:
2270 """
2271 Extract concrete csv dialect instance.
2273 Returns
2274 -------
2275 csv.Dialect or None
2276 """
2277 if kwds.get("dialect") is None:
2278 return None
2280 dialect = kwds["dialect"]
2281 if isinstance(dialect, str) and dialect in csv.list_dialects():
2282 # get_dialect is typed to return a `_csv.Dialect` for some reason in typeshed
2283 tdialect = cast(csv.Dialect, csv.get_dialect(dialect))
2284 _validate_dialect(tdialect)
2286 else:
2287 _validate_dialect(dialect)
2288 tdialect = cast(csv.Dialect, dialect)
2290 return tdialect
2293MANDATORY_DIALECT_ATTRS = (
2294 "delimiter",
2295 "doublequote",
2296 "escapechar",
2297 "skipinitialspace",
2298 "quotechar",
2299 "quoting",
2300)
2303def _validate_dialect(dialect: csv.Dialect | str) -> None:
2304 """
2305 Validate csv dialect instance.
2307 Raises
2308 ------
2309 ValueError
2310 If incorrect dialect is provided.
2311 """
2312 for param in MANDATORY_DIALECT_ATTRS:
2313 if not hasattr(dialect, param):
2314 raise ValueError(f"Invalid dialect {dialect} provided")
2317def _merge_with_dialect_properties(
2318 dialect: csv.Dialect,
2319 defaults: dict[str, Any],
2320) -> dict[str, Any]:
2321 """
2322 Merge default kwargs in TextFileReader with dialect parameters.
2324 Parameters
2325 ----------
2326 dialect : csv.Dialect
2327 Concrete csv dialect. See csv.Dialect documentation for more details.
2328 defaults : dict
2329 Keyword arguments passed to TextFileReader.
2331 Returns
2332 -------
2333 kwds : dict
2334 Updated keyword arguments, merged with dialect parameters.
2335 """
2336 kwds = defaults.copy()
2338 for param in MANDATORY_DIALECT_ATTRS:
2339 dialect_val = getattr(dialect, param)
2341 parser_default = parser_defaults[param]
2342 provided = kwds.get(param, parser_default)
2344 # Messages for conflicting values between the dialect
2345 # instance and the actual parameters provided.
2346 conflict_msgs = []
2348 # Don't warn if the default parameter was passed in,
2349 # even if it conflicts with the dialect (gh-23761).
2350 if provided not in (parser_default, dialect_val):
2351 msg = (
2352 f"Conflicting values for '{param}': '{provided}' was "
2353 f"provided, but the dialect specifies '{dialect_val}'. "
2354 "Using the dialect-specified value."
2355 )
2357 # Annoying corner case for not warning about
2358 # conflicts between dialect and delimiter parameter.
2359 # Refer to the outer "_read_" function for more info.
2360 if not (param == "delimiter" and kwds.pop("sep_override", False)):
2361 conflict_msgs.append(msg)
2363 if conflict_msgs:
2364 warnings.warn(
2365 "\n\n".join(conflict_msgs), ParserWarning, stacklevel=find_stack_level()
2366 )
2367 kwds[param] = dialect_val
2368 return kwds
2371def _validate_skipfooter(kwds: dict[str, Any]) -> None:
2372 """
2373 Check whether skipfooter is compatible with other kwargs in TextFileReader.
2375 Parameters
2376 ----------
2377 kwds : dict
2378 Keyword arguments passed to TextFileReader.
2380 Raises
2381 ------
2382 ValueError
2383 If skipfooter is not compatible with other parameters.
2384 """
2385 if kwds.get("skipfooter"):
2386 if kwds.get("iterator") or kwds.get("chunksize"):
2387 raise ValueError("'skipfooter' not supported for iteration")
2388 if kwds.get("nrows"):
2389 raise ValueError("'skipfooter' not supported with 'nrows'")