Shortcuts on this page
r m x toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Module contains tools for processing files into DataFrames or other objects
4GH#48849 provides a convenient way of deprecating keyword arguments
5"""
7from __future__ import annotations
9import codecs
10from collections import (
11 abc,
12 defaultdict,
13)
14from concurrent.futures import ThreadPoolExecutor
15import contextlib
16import csv
17import io
18import mmap
19import os
20import queue
21import sys
22import threading
23from typing import (
24 IO,
25 TYPE_CHECKING,
26 Any,
27 Generic,
28 Literal,
29 Self,
30 TypedDict,
31 Unpack,
32 cast,
33 overload,
34)
35import warnings
37import numpy as np
39from pandas._config import get_option
41from pandas._libs import lib
42from pandas._libs.parsers import STR_NA_VALUES
43from pandas.compat._cpu import available_cpu_count
44from pandas.errors import (
45 AbstractMethodError,
46 EmptyDataError,
47 Pandas4Warning,
48 ParserError,
49 ParserWarning,
50)
51from pandas.util._decorators import (
52 set_module,
53)
54from pandas.util._exceptions import find_stack_level
55from pandas.util._validators import (
56 check_dtype_backend,
57 validate_bool_kwarg,
58)
60from pandas.core.dtypes.common import (
61 is_float,
62 is_integer,
63 is_list_like,
64 pandas_dtype,
65)
66from pandas.core.dtypes.dtypes import ArrowDtype
67from pandas.core.dtypes.inference import is_file_like
69from pandas import Series
70from pandas.core.arrays import (
71 DatetimeArray,
72 TimedeltaArray,
73)
74from pandas.core.frame import DataFrame
75from pandas.core.indexes.api import (
76 RangeIndex,
77 ensure_index,
78)
79from pandas.core.internals.api import _make_block
80from pandas.core.internals.managers import BlockManager
82from pandas.io.common import (
83 IOHandles,
84 get_handle,
85 infer_compression,
86 stringify_path,
87 validate_header_arg,
88)
89from pandas.io.parsers.arrow_parser_wrapper import ArrowParserWrapper
90from pandas.io.parsers.base_parser import (
91 ParserBase,
92 is_index_col,
93 parser_defaults,
94)
95from pandas.io.parsers.c_parser_wrapper import (
96 CParserWrapper,
97 _concatenate_chunks,
98)
99from pandas.io.parsers.python_parser import (
100 FixedWidthFieldParser,
101 PythonParser,
102)
104if TYPE_CHECKING:
105 from collections.abc import (
106 Callable,
107 Hashable,
108 Iterable,
109 Mapping,
110 Sequence,
111 )
112 from types import TracebackType
114 from pandas._typing import (
115 CompressionOptions,
116 CSVEngine,
117 DtypeArg,
118 DtypeBackend,
119 FilePath,
120 HashableT,
121 IndexLabel,
122 ReadCsvBuffer,
123 StorageOptions,
124 UsecolsArgType,
125 )
127 class _read_shared(TypedDict, Generic[HashableT], total=False):
128 # annotations shared between read_csv/fwf/table's overloads
129 # NOTE: Keep in sync with the annotations of the implementation
130 sep: str | lib.NoDefault | None
131 delimiter: str | lib.NoDefault | None
132 header: int | Sequence[int] | Literal["infer"] | None
133 names: Sequence[Hashable] | lib.NoDefault | None
134 index_col: IndexLabel | Literal[False] | None
135 usecols: UsecolsArgType
136 dtype: DtypeArg | None
137 engine: CSVEngine | None
138 converters: Mapping[HashableT, Callable] | None
139 true_values: list | None
140 false_values: list | None
141 skipinitialspace: bool
142 skiprows: list[int] | int | Callable[[Hashable], bool] | None
143 skipfooter: int
144 nrows: int | None
145 na_values: (
146 Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None
147 )
148 keep_default_na: bool
149 na_filter: bool
150 skip_blank_lines: bool
151 parse_dates: bool | Sequence[Hashable] | None
152 date_format: str | dict[Hashable, str] | None
153 dayfirst: bool
154 cache_dates: bool
155 compression: CompressionOptions
156 thousands: str | None
157 decimal: str
158 lineterminator: str | None
159 quotechar: str
160 quoting: int
161 doublequote: bool
162 escapechar: str | None
163 comment: str | None
164 encoding: str | None
165 encoding_errors: str | None
166 dialect: str | csv.Dialect | type[csv.Dialect] | None
167 on_bad_lines: str
168 low_memory: bool
169 memory_map: bool
170 float_precision: Literal["high", "legacy", "round_trip"] | None
171 storage_options: StorageOptions | None
172 dtype_backend: DtypeBackend | lib.NoDefault
174else:
175 _read_shared = dict
178class _C_Parser_Defaults(TypedDict):
179 na_filter: Literal[True]
180 low_memory: Literal[True]
181 memory_map: Literal[False]
182 float_precision: None
185_c_parser_defaults: _C_Parser_Defaults = {
186 "na_filter": True,
187 "low_memory": True,
188 "memory_map": False,
189 "float_precision": None,
190}
193class _Fwf_Defaults(TypedDict):
194 colspecs: Literal["infer"]
195 infer_nrows: Literal[100]
196 widths: None
199_fwf_defaults: _Fwf_Defaults = {"colspecs": "infer", "infer_nrows": 100, "widths": None}
200_c_unsupported = {"skipfooter"}
201_python_unsupported = {"low_memory", "float_precision"}
203# Documented as `bool` and consumed for their truthiness, so an unvalidated
204# non-bool like "False" silently means the opposite (GH#68341).
205_bool_kwargs = frozenset(
206 {
207 "cache_dates",
208 "dayfirst",
209 "doublequote",
210 "iterator",
211 "keep_default_na",
212 "low_memory",
213 "memory_map",
214 "na_filter",
215 "skip_blank_lines",
216 "skipinitialspace",
217 }
218)
221def _is_bool_like(value: object) -> bool:
222 # Only 0 and 1 stand in for the bools, numpy ints included; a larger int is
223 # truthy but not a bool, see test_bool_kwarg_int_not_zero_or_one (GH#68341)
224 return lib.is_bool(value) or (is_integer(value) and value in (0, 1))
227def _validate_bool_kwargs(kwds: Mapping[str, Any]) -> None:
228 # iterate kwds, not the frozenset, so the kwarg named is the first in
229 # signature order rather than whichever one hash randomization picks
230 for kwd, value in kwds.items():
231 if kwd in _bool_kwargs and not _is_bool_like(value):
232 # always raises; validate_bool_kwarg owns the message
233 validate_bool_kwarg(value, kwd, none_allowed=False)
236# Minimum file size (bytes) to attempt parallel CSV reading.
237# Below this threshold the overhead of splitting and threading outweighs the benefit.
238# Break-even is around 1-2 MB; 5 MB leaves margin for cold-cache reads.
239_PARALLEL_READ_MIN_BYTES = 5 * 1024 * 1024 # 5 MB
240# Minimum rows per parallel chunk, bounding how finely a file is split.
241_PARALLEL_MIN_CHUNK_ROWS = 2000
242# Target bytes per parallel chunk. Between pyarrow's 1 MB read block and
243# DuckDB's 8 MB per-thread unit.
244_PARALLEL_CHUNK_BYTES = 4 * 1024 * 1024 # 4 MB
245# Ceiling on the (column x chunk) pieces the byte-driven count may create,
246# bounding the per-column cost (a GIL-held allocation and dtype inference)
247# that every chunk repeats however wide the frame is.
248_PARALLEL_MAX_COLUMN_PIECES = 1800
249# Size of the last chunk relative to a full one. Chunks are handed out first
250# come, first served, so the read ends when the last chunk *started* finishes;
251# tapering leaves a worker that arrives late something short to take. 1.0
252# disables the taper.
253_PARALLEL_TAPER_RATIO = 0.2
255# Ceiling on the *default* parallel-read worker count: parallel CSV reading
256# sees diminishing returns beyond a handful of workers, and a low default
257# avoids oversubscribing the machine. mode.max_threads overrides it in either
258# direction.
259_MAX_DEFAULT_WORKERS = 6
260_pyarrow_unsupported = {
261 "skipfooter",
262 "float_precision",
263 "chunksize",
264 "comment",
265 "nrows",
266 "thousands",
267 "memory_map",
268 "dialect",
269 "quoting",
270 "lineterminator",
271 "converters",
272 "iterator",
273 "dayfirst",
274 "skipinitialspace",
275 "low_memory",
276 "na_filter",
277}
280@overload
281def validate_integer(name: str, val: None, min_val: int = ...) -> None: ...
284@overload
285def validate_integer(name: str, val: float, min_val: int = ...) -> int: ...
288@overload
289def validate_integer(name: str, val: int | None, min_val: int = ...) -> int | None: ...
292def validate_integer(
293 name: str, val: int | float | None, min_val: int = 0
294) -> int | None:
295 """
296 Checks whether the 'name' parameter for parsing is either
297 an integer OR float that can SAFELY be cast to an integer
298 without losing accuracy. Raises a ValueError if that is
299 not the case.
301 Parameters
302 ----------
303 name : str
304 Parameter name (used for error reporting)
305 val : int or float
306 The value to check
307 min_val : int
308 Minimum allowed value (val < min_val will result in a ValueError)
309 """
310 if val is None:
311 return val
313 msg = f"'{name:s}' must be an integer >={min_val:d}"
314 if is_float(val):
315 if int(val) != val:
316 raise ValueError(msg)
317 val = int(val)
318 elif not (is_integer(val) and val >= min_val):
319 raise ValueError(msg)
321 return int(val)
324def _validate_names(names: Sequence[Hashable] | None) -> None:
325 """
326 Raise ValueError if the `names` parameter contains duplicates or has an
327 invalid data type.
329 Parameters
330 ----------
331 names : array-like or None
332 An array containing a list of the names used for the output DataFrame.
334 Raises
335 ------
336 ValueError
337 If names are not unique or are not ordered (e.g. set).
338 """
339 if names is not None:
340 if len(names) != len(set(names)):
341 raise ValueError("Duplicate names are not allowed.")
342 if not (
343 is_list_like(names, allow_sets=False) or isinstance(names, abc.KeysView)
344 ):
345 raise ValueError("Names should be an ordered collection.")
348def _read(
349 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str], kwds
350) -> DataFrame | TextFileReader:
351 """Generic reader of line files."""
352 # before the `iterator` peek below, which reads it for truthiness
353 _validate_bool_kwargs(kwds)
355 # if we pass a date_format and parse_dates=False, we should not parse the
356 # dates GH#44366
357 if kwds.get("parse_dates", None) is None:
358 if kwds.get("date_format", None) is None:
359 kwds["parse_dates"] = False
360 else:
361 kwds["parse_dates"] = True
363 # Extract some of the arguments (pass chunksize on).
364 iterator = kwds.get("iterator", False)
365 chunksize = kwds.get("chunksize", None)
367 # Check type of encoding_errors
368 errors = kwds.get("encoding_errors", "strict")
369 if not isinstance(errors, str):
370 raise ValueError(
371 f"encoding_errors must be a string, got {type(errors).__name__}"
372 )
374 if kwds.get("engine") == "pyarrow":
375 if iterator:
376 raise ValueError(
377 "The 'iterator' option is not supported with the 'pyarrow' engine"
378 )
380 if chunksize is not None:
381 raise ValueError(
382 "The 'chunksize' option is not supported with the 'pyarrow' engine"
383 )
384 else:
385 chunksize = validate_integer("chunksize", chunksize, 1)
387 nrows = kwds.get("nrows", None)
389 # Check for duplicates in names.
390 _validate_names(kwds.get("names", None))
392 if kwds.get("float_precision") is not None:
393 warnings.warn(
394 "The 'float_precision' argument is deprecated. "
395 "Use the default float precision instead.",
396 Pandas4Warning,
397 stacklevel=find_stack_level(),
398 )
400 # For large local uncompressed files with the C engine, attempt parallel reading.
401 # Each worker gets its own view of the mmapped file and its own TextReader so
402 # the GIL-free tokenisation and type-conversion code in parsers.pyx runs truly
403 # in parallel.
404 if not iterator and chunksize is None and nrows is None:
405 _n_workers = _default_n_workers()
406 if _n_workers > 1 and _can_parallelize_csv(filepath_or_buffer, kwds):
407 _filepath = stringify_path(filepath_or_buffer)
408 assert isinstance(_filepath, str) # guaranteed by _can_parallelize_csv
409 try:
410 result = _read_csv_parallel(_filepath, kwds, _n_workers)
411 except (ParserError, UnicodeDecodeError, OverflowError):
412 # e.g. the one-line sample that infers the column names ends
413 # inside a quoted field (GH#66259). The serial path below
414 # handles anything the parallel path cannot -- and raises in
415 # turn if it too fails.
416 # Other exceptions from outside the chunk reads propagate: they
417 # signal a parallel-path bug, not ineligible input.
418 result = None
419 if result is not None:
420 return result
422 # Create the parser.
423 parser = TextFileReader(filepath_or_buffer, **kwds)
425 if chunksize or iterator:
426 return parser
428 with parser:
429 return parser.read(nrows)
432def _default_n_workers() -> int:
433 """
434 Default worker count for a parallel ``read_csv``.
436 ``mode.max_threads`` wins whenever it is set (except on Emscripten, which
437 cannot spawn threads at all). Otherwise it is the smallest of the machine's
438 logical CPU count, ``_MAX_DEFAULT_WORKERS``, and the CPUs actually available
439 to the process (CPU affinity / cgroup limits) -- so that an embedded or
440 containerised pandas does not oversubscribe its allocation.
441 """
442 max_threads = get_option("mode.max_threads")
443 if sys.platform == "emscripten":
444 # WASM cannot spawn threads, regardless of mode.max_threads.
445 return 1
446 if max_threads is not None:
447 return max_threads
448 n_workers = min(os.cpu_count() or 1, _MAX_DEFAULT_WORKERS)
449 # os.cpu_count() counts the machine's CPUs, not the ones this process may
450 # use, so it alone would put _MAX_DEFAULT_WORKERS parse threads on a
451 # single-CPU container.
452 available = available_cpu_count()
453 if available is not None:
454 n_workers = min(n_workers, available)
455 return n_workers
458def _can_parallelize_csv(filepath_or_buffer, kwds: dict) -> bool:
459 """
460 Return True when a ``read_csv`` call is eligible for parallel execution.
462 Parallel reading works by splitting the file's data section into
463 byte-range chunks at newline boundaries and processing them concurrently
464 with threads. The following conditions must all hold:
466 * *filepath_or_buffer* is a local file path (not a URL or file object).
467 * The file is not compressed.
468 * The C engine is selected (``engine='c'``, the default).
469 * The call is not in iterator / chunked mode.
470 * The entire file is being read (no ``nrows`` limit).
471 * ``skiprows`` is a plain integer (or absent) - list/callable forms require
472 tracking absolute line numbers across chunks.
473 * ``header`` is a single integer or ``None`` - multi-level headers complicate
474 the preamble boundary.
475 * ``index_col`` is ``None`` or ``False`` - a column-based index would need its
476 name propagated to non-first chunks.
477 * ``usecols`` is ``None`` - column selection changes the mapping between raw
478 column positions and names in non-first chunks.
479 * The separator is a single ASCII character or ``r"\\s+"``, and ``quotechar``
480 is a single ASCII character - anything else forces the python engine
481 inside ``TextFileReader``.
482 * The encoding is ``utf-8`` / ``utf-8-sig`` - chunk workers feed raw file
483 bytes to the C tokenizer, which decodes words as UTF-8. ``ascii`` is
484 excluded so a non-ASCII byte still raises rather than being masked.
485 * ``comment`` is ``None`` and the preamble has no blank lines - full-line
486 comments and blank lines shift where pandas locates the header relative
487 to the physical line count used by :func:`_find_data_start_offset`.
488 * ``escapechar`` is ``None`` and ``dialect`` is not used - an escaped
489 literal newline would be split mid-field.
490 * ``parse_dates`` is unset - datetime format inference is per-chunk and may
491 disagree with the serial whole-column inference.
492 * ``low_memory`` is truthy - a falsy ``low_memory`` guarantees whole-file
493 type inference, which per-chunk parallel reading cannot honour
494 (``low_memory=True`` already documents per-chunk inference divergence).
495 * ``storage_options`` is ``None`` - it raises for local paths in the serial
496 path, and that error must not be masked.
497 * ``on_bad_lines`` is not ``"warn"`` - chunk workers would report
498 chunk-relative (i.e. wrong) line numbers.
499 * Both the file and its data section (i.e. excluding the header preamble)
500 are at least ``_PARALLEL_READ_MIN_BYTES`` bytes large.
502 Note that a chunk boundary landing on a newline embedded inside a quoted
503 field leaves that chunk's parser inside an open quote at EOF, so the chunk
504 fails and the read falls back to serial - it does not corrupt data
505 silently.
506 """
507 # Must be a local file path, not a URL or file-like object.
508 if is_file_like(filepath_or_buffer):
509 return False
510 filepath = stringify_path(filepath_or_buffer)
511 if not isinstance(filepath, str):
512 return False
513 if "://" in filepath:
514 return False
516 # storage_options on a local path raises ValueError in get_handle; the
517 # parallel path strips it and would mask that error.
518 if kwds.get("storage_options") is not None:
519 return False
521 # Uncompressed files only (splitting is only meaningful for raw bytes).
522 compression = kwds.get("compression", "infer")
523 if isinstance(compression, dict):
524 compression = compression.get("method")
525 if compression is not None:
526 try:
527 if infer_compression(filepath, compression) is not None:
528 return False
529 except ValueError:
530 # unrecognized compression argument; let the serial path raise
531 return False
533 # Iterator / chunked mode: caller handles chunking themselves.
534 if kwds.get("iterator", False) or kwds.get("chunksize") is not None:
535 return False
537 # Only the C engine benefits; pyarrow has its own parallelism.
538 # engine=None means "use the default", which is "c".
539 if kwds.get("engine") not in ("c", None):
540 return False
542 # nrows: we'd need to distribute the row budget across chunks - skip for now.
543 if kwds.get("nrows") is not None:
544 return False
546 # The chunk splitter scans for raw \n bytes. A custom lineterminator
547 # (which the C engine does honour) would misalign chunk boundaries.
548 lineterminator = kwds.get("lineterminator")
549 if lineterminator is not None and lineterminator != "\n":
550 return False
552 # Chunk workers feed raw file bytes to the C tokenizer, which decodes
553 # words as UTF-8. The serial path instead transcodes through a
554 # TextIOWrapper, so anything that is not already valid UTF-8 (latin-1,
555 # cp1252, UTF-16/32, ...) must take the serial path. "ascii" is excluded
556 # for the same reason: the workers would decode a non-ASCII byte as UTF-8
557 # and succeed, masking the UnicodeDecodeError the serial path raises for
558 # bytes outside the declared encoding (GH#64347).
559 encoding = kwds.get("encoding")
560 if encoding is not None:
561 try:
562 codec_name = codecs.lookup(encoding).name
563 except LookupError:
564 return False
565 if codec_name not in ("utf-8", "utf-8-sig"):
566 return False
568 # Separators that force the python-engine fallback inside TextFileReader
569 # (sep=None sniffing, or multi-char/regex seps other than r"\s+") cannot
570 # use the C-engine buffer-loading fast path.
571 delimiter = kwds.get("delimiter", ",")
572 if delimiter is None or (len(delimiter) > 1 and delimiter != r"\s+"):
573 return False
575 # A separator or quotechar wider than one byte forces the python engine
576 # too, and warns on the way. The eligibility check has to catch these
577 # here: by the time the name-inference read reports the python engine, it
578 # has already raised that warning, which the serial read then raises
579 # again. GH#66259
580 if len(delimiter) == 1 and ord(delimiter) > 127:
581 return False
582 quotechar = kwds.get("quotechar")
583 if isinstance(quotechar, str) and len(quotechar) == 1 and ord(quotechar) > 127:
584 return False
586 # Full-line comments before/inside the preamble shift the header location
587 # in ways _find_data_start_offset cannot see.
588 if kwds.get("comment") is not None:
589 return False
591 # An escaped literal newline (escapechar followed by "\n") would be split
592 # mid-field.
593 if kwds.get("escapechar") is not None:
594 return False
596 # dialect is merged into the kwds inside TextFileReader, i.e. after this
597 # function runs, so a dialect-specified escapechar would bypass the check
598 # above. Be conservative and take the serial path.
599 if kwds.get("dialect") is not None:
600 return False
602 # Datetime format inference is per-chunk and may disagree with the serial
603 # whole-column inference.
604 if kwds.get("parse_dates"):
605 return False
607 # A falsy low_memory guarantees whole-file type inference, which per-chunk
608 # parallel reading cannot honour (GH#64347). Test truthiness, not identity:
609 # low_memory is not coerced to bool, and the serial path in
610 # CParserWrapper.read also branches on truthiness, so low_memory=0 /
611 # np.False_ must take the serial path too (GH#66327).
612 if not kwds.get("low_memory", True):
613 return False
615 # on_bad_lines="warn" includes line numbers in its warnings; chunk workers
616 # would report chunk-relative (i.e. wrong) ones.
617 if kwds.get("on_bad_lines") == ParserBase.BadLineHandleMethod.BLHM_WARN:
618 return False
620 # Callable / list skiprows require tracking absolute line numbers.
621 skiprows = kwds.get("skiprows")
622 if skiprows is not None and not isinstance(skiprows, (int, np.integer)):
623 return False
625 # Multi-level (list) headers span multiple preamble lines - skip for now.
626 if isinstance(kwds.get("header", "infer"), (list, tuple)):
627 return False
629 # skipfooter: the C engine doesn't support it anyway, but be explicit.
630 if kwds.get("skipfooter", 0) > 0:
631 return False
633 # A column-based index_col would need its name propagated to non-first chunks.
634 index_col = kwds.get("index_col", None)
635 if index_col is not None and index_col is not False:
636 return False
638 # usecols changes the column name ↔ position mapping in non-first chunks.
639 if kwds.get("usecols") is not None:
640 return False
642 # Only bother for files large enough to amortise the threading overhead.
643 try:
644 file_size = os.path.getsize(filepath)
645 if file_size < _PARALLEL_READ_MIN_BYTES:
646 return False
647 except OSError:
648 return False
650 # Check that the *data section* (after header/skiprows preamble) is large
651 # enough to split into at least 2 chunks. Without this, _read_csv_parallel
652 # would have to return None for files whose preamble eats most of the bytes.
653 _header = kwds.get("header", "infer")
654 if _header == "infer":
655 _header = 0 if kwds.get("names") is None else None
656 _skiprows = int(kwds.get("skiprows") or 0)
657 data_start = _find_data_start_offset(filepath, _header, _skiprows)
658 if file_size - data_start < _PARALLEL_READ_MIN_BYTES:
659 return False
661 # Blank lines in the preamble shift where pandas locates the header
662 # relative to the physical line count used by _find_data_start_offset.
663 with open(filepath, "rb") as fd:
664 preamble = fd.read(data_start)
665 if any(not line.rstrip(b"\r") for line in preamble.splitlines()):
666 return False
668 return True
671def _find_data_start_offset(
672 filepath: str,
673 header: int | None,
674 skiprows: int,
675) -> int:
676 """
677 Return the byte offset of the first data row in *filepath*.
679 Reads ``int(skiprows) + (header + 1 if header is not None else 0)``
680 physical lines from the start of the file and returns the position after
681 the last preamble byte. This is the byte from which each non-first chunk
682 must begin.
683 """
684 if header is None:
685 header_lines = 0
686 else:
687 header_lines = int(header) + 1 # header=0 → 1 header line
689 preamble_lines = int(skiprows) + header_lines
690 with open(filepath, "rb") as fd:
691 for _ in range(preamble_lines):
692 if not fd.readline():
693 break
694 return fd.tell()
697def _chunk_size_weights(n_chunks: int, n_tapered: int) -> list[float]:
698 """
699 Relative sizes for *n_chunks* chunks whose last *n_tapered* shrink.
701 The chunks before the tail are full size; across the tail the size falls
702 linearly to ``_PARALLEL_TAPER_RATIO`` of one, so the final chunk is the
703 smallest piece of work in the file.
704 """
705 weights = [1.0] * n_chunks
706 n_tapered = min(n_tapered, n_chunks)
707 for position in range(n_tapered):
708 share = (position + 1) / n_tapered
709 weights[n_chunks - n_tapered + position] = (
710 1.0 - (1.0 - _PARALLEL_TAPER_RATIO) * share
711 )
712 return weights
715def _find_chunk_byte_offsets(
716 filepath: str,
717 n_chunks: int,
718 data_start: int,
719 n_workers: int = 0,
720) -> list[int]:
721 """
722 Compute byte offsets that partition the data portion of *filepath* into
723 *n_chunks* pieces aligned to newline boundaries.
725 The pieces are equal-sized, except that when *n_workers* is given and the
726 chunks take more than one round the last ``2 * n_workers`` taper down: the
727 read finishes when the last chunk a worker picked up does, so ending on
728 small chunks costs less than ending on a full one. Within a single round
729 every chunk runs at once, so the makespan is the largest of them and any
730 taper would only inflate it.
732 Returns a list of ``n + 1`` offsets where the byte range
733 ``[offsets[i], offsets[i+1])`` defines chunk *i*.
734 """
735 file_size = os.path.getsize(filepath)
736 data_size = file_size - data_start
737 offsets = [data_start]
739 if n_chunks <= 1 or data_size <= 0:
740 offsets.append(file_size)
741 return offsets
743 n_tapered = 2 * n_workers if n_chunks > n_workers else 0
744 weights = _chunk_size_weights(n_chunks, n_tapered)
745 total_weight = sum(weights)
746 running_weight = 0.0
747 with open(filepath, "rb") as fd:
748 for i in range(1, n_chunks):
749 running_weight += weights[i - 1]
750 target = data_start + int(data_size * running_weight / total_weight)
751 if target >= file_size:
752 break
753 fd.seek(target)
754 fd.readline() # advance past the partial line at the split point
755 pos = fd.tell()
756 if pos >= file_size:
757 break
758 if pos == offsets[-1]:
759 # This target fell inside the line the previous boundary
760 # already ended; skip it rather than abandoning the boundaries
761 # after it, so one long line cannot collapse the whole split.
762 continue
763 offsets.append(pos)
765 offsets.append(file_size)
766 return offsets
769def _raise_collected(warning_sink: list[tuple[str, type[Warning]]]) -> None:
770 """Raise each distinct warning the parallel read's parsers collected."""
771 for warn_msg, warn_category in dict.fromkeys(warning_sink):
772 warnings.warn(warn_msg, warn_category, stacklevel=find_stack_level())
775def _read_csv_parallel(
776 filepath: str,
777 kwds: dict,
778 n_workers: int,
779) -> DataFrame | None:
780 """
781 Read a large CSV file in parallel using *n_workers* threads.
783 Every read this makes - the one-line name inference and each chunk - raises
784 its own copy of the same ``ParserWarning``, and a worker raising one lands
785 the stacklevel walk in threading internals. So the reads collect their
786 warnings into a sink (see :attr:`TextReader.warning_sink`) and each distinct
787 one is raised once here, from the caller's frame - except when the caller
788 goes on to read the file serially, since that read raises them itself.
789 GH#66259
791 The file's data section (everything after the header / skiprows preamble)
792 is split into byte-range chunks aligned to newline boundaries, sized from
793 *n_workers* and from the file's own rows, bytes and column count. Each
794 chunk is parsed by an independent
795 :class:`TextFileReader` / C-engine instance. Because the hot paths in
796 ``pandas/_libs/parsers.pyx`` (tokenisation, int/float/bool conversion)
797 are wrapped in ``with nogil:`` blocks, threads achieve real CPU-level
798 parallelism.
800 The caller must ensure the file is eligible via :func:`_can_parallelize_csv`
801 before calling this function.
803 Returns ``None`` when parallel reading turns out not to be applicable
804 after all (the data section cannot be split, the engine falls back to
805 python, the one-line sample used to infer column names has no columns,
806 per-chunk dtype inference disagrees in a way that would not match the
807 serial result, or any chunk fails to parse); the caller then reads
808 serially.
809 """
810 warning_sink: list[tuple[str, type[Warning]]] = []
811 try:
812 result = _read_csv_chunks(filepath, kwds, n_workers, warning_sink)
813 except (ParserError, UnicodeDecodeError, OverflowError):
814 # The caller answers these with a serial read of the whole file.
815 raise
816 except Exception:
817 # Nothing re-reads the file after this, so it is here or nowhere. This
818 # over-warns when a serial read would have died before reaching the
819 # warning's column; losing the warning otherwise is the worse trade.
820 _raise_collected(warning_sink)
821 raise
822 if result is not None:
823 _raise_collected(warning_sink)
824 return result
827def _read_csv_chunks(
828 filepath: str,
829 kwds: dict,
830 n_workers: int,
831 warning_sink: list[tuple[str, type[Warning]]],
832) -> DataFrame | None:
833 """
834 Body of :func:`_read_csv_parallel`; see there for what the arguments mean
835 and when the result is ``None``. Appends the ``ParserWarning``\\s its
836 parsers would have raised to *warning_sink* instead of raising them.
837 """
838 # Resolve the effective header value (mirrors TextFileReader.__init__).
839 header = kwds.get("header", "infer")
840 if header == "infer":
841 header = 0 if kwds.get("names") is None else None
843 skiprows = int(kwds.get("skiprows") or 0)
845 # Byte offset at which real data rows begin.
846 data_start = _find_data_start_offset(filepath, header, skiprows)
848 # Oversubscribe the workers so one slow chunk cannot strand a core. Take
849 # the median of sampled line lengths - a single probe lets one atypical
850 # line skew the estimate. The count is raised to follow the file's size
851 # once the column count is known, below.
852 data_size = os.path.getsize(filepath) - data_start
853 line_lens = []
854 with open(filepath, "rb") as fh:
855 for probe in range(9):
856 fh.seek(data_start + data_size * probe // 9)
857 fh.readline(1 << 20) # advance past the partial line at the probe
858 line = fh.readline(1 << 20)
859 if line:
860 line_lens.append(len(line))
861 line_lens.sort()
862 bytes_per_row = line_lens[len(line_lens) // 2] if line_lens else data_size
863 est_rows = data_size // bytes_per_row
864 n_target = min(n_workers * 3, est_rows // _PARALLEL_MIN_CHUNK_ROWS)
865 if n_target < 2:
866 # Too few rows to split, unless each half is big enough to amortise
867 # the per-column costs anyway.
868 if data_size < 2 * _PARALLEL_READ_MIN_BYTES:
869 return None
870 n_target = 2
872 # ------------------------------------------------------------------
873 # Infer column names from the preamble + one data line (very fast).
874 # ------------------------------------------------------------------
875 base_kwds: dict = {
876 **kwds,
877 "compression": None,
878 "memory_map": False,
879 "storage_options": None,
880 }
882 with open(filepath, "rb") as fd:
883 preamble = fd.read(data_start)
884 first_line = fd.readline()
885 # The tokenizer also ends a line at a bare \r, which readline does not, so
886 # data_start could be misplaced without the row count check below noticing.
887 if b"\r" in (preamble + first_line).replace(b"\r\n", b""):
888 return None
890 # Only the column names and the row count are kept, so the sample is parsed
891 # as strings: a value that converts on its own line but not for the whole
892 # column (an int above 2**63 under dtype_backend="pyarrow", say) would
893 # otherwise raise here for a file the serial path reads fine. GH#66259
894 name_buf = io.BytesIO(preamble + first_line)
895 name_kwds = {
896 **base_kwds,
897 "dtype": str,
898 "converters": None,
899 "dtype_backend": lib.no_default,
900 }
901 try:
902 name_reader = TextFileReader(name_buf, **name_kwds)
903 except EmptyDataError:
904 # The one data line we sliced off is blank (e.g. header=None on a file
905 # whose first physical line is empty), so the name-inference read sees
906 # no columns. The serial path skips the blank line and reads the file
907 # fine, so fall back rather than propagate. GH#66259
908 return None
909 if not isinstance(name_reader._engine, CParserWrapper):
910 # TextFileReader fell back to the python engine for a reason the
911 # eligibility checks did not anticipate; load_buffer needs the C engine.
912 name_reader.close()
913 return None
914 name_reader._engine._warning_sink = warning_sink
915 name_reader._engine._reader.warning_sink = warning_sink
916 if name_reader._engine._reader.leading_cols:
917 # Data rows have more fields than the header (implicit index). The
918 # chunk workers would fail on the extra field; bail out up front.
919 name_reader.close()
920 return None
921 assert name_reader._engine.orig_names is not None
922 col_names: list = list(name_reader._engine.orig_names)
923 # name_buf holds the preamble plus exactly one data line. Parsing it must
924 # yield exactly one row; anything else means the preamble's physical line
925 # count disagrees with its logical row count (e.g. a quoted embedded
926 # newline in the header), so data_start is wrong.
927 n_first_rows = len(name_reader.read())
928 name_reader.close()
929 if n_first_rows != 1:
930 return None
932 # A dict ``dtype`` is applied per raw header name: when a name is repeated,
933 # the serial path assigns that dtype to every de-duplicated column (``a``
934 # and ``a.1``), but the workers receive the already-de-duplicated
935 # ``col_names`` and would match only ``a``. ``col_names`` no longer shows
936 # the duplication, so recover the raw header names (parse the header line
937 # as data) and fall back to serial when they collide. GH#64347
938 if header is not None and isinstance(kwds.get("dtype"), dict):
939 header_line = preamble.splitlines(keepends=True)[-1]
940 raw_reader = TextFileReader(
941 io.BytesIO(header_line),
942 **{
943 **base_kwds,
944 "header": None,
945 "names": None,
946 "usecols": None,
947 "index_col": False,
948 "dtype": str,
949 "converters": None,
950 "skiprows": None,
951 "na_filter": False,
952 },
953 )
954 try:
955 raw_names = list(raw_reader.read().iloc[0])
956 finally:
957 raw_reader.close()
958 if len(set(raw_names)) != len(raw_names):
959 return None
961 # n_target so far scales with the worker count, which says nothing about
962 # the file: a 128 MB file gets the same split as one just over the size
963 # gate. Raise it to follow the bytes, bounded by the piece budget and by
964 # the row floor, since a file can be byte-rich and row-poor. Only ever
965 # raised, so no file comes out coarser than before.
966 size_target = min(
967 data_size // _PARALLEL_CHUNK_BYTES,
968 _PARALLEL_MAX_COLUMN_PIECES // max(len(col_names), 1),
969 est_rows // _PARALLEL_MIN_CHUNK_ROWS,
970 )
971 # A count that is not a multiple of the worker count spends its last round
972 # mostly idle: 31 chunks over 6 workers is 5.17 rounds' work that takes 6.
973 # Round up, and only this term, so neither the worker-derived floor nor a
974 # file below one block moves.
975 if size_target > n_workers:
976 size_target = -(-size_target // n_workers) * n_workers
977 n_target = max(n_target, size_target)
979 offsets = _find_chunk_byte_offsets(filepath, n_target, data_start, n_workers)
980 n_chunks = len(offsets) - 1
981 if n_chunks < 2:
982 # e.g. a data section with no interior newlines (one giant line)
983 return None
984 # Spare threads would only spin up to find the queue empty.
985 n_workers = min(n_workers, n_chunks)
987 # ------------------------------------------------------------------
988 # Dispatch all chunks in parallel. Each worker gets a zero-copy
989 # memoryview slice of the mmapped file and calls load_buffer() so
990 # tokenisation is fully GIL-free.
991 # ------------------------------------------------------------------
992 chunk_kwds: dict = {
993 **base_kwds,
994 "header": None,
995 "names": col_names,
996 "skiprows": None,
997 # A worker's byte slice already bounds peak memory, and skipping
998 # low_memory avoids a per-worker GIL-held concatenate.
999 "low_memory": False,
1000 }
1002 # Each thread reuses one parser to drain the queue, so no worker stalls
1003 # the gather on a straggler and no chunk pays parser construction.
1004 chunk_queue: queue.SimpleQueue = queue.SimpleQueue()
1005 for chunk_idx in range(n_chunks):
1006 chunk_queue.put(chunk_idx)
1007 results: list = [None] * n_chunks
1008 chunk_failed = threading.Event()
1009 workers_readers: list = []
1011 def _worker(mv: memoryview) -> None:
1012 reader = TextFileReader(io.BytesIO(b""), **chunk_kwds)
1013 assert isinstance(reader._engine, CParserWrapper)
1014 # Buffers are reused across the queued chunks and freed at close;
1015 # trimming between chunks would stall other workers on reallocs.
1016 reader._engine._reader.trim_after_read = False
1017 # Hand back string columns as raw pending handles: the gather below
1018 # combines each column's chunks into one ExtensionArray, so the
1019 # GIL-held pyarrow wrap happens once per column rather than once per
1020 # chunk in every worker.
1021 reader._engine.wrap_deferred = False
1022 # Row-blocked conversion of numeric columns pays off once enough
1023 # workers contend for memory bandwidth (see TextReader.block_workers).
1024 reader._engine._reader.block_workers = n_workers
1025 reader._engine._warning_sink = warning_sink
1026 reader._engine._reader.warning_sink = warning_sink
1027 workers_readers.append(reader)
1028 while not chunk_failed.is_set():
1029 try:
1030 chunk_idx = chunk_queue.get_nowait()
1031 except queue.Empty:
1032 return
1033 data = mv[offsets[chunk_idx] : offsets[chunk_idx + 1]]
1034 # The serial C parser strips a UTF-8 BOM at byte 0 of the file
1035 # only; strip_bom mirrors that for the chunk starting there.
1036 reader._engine._reader.load_buffer(data, strip_bom=offsets[chunk_idx] == 0)
1037 # the reader holds its own reference for the parse duration
1038 del data
1039 # On a reused parser a zero-row chunk would raise StopIteration
1040 # and close the reader; reset so it returns empty meta instead.
1041 reader._engine._first_chunk = True
1042 try:
1043 # Raw column arrays, not DataFrames: per-chunk frame assembly
1044 # holds the GIL and the concat would redo the block machinery.
1045 _, chunk_columns, col_dict = reader._engine.read()
1046 except Exception:
1047 chunk_failed.set()
1048 return
1049 results[chunk_idx] = (chunk_columns, col_dict)
1051 def _copy_pieces(jobs: list[tuple[np.ndarray, int, np.ndarray]]) -> None:
1052 # ndarray slice assignment releases the GIL for the memcpy, so the
1053 # gather parallelises across the pool.
1054 for dst, pos, piece in jobs:
1055 dst[pos : pos + len(piece)] = piece
1057 def _piece_ndarray(piece) -> np.ndarray | None:
1058 # The ndarray behind pieces whose final block is ndarray-backed:
1059 # plain ndarrays plus tz-naive datetime64/timedelta64.
1060 if isinstance(piece, np.ndarray):
1061 if piece.dtype == object:
1062 # Object pieces take the leftover path below, which runs
1063 # the same constructor inference as a serial read.
1064 return None
1065 return piece
1066 if isinstance(piece, (DatetimeArray, TimedeltaArray)) and isinstance(
1067 piece.dtype, np.dtype
1068 ):
1069 return piece._ndarray
1070 return None
1072 dtype_arg = kwds.get("dtype")
1073 # Mirror TextFileReader.read: a dict or str/object dtype needs a
1074 # Series-per-column wrap, so it skips direct block assembly.
1075 needs_series_wrap = isinstance(dtype_arg, dict) or (
1076 dtype_arg is not None and pandas_dtype(dtype_arg) in (np.str_, np.object_)
1077 )
1079 block_specs: list[tuple] = []
1080 leftover_pos: list[int] = []
1081 col_list: list = []
1082 chunk_dicts: list[dict] = []
1083 columns: list[Hashable] = []
1084 total = 0
1085 readers_closing = False
1087 # mmap the file once and pass zero-copy memoryview slices to each thread
1088 # instead of having each thread open/seek/read its own copy. mmap dups
1089 # the file descriptor, so the file object can be closed right away. Map it
1090 # last, so that every path out of here runs the close below. GH#66259
1091 with open(filepath, "rb") as fh:
1092 mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
1093 try:
1094 with (
1095 contextlib.ExitStack() as views,
1096 ThreadPoolExecutor(max_workers=n_workers) as pool,
1097 ):
1098 # One view per worker: slicing a shared memoryview from several
1099 # threads races on free-threaded builds (GH#71418).
1100 worker_views = [
1101 views.enter_context(memoryview(mm)) for _ in range(n_workers)
1102 ]
1103 for fut in [pool.submit(_worker, mv) for mv in worker_views]:
1104 fut.result()
1105 if chunk_failed.is_set():
1106 # Let the serial read raise: a chunk's error can be an artifact
1107 # of the split (a boundary inside a quoted newline), and even a
1108 # genuine one differs from serial in which error comes first
1109 # and in the row positions it reports.
1110 return None
1112 # A column of only NA tokens and ints too large for int64 converts
1113 # to no numeric dtype, and is then emitted with its NA tokens left
1114 # as literal strings (GH#14983). Which chunks hit that depends on
1115 # how the file was split, and the resulting dtype is str either
1116 # way, so the reconciliation below cannot see the difference -- the
1117 # values would just silently disagree with a serial read. GH#66259
1118 if any(
1119 reader._engine._reader.na_left_literal for reader in workers_readers
1120 ):
1121 return None
1123 # Freeing parser buffers costs a few ms of madvise; run it on the
1124 # pool, overlapped with the gather copies below.
1125 close_futures = [pool.submit(reader.close) for reader in workers_readers]
1126 readers_closing = True
1128 # Zero-row chunks carry empty-meta object dtypes that would
1129 # needlessly trip the dtype reconciliation below.
1130 chunk_results = [
1131 res
1132 for res in cast("list[tuple]", results)
1133 if res[1] and len(next(iter(res[1].values()))) > 0
1134 ]
1135 if not chunk_results:
1136 # every chunk parsed to zero rows (e.g. all blank lines);
1137 # let the serial path build the empty frame
1138 return None
1139 columns = list(chunk_results[0][0])
1140 chunk_dicts = [col_dict for _, col_dict in chunk_results]
1141 col_list = list(chunk_dicts[0])
1142 if col_list:
1143 total = sum(len(chunk_dict[col_list[0]]) for chunk_dict in chunk_dicts)
1145 # Per-chunk dtype inference can disagree with whole-file inference
1146 # (a lone non-numeric row makes only its own chunk object). Mixed
1147 # signed-int/float chunks gather to the float64 serial would give;
1148 # anything else must be re-read serially. ArrowDtype is excluded
1149 # because pyarrow widens int to double with a *checked* cast, which
1150 # raises above 2**53 where the serial whole-file read just infers
1151 # double. GH#66259
1152 for name in col_list:
1153 chunk_dtypes = {chunk_dict[name].dtype for chunk_dict in chunk_dicts}
1154 if len(chunk_dtypes) > 1 and not all(
1155 dt.kind in "if" and not isinstance(dt, ArrowDtype)
1156 for dt in chunk_dtypes
1157 ):
1158 return None
1160 if not needs_series_wrap:
1161 # Gather same-dtype ndarray columns straight into one
1162 # preallocated consolidated block per dtype; anything else
1163 # falls to _concatenate_chunks below.
1164 groups: dict[np.dtype, list[int]] = {}
1165 col_nd_pieces: dict[int, list[np.ndarray]] = {}
1166 for pos, name in enumerate(col_list):
1167 nd_pieces = [
1168 _piece_ndarray(chunk_dict[name]) for chunk_dict in chunk_dicts
1169 ]
1170 first = nd_pieces[0]
1171 if first is not None and all(
1172 piece is not None and piece.dtype == first.dtype
1173 for piece in nd_pieces
1174 ):
1175 groups.setdefault(first.dtype, []).append(pos)
1176 col_nd_pieces[pos] = cast("list[np.ndarray]", nd_pieces)
1177 else:
1178 leftover_pos.append(pos)
1180 copy_jobs: list[tuple[np.ndarray, int, np.ndarray]] = []
1181 for group_dtype, positions in groups.items():
1182 values2d = np.empty((len(positions), total), dtype=group_dtype)
1183 for row, pos in enumerate(positions):
1184 offset = 0
1185 for piece in col_nd_pieces[pos]:
1186 copy_jobs.append((values2d[row], offset, piece))
1187 offset += len(piece)
1188 block_specs.append((values2d, np.array(positions, dtype=np.intp)))
1190 # Batch the copies instead of submitting one task per
1191 # (column, chunk): on a wide frame that is n_columns * n_chunks
1192 # tasks, each moving a sliver of one column, and the dispatch
1193 # then costs many times the memcpy it schedules.
1194 batch = -(-len(copy_jobs) // (n_workers * 4)) or 1
1195 copy_futures = [
1196 pool.submit(_copy_pieces, copy_jobs[start : start + batch])
1197 for start in range(0, len(copy_jobs), batch)
1198 ]
1199 for fut in copy_futures:
1200 fut.result()
1201 for fut in close_futures:
1202 fut.result()
1203 finally:
1204 if not readers_closing:
1205 for reader in workers_readers:
1206 try:
1207 reader.close()
1208 except Exception:
1209 pass
1210 # The closes above drop the chunk slices the readers hold, so the
1211 # mapping is normally unmapped right here. suppress: a traceback that
1212 # still pins a slice would make close() raise BufferError and mask the
1213 # exception on its way out; the mapping then goes when it is collected.
1214 with contextlib.suppress(BufferError):
1215 mm.close()
1217 index = RangeIndex(total)
1219 if needs_series_wrap:
1220 data = _concatenate_chunks(chunk_dicts, columns, warn_mixed=False)
1221 if isinstance(dtype_arg, dict):
1222 dtype: defaultdict = defaultdict(lambda: None)
1223 dtype.update(dtype_arg)
1224 else:
1225 dtype = defaultdict(lambda: dtype_arg)
1226 new_col_dict = {
1227 k: Series(
1228 v,
1229 index=index,
1230 dtype=(
1231 dtype[k]
1232 if pandas_dtype(dtype[k]) in (np.str_, np.object_)
1233 else None
1234 ),
1235 copy=False,
1236 )
1237 for k, v in data.items()
1238 }
1239 return DataFrame(new_col_dict, columns=columns, index=index, copy=False)
1241 if leftover_pos:
1242 leftover_names = [col_list[pos] for pos in leftover_pos]
1243 leftover_dicts = [
1244 {name: chunk_dict[name] for name in leftover_names}
1245 for chunk_dict in chunk_dicts
1246 ]
1247 gathered = _concatenate_chunks(leftover_dicts, leftover_names, warn_mixed=False)
1248 for pos, name in zip(leftover_pos, leftover_names, strict=True):
1249 values = gathered[name]
1250 if isinstance(values, np.ndarray) and values.dtype == object:
1251 # Match the serial constructor's inference for object
1252 # columns (object -> str dtype under future.infer_string).
1253 values = Series(values, index=index, copy=False)._values
1254 if isinstance(values, np.ndarray):
1255 # _make_block expects numpy values already in 2D block shape
1256 values = values.reshape(1, -1)
1257 block_specs.append((values, np.array([pos], dtype=np.intp)))
1259 block_objs = [_make_block(values, placement) for values, placement in block_specs]
1260 mgr = BlockManager(block_objs, [ensure_index(columns), index])
1261 return DataFrame._from_mgr(mgr, mgr.axes)
1264@overload
1265def read_csv(
1266 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1267 *,
1268 iterator: Literal[True],
1269 chunksize: int | None = ...,
1270 **kwds: Unpack[_read_shared[HashableT]],
1271) -> TextFileReader: ...
1274@overload
1275def read_csv(
1276 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1277 *,
1278 iterator: bool = ...,
1279 chunksize: int,
1280 **kwds: Unpack[_read_shared[HashableT]],
1281) -> TextFileReader: ...
1284@overload
1285def read_csv(
1286 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1287 *,
1288 iterator: Literal[False] = ...,
1289 chunksize: None = ...,
1290 **kwds: Unpack[_read_shared[HashableT]],
1291) -> DataFrame: ...
1294@overload
1295def read_csv(
1296 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1297 *,
1298 iterator: bool = ...,
1299 chunksize: int | None = ...,
1300 **kwds: Unpack[_read_shared[HashableT]],
1301) -> DataFrame | TextFileReader: ...
1304@set_module("pandas")
1305def read_csv(
1306 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1307 *,
1308 sep: str | lib.NoDefault | None = lib.no_default,
1309 delimiter: str | lib.NoDefault | None = lib.no_default,
1310 # Column and Index Locations and Names
1311 header: int | Sequence[int] | Literal["infer"] | None = "infer",
1312 names: Sequence[Hashable] | lib.NoDefault | None = lib.no_default,
1313 index_col: IndexLabel | Literal[False] | None = None,
1314 usecols: UsecolsArgType = None,
1315 # General Parsing Configuration
1316 dtype: DtypeArg | None = None,
1317 engine: CSVEngine | None = None,
1318 converters: Mapping[HashableT, Callable] | None = None,
1319 true_values: list | None = None,
1320 false_values: list | None = None,
1321 skipinitialspace: bool = False,
1322 skiprows: list[int] | int | Callable[[Hashable], bool] | None = None,
1323 skipfooter: int = 0,
1324 nrows: int | None = None,
1325 # NA and Missing Data Handling
1326 na_values: (
1327 Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None
1328 ) = None,
1329 keep_default_na: bool = True,
1330 na_filter: bool = True,
1331 skip_blank_lines: bool = True,
1332 # Datetime Handling
1333 parse_dates: bool | Sequence[Hashable] | None = None,
1334 date_format: str | dict[Hashable, str] | None = None,
1335 dayfirst: bool = False,
1336 cache_dates: bool = True,
1337 # Iteration
1338 iterator: bool = False,
1339 chunksize: int | None = None,
1340 # Quoting, Compression, and File Format
1341 compression: CompressionOptions = "infer",
1342 thousands: str | None = None,
1343 decimal: str = ".",
1344 lineterminator: str | None = None,
1345 quotechar: str = '"',
1346 quoting: int = csv.QUOTE_MINIMAL,
1347 doublequote: bool = True,
1348 escapechar: str | None = None,
1349 comment: str | None = None,
1350 encoding: str | None = None,
1351 encoding_errors: str | None = "strict",
1352 dialect: str | csv.Dialect | type[csv.Dialect] | None = None,
1353 # Error Handling
1354 on_bad_lines: str = "error",
1355 # Internal
1356 low_memory: bool = _c_parser_defaults["low_memory"],
1357 memory_map: bool = False,
1358 float_precision: Literal["high", "legacy", "round_trip"] | None = None,
1359 storage_options: StorageOptions | None = None,
1360 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1361) -> DataFrame | TextFileReader:
1362 """
1363 Read a comma-separated values (csv) file into DataFrame.
1365 Also supports optionally iterating or breaking of the file
1366 into chunks.
1368 Additional help can be found in the online docs for
1369 `IO Tools <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html>`_.
1371 Parameters
1372 ----------
1373 filepath_or_buffer : str, path object or file-like object
1374 Any valid string path is acceptable. The string could be a URL. Valid
1375 URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is
1376 expected. A local file could be: file://localhost/path/to/table.csv.
1378 Certain URL schemes may require additional packages. For example, S3
1379 URLs require the ``s3fs`` library. See
1380 :ref:`install.optional_dependencies` for a full list.
1382 If you want to pass in a path object, pandas accepts any ``os.PathLike``.
1384 By file-like object, we refer to objects with a ``read()`` method that
1385 accepts an optional size argument, such as a file handle (e.g. via
1386 builtin ``open`` function) or ``StringIO``.
1387 sep : str, default ','
1388 Character or regex pattern to treat as the delimiter. ``sep=None`` detects
1389 the separator from the first valid row of the file with Python's builtin
1390 sniffer tool, ``csv.Sniffer``; it is supported only by the Python parsing
1391 engine, which will be used automatically.
1392 In addition, separators longer than 1 character and different from
1393 ``'\\s+'`` will be interpreted as regular expressions and will force
1394 the use of the Python parsing engine. Note that regex delimiters are prone
1395 to ignoring quoted data. Regex example: ``'\\r\\t'``.
1396 delimiter : str, optional
1397 Alias for ``sep``.
1398 header : int, Sequence of int, 'infer' or None, default 'infer'
1399 Row index or indices in the file to use as DataFrame column labels.
1400 Indexing starts at 0 and counts only non-blank, non-commented lines
1401 when ``skip_blank_lines=True``, so ``header=0`` denotes the first
1402 line of data rather than the first line of the file. Valid arguments
1403 are:
1405 * ``int``: line index at which column labels are read.
1406 * Sequence of ``int``: line indices at which column labels are read
1407 and combined into a :class:`~pandas.MultiIndex` on the columns,
1408 e.g. ``[0, 1, 3]``. Intervening rows that are not specified will
1409 be skipped (e.g. row 2 in this example).
1410 * ``None``: no row in the file is interpreted as column labels.
1411 Columns are labelled by integer position, or by the values passed
1412 to ``names`` when provided. Use this for files with no header
1413 row. If the file has a header row that should be overridden by
1414 ``names``, pass ``header=0`` instead.
1415 * ``'infer'`` (default): equivalent to ``header=0`` when no
1416 ``names`` are passed, otherwise equivalent to ``header=None``.
1418 When inferred from the file contents, headers are kept distinct from
1419 each other by renaming duplicate names with a numeric suffix of the form
1420 ``".{count}"`` starting from 1, e.g. ``"foo"`` and ``"foo.1"``.
1421 Empty headers are named ``"Unnamed: {i}"`` or ``
1422 "Unnamed: {i}_level_{level}"``
1423 in the case of MultiIndex columns.
1424 names : Sequence of Hashable, optional
1425 Sequence of column labels to apply. If the file contains a header row,
1426 then you should explicitly pass ``header=0`` to override the column names.
1427 Duplicates in this list are not allowed.
1428 index_col : Hashable, Sequence of Hashable or False, optional
1429 Column(s) to use as row label(s), denoted either by column labels or column
1430 indices. If a sequence of labels or indices is given,
1431 :class:`~pandas.MultiIndex`
1432 will be formed for the row labels.
1434 Note: ``index_col=False`` can be used to force pandas to *not* use the first
1435 column as the index, e.g., when you have a malformed file with delimiters at
1436 the end of each line.
1437 usecols : Sequence of Hashable or Callable, optional
1438 Subset of columns to select, denoted either
1439 by column labels or column indices.
1440 If list-like, all elements must either
1441 be positional (i.e. integer indices into the document columns) or strings
1442 that correspond to column names provided either by the user in ``names`` or
1443 inferred from the document header row(s).
1444 If ``names`` are given, the document
1445 header row(s) are not taken into account. For example, a valid list-like
1446 ``usecols`` parameter would be ``[0, 1, 2]`` or ``['foo', 'bar', 'baz']``.
1447 Element order is ignored, so ``usecols=[0, 1]`` is the same as ``[1, 0]``.
1448 To instantiate a :class:`~pandas.DataFrame` from ``data`` with element order
1449 preserved use ``pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]``
1450 for columns in ``['foo', 'bar']`` order or
1451 ``pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]``
1452 for ``['bar', 'foo']`` order.
1454 If callable, the callable function will be evaluated against the column
1455 names, returning names where the callable function evaluates to ``True``. An
1456 example of a valid callable argument would be ``lambda x: x.upper() in
1457 ['AAA', 'BBB', 'DDD']``. Using this parameter results in much faster
1458 parsing time and lower memory usage.
1459 dtype : dtype or dict of {Hashable : dtype}, optional
1460 Data type(s) to apply to either the whole dataset or individual columns.
1461 E.g., ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}``
1462 Use ``str`` or ``object`` together with suitable ``na_values`` settings
1463 to preserve and not interpret ``dtype``.
1464 Specifying a ``dtype`` does not change how missing values are parsed;
1465 values recognized as missing (see ``na_values`` and ``keep_default_na``)
1466 are still read as ``NaN`` (or the dtype's NA value) rather than cast to
1467 the requested ``dtype``.
1468 If ``converters`` are specified, they will be applied INSTEAD
1469 of ``dtype`` conversion. Specify a ``defaultdict`` as input where
1470 the default determines the ``dtype``
1471 of the columns which are not explicitly
1472 listed.
1473 engine : {'c', 'python', 'pyarrow'}, optional
1474 Parser engine to use. The C and pyarrow engines are faster,
1475 while the python engine
1476 is currently more feature-complete. The pyarrow engine is
1477 multithreaded, and the C engine reads sufficiently large files in
1478 parallel. Some features of the "pyarrow" engine
1479 are unsupported or may not work correctly.
1480 converters : dict of {Hashable : Callable}, optional
1481 Functions for converting values in specified columns. Keys can either
1482 be column labels or column indices. The function is applied to the raw
1483 text read from the file, so an empty field is passed as an empty string
1484 ``''``; ``na_values`` and ``keep_default_na`` are then applied to the
1485 value the function returns.
1486 true_values : list, optional
1487 Values to consider as ``True`` in addition
1488 to case-insensitive variants of 'True'.
1489 false_values : list, optional
1490 Values to consider as ``False`` in addition to case-insensitive
1491 variants of 'False'.
1492 skipinitialspace : bool, default False
1493 Skip spaces after delimiter.
1494 skiprows : int, list of int or Callable, optional
1495 Line numbers to skip (0-indexed) or number of lines to skip (``int``)
1496 at the start of the file.
1498 If callable, the callable function will be evaluated against the row
1499 indices, returning ``True`` if the row should be skipped and ``False``
1500 otherwise.
1501 An example of a valid callable argument would be ``lambda x: x in [0, 2]``.
1502 skipfooter : int, default 0
1503 Number of lines at bottom of file to skip (Unsupported with ``engine='c'``).
1504 nrows : int, optional
1505 Number of rows of file to read. Useful for reading pieces of large files.
1506 Refers to the number of data rows in the returned DataFrame, excluding:
1508 * The header row containing column names.
1509 * Rows before the header row, if ``header=1`` or larger.
1511 Example usage:
1513 * To read the first 999,999 (non-header) rows:
1514 ``read_csv(..., nrows=999999)``
1516 * To read rows 1,000,000 through 1,999,999:
1517 ``read_csv(..., skiprows=1000000, nrows=999999)``
1519 na_values : Hashable, Iterable of Hashable or dict of {Hashable : Iterable},
1520 optional
1521 Additional strings to recognize as ``NA``/``NaN``. If ``dict``
1522 passed, specific
1523 per-column ``NA`` values. By default the following values
1524 are interpreted as
1525 ``NaN``: empty string, "NaN", "N/A", "NULL", and other common
1526 representations of missing data.
1527 keep_default_na : bool, default True
1528 Whether or not to include the default ``NaN`` values when parsing the data.
1529 Depending on whether ``na_values`` is passed in, the behavior is as follows:
1531 * If ``keep_default_na`` is ``True``, and ``na_values``
1532 are specified, ``na_values``
1533 is appended to the default ``NaN`` values used for parsing.
1534 * If ``keep_default_na`` is ``True``, and ``na_values`` are not specified, only
1535 the default ``NaN`` values are used for parsing.
1536 * If ``keep_default_na`` is ``False``, and ``na_values`` are specified, only
1537 the ``NaN`` values specified ``na_values`` are used for parsing.
1538 * If ``keep_default_na`` is ``False``, and ``na_values`` are not specified, no
1539 strings will be parsed as ``NaN``.
1541 Note that if ``na_filter`` is passed in as ``False``,
1542 the ``keep_default_na`` and
1543 ``na_values`` parameters will be ignored.
1544 na_filter : bool, default True
1545 Detect missing value markers (empty strings and the value of ``na_values``). In
1546 data without any ``NA`` values, passing ``na_filter=False`` can improve the
1547 performance of reading a large file.
1548 skip_blank_lines : bool, default True
1549 If ``True``, skip over blank lines rather than interpreting as ``NaN`` values.
1550 parse_dates : bool, None, list of Hashable, default None
1551 The behavior is as follows:
1553 * ``bool``. If ``True`` -> try parsing the index.
1554 * ``None``. Behaves like ``True`` if ``date_format`` is specified.
1555 * ``list`` of ``int`` or names.
1556 e.g. If ``[1, 2, 3]`` -> try parsing columns 1, 2, 3
1557 each as a separate date column.
1559 If a column or index cannot be represented as an array of ``datetime``,
1560 say because of an unparsable value or a mixture of timezones, the column
1561 or index will be returned unaltered as an ``object`` data type. For
1562 non-standard ``datetime`` parsing, use :func:`~pandas.to_datetime` after
1563 :func:`~pandas.read_csv`.
1565 Note: A fast-path exists for iso8601-formatted dates.
1566 date_format : str or dict of column -> format, optional
1567 Format to use for parsing dates and/or times when
1568 used in conjunction with ``parse_dates``.
1569 The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See
1570 `strftime documentation
1571 <https://docs.python.org/3/library/datetime.html
1572 #strftime-and-strptime-behavior>`_ for more information on choices, though
1573 note that :const:`"%f"`` will parse all the way up to nanoseconds.
1574 You can also pass:
1576 - "ISO8601", to parse any `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_
1577 time string (not necessarily in exactly the same format);
1578 - "mixed", to infer the format for each element individually. This is risky,
1579 and you should probably use it along with `dayfirst`.
1581 .. versionadded:: 2.0.0
1582 dayfirst : bool, default False
1583 DD/MM format dates, international and European format.
1584 cache_dates : bool, default True
1585 If ``True``, use a cache of unique, converted dates to apply the ``datetime``
1586 conversion. May produce significant speed-up when parsing duplicate
1587 date strings, especially ones with timezone offsets.
1589 iterator : bool, default False
1590 Return ``TextFileReader`` object for iteration or getting chunks with
1591 ``get_chunk()``.
1592 chunksize : int, optional
1593 Number of lines to read from the file per chunk. Passing a value will cause the
1594 function to return a ``TextFileReader`` object for iteration.
1595 See the `IO Tools docs
1596 <https://pandas.pydata.org/docs/user_guide/io.html#io-chunking>`_
1597 for more information on ``iterator`` and ``chunksize``.
1599 compression : str or dict, default 'infer'
1600 For on-the-fly decompression of on-disk data.
1601 If 'infer' and 'filepath_or_buffer' is
1602 path-like, then detect compression from the following extensions: '.gz',
1603 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
1604 (otherwise no compression).
1605 If using 'zip' or 'tar', the ZIP file must contain only
1606 one data file to be read in.
1607 Set to ``None`` for no decompression.
1608 Can also be a dict with key ``'method'`` set
1609 to one of {``'zip'``, ``'gzip'``, ``'bz2'``,
1610 ``'zstd'``, ``'xz'``, ``'tar'``} and
1611 other key-value pairs are forwarded to
1612 ``zipfile.ZipFile``, ``gzip.GzipFile``,
1613 ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
1614 ``tarfile.TarFile``, respectively.
1615 As an example, the following could be passed for
1616 Zstandard decompression using a
1617 custom compression dictionary:
1618 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
1620 thousands : str (length 1), optional
1621 Character acting as the thousands separator in numerical values.
1622 decimal : str (length 1), default '.'
1623 Character to recognize as decimal point (e.g., use ',' for European data).
1624 lineterminator : str (length 1), optional
1625 Character used to denote a line break. Only valid with C parser.
1626 quotechar : str (length 1), optional
1627 Character used to denote the start and end of a quoted item. Quoted
1628 items can include the ``delimiter`` and it will be ignored.
1629 quoting : {0, 1, 2, 3}, default csv.QUOTE_MINIMAL
1630 Control field quoting behavior per ``csv.QUOTE_*`` constants. Use one of
1631 ``0`` or ``csv.QUOTE_MINIMAL``, ``1`` or ``csv.QUOTE_ALL``,
1632 ``2`` or ``csv.QUOTE_NONNUMERIC``, or ``3`` or ``csv.QUOTE_NONE``.
1633 Default is ``csv.QUOTE_MINIMAL`` (i.e., 0) which implies that
1634 only fields containing special
1635 characters are quoted (e.g., characters defined
1636 in ``quotechar``, ``delimiter``,
1637 or ``lineterminator``.
1638 doublequote : bool, default True
1639 When ``quotechar`` is specified and ``quoting`` is not ``QUOTE_NONE``, indicate
1640 whether or not to interpret two consecutive ``quotechar`` elements INSIDE a
1641 field as a single ``quotechar`` element.
1642 escapechar : str (length 1), optional
1643 Character used to escape other characters.
1644 comment : str (length 1), optional
1645 Character indicating that the remainder of line should not be parsed.
1646 If found at the beginning
1647 of a line, the line will be ignored altogether. This parameter must be a
1648 single character. Like empty lines (as long as ``skip_blank_lines=True``),
1649 fully commented lines are ignored by the parameter ``header`` but not by
1650 ``skiprows``. For example, if ``comment='#'``, parsing
1651 ``#empty\\na,b,c\\n1,2,3`` with ``header=0`` will result in ``'a,b,c'`` being
1652 treated as the header.
1653 encoding : str, optional, default 'utf-8'
1654 Encoding to use for UTF when reading/writing (ex. ``'utf-8'``). `List of Python
1655 standard encodings
1656 <https://docs.python.org/3/library/codecs.html#standard-encodings>`_ .
1658 encoding_errors : str, optional, default 'strict'
1659 How encoding errors are treated. `List of possible values
1660 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
1662 dialect : str or csv.Dialect, optional
1663 If provided, this parameter will override values (default or not) for the
1664 following parameters: ``delimiter``, ``doublequote``, ``escapechar``,
1665 ``skipinitialspace``, ``quotechar``, and ``quoting``. If it is necessary to
1666 override values, a ``ParserWarning`` will be issued. See ``csv.Dialect``
1667 documentation for more details.
1668 on_bad_lines : {'error', 'warn', 'skip'} or Callable, default 'error'
1669 Specifies what to do upon encountering a bad line (a line with too many fields).
1670 Allowed values are:
1672 - ``'error'``, raise an Exception when a bad line is encountered.
1673 - ``'warn'``, raise a warning when a bad line is
1674 encountered and skip that line.
1675 - ``'skip'``, skip bad lines without raising or warning when
1676 they are encountered.
1677 - Callable, function that will process a single bad line.
1678 - With ``engine='python'``, function with signature
1679 ``(bad_line: list[str]) -> list[str] | None``.
1680 ``bad_line`` is a list of strings split by the ``sep``.
1681 If the function returns ``None``, the bad line will be ignored.
1682 If the function returns a new ``list`` of strings with
1683 more elements than
1684 expected, a ``ParserWarning`` will be emitted while
1685 dropping extra elements.
1686 - With ``engine='pyarrow'``, function with signature
1687 as described in pyarrow documentation: `invalid_row_handler
1688 <https://arrow.apache.org/docs/python
1689 /generated/pyarrow.csv.ParseOptions.html
1690 #pyarrow.csv.ParseOptions.invalid_row_handler>`_.
1692 .. versionchanged:: 2.2.0
1694 Callable for ``engine='pyarrow'``
1696 low_memory : bool, default True
1697 Internally process the file in chunks, resulting in lower memory use
1698 while parsing, but possibly mixed type inference. Because each chunk
1699 is type-inferred independently, the same literal can be parsed as an
1700 ``int`` in one chunk and a ``str`` in another; after concatenation the
1701 column has ``object`` dtype and contains both representations, so
1702 comparisons such as ``df["col"] == 12345`` or
1703 ``df["col"] == "12345"`` will match only a subset of the rows holding
1704 that value. A :class:`~pandas.errors.DtypeWarning` is emitted when
1705 this occurs. To ensure no mixed types either set ``False``, or specify
1706 the type with the ``dtype`` parameter.
1707 Note that the entire file is read into a single :class:`~pandas.DataFrame`
1708 regardless, use the ``chunksize`` or ``iterator``
1709 parameter to return the data in
1710 chunks. (Only valid with C parser).
1711 memory_map : bool, default False
1712 If a filepath is provided for ``filepath_or_buffer``, map the file object
1713 directly onto memory and access the data directly from there. Using this
1714 option can improve performance because there is no longer any I/O overhead.
1715 float_precision : {'high', 'legacy', 'round_trip'}, optional
1716 Specifies which converter the C engine should use for floating-point
1717 values. The options are ``None`` or ``'high'`` for the ordinary converter,
1718 ``'legacy'`` for the original lower precision pandas converter, and
1719 ``'round_trip'`` for the round-trip converter.
1721 .. deprecated:: 3.1.0
1722 All float precision modes now use the same converter.
1723 The ``float_precision`` argument will be removed in a future version.
1725 storage_options : dict, optional
1726 Extra options that make sense for a particular storage connection, e.g.
1727 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
1728 are forwarded to ``urllib.request.Request`` as header options. For other
1729 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
1730 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
1731 details, and for more examples on storage options refer `here
1732 <https://pandas.pydata.org/docs/user_guide/io.html?
1733 highlight=storage_options#reading-writing-remote-files>`_.
1735 dtype_backend : {'numpy_nullable', 'pyarrow'}
1736 Back-end data type applied to the resultant :class:`DataFrame`
1737 (still experimental). If not specified, the default behavior
1738 is to not use nullable data types. If specified, the behavior
1739 is as follows:
1741 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
1742 * ``"pyarrow"``: returns
1743 pyarrow-backed nullable :class:`ArrowDtype` :class:`DataFrame`
1745 .. versionadded:: 2.0
1747 Returns
1748 -------
1749 DataFrame or TextFileReader
1750 A comma-separated values (csv) file is returned as two-dimensional
1751 data structure with labeled axes.
1753 See Also
1754 --------
1755 DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
1756 read_table : Read general delimited file into DataFrame.
1757 read_fwf : Read a table of fixed-width formatted lines into DataFrame.
1759 Notes
1760 -----
1761 Column labels read from a header row are always strings. To use column
1762 labels of another type, pass them explicitly with ``names`` and set
1763 ``header=0`` when the file contains a header row.
1765 Sufficiently large local uncompressed files read with the C engine may be
1766 parsed by multiple threads in parallel. Use the ``mode.max_threads``
1767 option to cap or disable this; see :ref:`io.csv.parallel` for details.
1769 Examples
1770 --------
1771 >>> pd.read_csv("data.csv") # doctest: +SKIP
1772 Name Value
1773 0 foo 1
1774 1 bar 2
1775 2 #baz 3
1777 Index and header can be specified via the `index_col` and `header` arguments.
1779 >>> pd.read_csv("data.csv", header=None) # doctest: +SKIP
1780 0 1
1781 0 Name Value
1782 1 foo 1
1783 2 bar 2
1784 3 #baz 3
1786 >>> pd.read_csv("data.csv", index_col="Value") # doctest: +SKIP
1787 Name
1788 Value
1789 1 foo
1790 2 bar
1791 3 #baz
1793 Column types are inferred but can be explicitly specified using the dtype argument.
1795 >>> pd.read_csv("data.csv", dtype={"Value": float}) # doctest: +SKIP
1796 Name Value
1797 0 foo 1.0
1798 1 bar 2.0
1799 2 #baz 3.0
1801 True, False, and NA values, and thousands separators have defaults,
1802 but can be explicitly specified, too. Supply the values you would like
1803 as strings or lists of strings!
1805 >>> pd.read_csv("data.csv", na_values=["foo", "bar"]) # doctest: +SKIP
1806 Name Value
1807 0 NaN 1
1808 1 NaN 2
1809 2 #baz 3
1811 Comment lines in the input file can be skipped using the `comment` argument.
1813 >>> pd.read_csv("data.csv", comment="#") # doctest: +SKIP
1814 Name Value
1815 0 foo 1
1816 1 bar 2
1818 By default, columns with dates will be read as ``object`` rather than ``datetime``.
1820 >>> df = pd.read_csv("tmp.csv") # doctest: +SKIP
1822 >>> df # doctest: +SKIP
1823 col 1 col 2 col 3
1824 0 10 10/04/2018 Sun 15 Jan 2023
1825 1 20 15/04/2018 Fri 12 May 2023
1827 >>> df.dtypes # doctest: +SKIP
1828 col 1 int64
1829 col 2 object
1830 col 3 object
1831 dtype: object
1833 Specific columns can be parsed as dates by using the `parse_dates` and
1834 `date_format` arguments.
1836 >>> df = pd.read_csv(
1837 ... "tmp.csv",
1838 ... parse_dates=[1, 2],
1839 ... date_format={"col 2": "%d/%m/%Y", "col 3": "%a %d %b %Y"},
1840 ... ) # doctest: +SKIP
1842 >>> df.dtypes # doctest: +SKIP
1843 col 1 int64
1844 col 2 datetime64[ns]
1845 col 3 datetime64[ns]
1846 dtype: object
1847 """
1848 # locals() should never be modified
1849 kwds = locals().copy()
1850 del kwds["filepath_or_buffer"]
1851 del kwds["sep"]
1853 kwds_defaults = _refine_defaults_read(
1854 dialect,
1855 delimiter,
1856 engine,
1857 sep,
1858 on_bad_lines,
1859 names,
1860 defaults={"delimiter": ","},
1861 dtype_backend=dtype_backend,
1862 lineterminator=lineterminator,
1863 )
1864 kwds.update(kwds_defaults)
1866 return _read(filepath_or_buffer, kwds)
1869@overload
1870def read_table(
1871 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1872 *,
1873 iterator: Literal[True],
1874 chunksize: int | None = ...,
1875 **kwds: Unpack[_read_shared[HashableT]],
1876) -> TextFileReader: ...
1879@overload
1880def read_table(
1881 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1882 *,
1883 iterator: bool = ...,
1884 chunksize: int,
1885 **kwds: Unpack[_read_shared[HashableT]],
1886) -> TextFileReader: ...
1889@overload
1890def read_table(
1891 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1892 *,
1893 iterator: Literal[False] = ...,
1894 chunksize: None = ...,
1895 **kwds: Unpack[_read_shared[HashableT]],
1896) -> DataFrame: ...
1899@overload
1900def read_table(
1901 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1902 *,
1903 iterator: bool = ...,
1904 chunksize: int | None = ...,
1905 **kwds: Unpack[_read_shared[HashableT]],
1906) -> DataFrame | TextFileReader: ...
1909@set_module("pandas")
1910def read_table(
1911 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
1912 *,
1913 sep: str | lib.NoDefault | None = lib.no_default,
1914 delimiter: str | lib.NoDefault | None = lib.no_default,
1915 # Column and Index Locations and Names
1916 header: int | Sequence[int] | Literal["infer"] | None = "infer",
1917 names: Sequence[Hashable] | lib.NoDefault | None = lib.no_default,
1918 index_col: IndexLabel | Literal[False] | None = None,
1919 usecols: UsecolsArgType = None,
1920 # General Parsing Configuration
1921 dtype: DtypeArg | None = None,
1922 engine: CSVEngine | None = None,
1923 converters: Mapping[HashableT, Callable] | None = None,
1924 true_values: list | None = None,
1925 false_values: list | None = None,
1926 skipinitialspace: bool = False,
1927 skiprows: list[int] | int | Callable[[Hashable], bool] | None = None,
1928 skipfooter: int = 0,
1929 nrows: int | None = None,
1930 # NA and Missing Data Handling
1931 na_values: (
1932 Hashable | Iterable[Hashable] | Mapping[Hashable, Iterable[Hashable]] | None
1933 ) = None,
1934 keep_default_na: bool = True,
1935 na_filter: bool = True,
1936 skip_blank_lines: bool = True,
1937 # Datetime Handling
1938 parse_dates: bool | Sequence[Hashable] | None = None,
1939 date_format: str | dict[Hashable, str] | None = None,
1940 dayfirst: bool = False,
1941 cache_dates: bool = True,
1942 # Iteration
1943 iterator: bool = False,
1944 chunksize: int | None = None,
1945 # Quoting, Compression, and File Format
1946 compression: CompressionOptions = "infer",
1947 thousands: str | None = None,
1948 decimal: str = ".",
1949 lineterminator: str | None = None,
1950 quotechar: str = '"',
1951 quoting: int = csv.QUOTE_MINIMAL,
1952 doublequote: bool = True,
1953 escapechar: str | None = None,
1954 comment: str | None = None,
1955 encoding: str | None = None,
1956 encoding_errors: str | None = "strict",
1957 dialect: str | csv.Dialect | type[csv.Dialect] | None = None,
1958 # Error Handling
1959 on_bad_lines: str = "error",
1960 # Internal
1961 low_memory: bool = _c_parser_defaults["low_memory"],
1962 memory_map: bool = False,
1963 float_precision: Literal["high", "legacy", "round_trip"] | None = None,
1964 storage_options: StorageOptions | None = None,
1965 dtype_backend: DtypeBackend | lib.NoDefault = lib.no_default,
1966) -> DataFrame | TextFileReader:
1967 """
1968 Read general delimited file into DataFrame.
1970 Also supports optionally iterating or breaking of the file
1971 into chunks.
1973 Additional help can be found in the online docs for
1974 `IO Tools <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html>`_.
1976 Parameters
1977 ----------
1978 filepath_or_buffer : str, path object or file-like object
1979 Any valid string path is acceptable. The string could be a URL. Valid
1980 URL schemes include http, ftp, s3, gs, and file. For file URLs, a host is
1981 expected. A local file could be: file://localhost/path/to/table.csv.
1983 Certain URL schemes may require additional packages. For example, S3
1984 URLs require the ``s3fs`` library. See
1985 :ref:`install.optional_dependencies` for a full list.
1987 If you want to pass in a path object, pandas accepts any ``os.PathLike``.
1989 By file-like object, we refer to objects with a ``read()`` method that
1990 accepts an optional size argument, such as a file handle (e.g. via
1991 builtin ``open`` function) or ``StringIO``.
1992 sep : str, default '\\t' (tab-stop)
1993 Character or regex pattern to treat as the delimiter. ``sep=None`` detects
1994 the separator from the first valid row of the file with Python's builtin
1995 sniffer tool, ``csv.Sniffer``; it is supported only by the Python parsing
1996 engine, which will be used automatically.
1997 In addition, separators longer than 1 character and different from
1998 ``'\\s+'`` will be interpreted as regular expressions and will force
1999 the use of the Python parsing engine. Note that regex delimiters are prone
2000 to ignoring quoted data. Regex example: ``'\\r\\t'``.
2001 delimiter : str, optional
2002 Alias for ``sep``.
2003 header : int, Sequence of int, 'infer' or None, default 'infer'
2004 Row index or indices in the file to use as DataFrame column labels.
2005 Indexing starts at 0 and counts only non-blank, non-commented lines
2006 when ``skip_blank_lines=True``, so ``header=0`` denotes the first
2007 line of data rather than the first line of the file. Valid arguments
2008 are:
2010 * ``int``: line index at which column labels are read.
2011 * Sequence of ``int``: line indices at which column labels are read
2012 and combined into a :class:`~pandas.MultiIndex` on the columns,
2013 e.g. ``[0, 1, 3]``. Intervening rows that are not specified will
2014 be skipped (e.g. row 2 in this example).
2015 * ``None``: no row in the file is interpreted as column labels.
2016 Columns are labelled by integer position, or by the values passed
2017 to ``names`` when provided. Use this for files with no header
2018 row. If the file has a header row that should be overridden by
2019 ``names``, pass ``header=0`` instead.
2020 * ``'infer'`` (default): equivalent to ``header=0`` when no
2021 ``names`` are passed, otherwise equivalent to ``header=None``.
2023 When inferred from the file contents, headers are kept distinct from
2024 each other by renaming duplicate names with a numeric suffix of the form
2025 ``".{count}"`` starting from 1, e.g. ``"foo"`` and ``"foo.1"``.
2026 Empty headers are named
2027 ``"Unnamed: {i}"`` or ``"Unnamed: {i}_level_{level}"``
2028 in the case of MultiIndex columns.
2029 names : Sequence of Hashable, optional
2030 Sequence of column labels to apply. If the file contains a header row,
2031 then you should explicitly pass ``header=0`` to override the column names.
2032 Duplicates in this list are not allowed.
2033 index_col : Hashable, Sequence of Hashable or False, optional
2034 Column(s) to use as row label(s), denoted either by column labels or column
2035 indices. If a sequence of labels or indices is given,
2036 :class:`~pandas.MultiIndex`
2037 will be formed for the row labels.
2039 Note: ``index_col=False`` can be used to force pandas to *not* use the first
2040 column as the index, e.g., when you have a malformed file with delimiters at
2041 the end of each line.
2042 usecols : Sequence of Hashable or Callable, optional
2043 Subset of columns to select, denoted either by column labels or column indices.
2044 If list-like, all elements must either
2045 be positional (i.e. integer indices into the document columns) or strings
2046 that correspond to column names provided either by the user in ``names`` or
2047 inferred from the document header row(s). If ``names`` are given, the document
2048 header row(s) are not taken into account. For example, a valid list-like
2049 ``usecols`` parameter would be ``[0, 1, 2]`` or ``['foo', 'bar', 'baz']``.
2050 Element order is ignored, so ``usecols=[0, 1]`` is the same as ``[1, 0]``.
2051 To instantiate a :class:`~pandas.DataFrame` from ``data`` with element order
2052 preserved use ``pd.read_csv(data, usecols=['foo', 'bar'])[['foo', 'bar']]``
2053 for columns in ``['foo', 'bar']`` order or
2054 ``pd.read_csv(data, usecols=['foo', 'bar'])[['bar', 'foo']]``
2055 for ``['bar', 'foo']`` order.
2057 If callable, the callable function will be evaluated against the column
2058 names, returning names where the callable function evaluates to ``True``. An
2059 example of a valid callable argument would be ``lambda x: x.upper() in
2060 ['AAA', 'BBB', 'DDD']``. Using this parameter results in much faster
2061 parsing time and lower memory usage.
2062 dtype : dtype or dict of {Hashable : dtype}, optional
2063 Data type(s) to apply to either the whole dataset or individual columns.
2064 E.g., ``{'a': np.float64, 'b': np.int32, 'c': 'Int64'}``
2065 Use ``str`` or ``object`` together with suitable ``na_values`` settings
2066 to preserve and not interpret ``dtype``.
2067 Specifying a ``dtype`` does not change how missing values are parsed;
2068 values recognized as missing (see ``na_values`` and ``keep_default_na``)
2069 are still read as ``NaN`` (or the dtype's NA value) rather than cast to
2070 the requested ``dtype``.
2071 If ``converters`` are specified, they will be applied INSTEAD
2072 of ``dtype`` conversion. Specify a ``defaultdict`` as input where
2073 the default determines the ``dtype`` of the columns which
2074 are not explicitly listed.
2075 engine : {'c', 'python', 'pyarrow'}, optional
2076 Parser engine to use. The C and pyarrow engines are faster,
2077 while the python engine
2078 is currently more feature-complete. The pyarrow engine is
2079 multithreaded, and the C engine reads sufficiently large files in
2080 parallel. The 'pyarrow' engine is an *experimental* engine,
2081 and some features are unsupported, or may not work correctly, with this engine.
2082 converters : dict of {Hashable : Callable}, optional
2083 Functions for converting values in specified columns. Keys can either
2084 be column labels or column indices. The function is applied to the raw
2085 text read from the file, so an empty field is passed as an empty string
2086 ``''``; ``na_values`` and ``keep_default_na`` are then applied to the
2087 value the function returns.
2088 true_values : list, optional
2089 Values to consider as ``True`` in addition to
2090 case-insensitive variants of 'True'.
2091 false_values : list, optional
2092 Values to consider as ``False`` in addition
2093 to case-insensitive variants of 'False'.
2094 skipinitialspace : bool, default False
2095 Skip spaces after delimiter.
2096 skiprows : int, list of int or Callable, optional
2097 Line numbers to skip (0-indexed) or number of lines to skip (``int``)
2098 at the start of the file.
2100 If callable, the callable function will be evaluated against the row
2101 indices, returning ``True`` if the row
2102 should be skipped and ``False`` otherwise.
2103 An example of a valid callable argument would be ``lambda x: x in [0, 2]``.
2104 skipfooter : int, default 0
2105 Number of lines at bottom of file to skip (Unsupported with ``engine='c'``).
2106 nrows : int, optional
2107 Number of rows of file to read. Useful for reading pieces of large files.
2108 Refers to the number of data rows in the returned DataFrame, excluding:
2110 * The header row containing column names.
2111 * Rows before the header row, if ``header=1`` or larger.
2113 Example usage:
2115 * To read the first 999,999 (non-header) rows:
2116 ``read_csv(..., nrows=999999)``
2118 * To read rows 1,000,000 through 1,999,999:
2119 ``read_csv(..., skiprows=1000000, nrows=999999)``
2121 na_values : Hashable, Iterable of Hashable or dict of {Hashable : Iterable},
2122 optional
2123 Additional strings to recognize as ``NA``/``NaN``.
2124 If ``dict`` passed, specific
2125 per-column ``NA`` values. By default the following values are interpreted as
2126 ``NaN``: empty string, "NaN", "N/A", "NULL", and other
2127 common representations of missing data.
2128 keep_default_na : bool, default True
2129 Whether or not to include the default ``NaN`` values when parsing the data.
2130 Depending on whether ``na_values`` is passed in, the behavior is as follows:
2132 * If ``keep_default_na`` is ``True``,
2133 and ``na_values`` are specified, ``na_values``
2134 is appended to the default ``NaN`` values used for parsing.
2135 * If ``keep_default_na`` is ``True``, and ``na_values`` are not specified, only
2136 the default ``NaN`` values are used for parsing.
2137 * If ``keep_default_na`` is ``False``, and ``na_values`` are specified, only
2138 the ``NaN`` values specified ``na_values`` are used for parsing.
2139 * If ``keep_default_na`` is ``False``, and ``na_values`` are not specified, no
2140 strings will be parsed as ``NaN``.
2142 Note that if ``na_filter`` is passed in as
2143 ``False``, the ``keep_default_na`` and
2144 ``na_values`` parameters will be ignored.
2145 na_filter : bool, default True
2146 Detect missing value markers (empty strings and the value of ``na_values``). In
2147 data without any ``NA`` values, passing ``na_filter=False`` can improve the
2148 performance of reading a large file.
2149 skip_blank_lines : bool, default True
2150 If ``True``, skip over blank lines rather than interpreting as ``NaN`` values.
2151 parse_dates : bool, None, list of Hashable, default None
2152 The behavior is as follows:
2154 * ``bool``. If ``True`` -> try parsing the index.
2155 * ``None``. Behaves like ``True`` if ``date_format`` is specified.
2156 * ``list`` of ``int`` or names.
2157 e.g. If ``[1, 2, 3]`` -> try parsing columns 1, 2, 3
2158 each as a separate date column.
2160 If a column or index cannot be represented as an array of ``datetime``,
2161 say because of an unparsable value or a mixture of timezones, the column
2162 or index will be returned unaltered as an ``object`` data type. For
2163 non-standard ``datetime`` parsing, use :func:`~pandas.to_datetime` after
2164 :func:`~pandas.read_csv`.
2166 Note: A fast-path exists for iso8601-formatted dates.
2167 date_format : str or dict of column -> format, optional
2168 Format to use for parsing dates and/or times when used
2169 in conjunction with ``parse_dates``.
2170 The strftime to parse time, e.g. :const:`"%d/%m/%Y"`. See
2171 `strftime documentation
2172 <https://docs.python.org/3/library/datetime.html
2173 #strftime-and-strptime-behavior>`_ for more information on choices, though
2174 note that :const:`"%f"`` will parse all the way up to nanoseconds.
2175 You can also pass:
2177 - "ISO8601", to parse any `ISO8601 <https://en.wikipedia.org/wiki/ISO_8601>`_
2178 time string (not necessarily in exactly the same format);
2179 - "mixed", to infer the format for each element individually. This is risky,
2180 and you should probably use it along with `dayfirst`.
2182 .. versionadded:: 2.0.0
2183 dayfirst : bool, default False
2184 DD/MM format dates, international and European format.
2185 cache_dates : bool, default True
2186 If ``True``, use a cache of unique, converted dates to apply the ``datetime``
2187 conversion. May produce significant speed-up when parsing duplicate
2188 date strings, especially ones with timezone offsets.
2190 iterator : bool, default False
2191 Return ``TextFileReader`` object for iteration or getting chunks with
2192 ``get_chunk()``.
2193 chunksize : int, optional
2194 Number of lines to read from the file per chunk. Passing a value will cause the
2195 function to return a ``TextFileReader`` object for iteration.
2196 See the `IO Tools docs
2197 <https://pandas.pydata.org/docs/dev/user_guide/io.html#io-chunking>`_
2198 for more information on ``iterator`` and ``chunksize``.
2200 compression : str or dict, default 'infer'
2201 For on-the-fly decompression of on-disk data. If 'infer'
2202 and 'filepath_or_buffer' is
2203 path-like, then detect compression from the following extensions: '.gz',
2204 '.bz2', '.zip', '.xz', '.zst', '.tar', '.tar.gz', '.tar.xz' or '.tar.bz2'
2205 (otherwise no compression).
2206 If using 'zip' or 'tar', the ZIP file must contain
2207 only one data file to be read in.
2208 Set to ``None`` for no decompression.
2209 Can also be a dict with key ``'method'`` set
2210 to one of {``'zip'``, ``'gzip'``, ``'bz2'``,
2211 ``'zstd'``, ``'xz'``, ``'tar'``} and
2212 other key-value pairs are forwarded to
2213 ``zipfile.ZipFile``, ``gzip.GzipFile``,
2214 ``bz2.BZ2File``, ``zstandard.ZstdDecompressor``, ``lzma.LZMAFile`` or
2215 ``tarfile.TarFile``, respectively.
2216 As an example, the following could be passed for
2217 Zstandard decompression using a
2218 custom compression dictionary:
2219 ``compression={'method': 'zstd', 'dict_data': my_compression_dict}``.
2221 thousands : str (length 1), optional
2222 Character acting as the thousands separator in numerical values.
2223 decimal : str (length 1), default '.'
2224 Character to recognize as decimal point (e.g., use ',' for European data).
2225 lineterminator : str (length 1), optional
2226 Character used to denote a line break. Only valid with C parser.
2227 quotechar : str (length 1), optional
2228 Character used to denote the start and end of a quoted item. Quoted
2229 items can include the ``delimiter`` and it will be ignored.
2230 quoting : {0, 1, 2, 3}, default csv.QUOTE_MINIMAL
2231 Control field quoting behavior per ``csv.QUOTE_*`` constants. Use one of
2232 ``0`` or ``csv.QUOTE_MINIMAL``, ``1`` or ``csv.QUOTE_ALL``,
2233 ``2`` or ``csv.QUOTE_NONNUMERIC``, or ``3`` or ``csv.QUOTE_NONE``.
2234 Default is ``csv.QUOTE_MINIMAL`` (i.e., 0) which
2235 implies that only fields containing special
2236 characters are quoted (e.g., characters defined
2237 in ``quotechar``, ``delimiter``,
2238 or ``lineterminator``.
2239 doublequote : bool, default True
2240 When ``quotechar`` is specified and ``quoting`` is not ``QUOTE_NONE``, indicate
2241 whether or not to interpret two consecutive ``quotechar`` elements INSIDE a
2242 field as a single ``quotechar`` element.
2243 escapechar : str (length 1), optional
2244 Character used to escape other characters.
2245 comment : str (length 1), optional
2246 Character indicating that the remainder of line should not be parsed.
2247 If found at the beginning
2248 of a line, the line will be ignored altogether. This parameter must be a
2249 single character. Like empty lines (as long as ``skip_blank_lines=True``),
2250 fully commented lines are ignored by the parameter ``header`` but not by
2251 ``skiprows``. For example, if ``comment='#'``, parsing
2252 ``#empty\\na,b,c\\n1,2,3`` with ``header=0`` will result in ``'a,b,c'`` being
2253 treated as the header.
2254 encoding : str, optional, default 'utf-8'
2255 Encoding to use for UTF when reading/writing (ex. ``'utf-8'``). `List of Python
2256 standard encodings
2257 <https://docs.python.org/3/library/codecs.html#standard-encodings>`_ .
2259 encoding_errors : str, optional, default 'strict'
2260 How encoding errors are treated. `List of possible values
2261 <https://docs.python.org/3/library/codecs.html#error-handlers>`_ .
2263 dialect : str or csv.Dialect, optional
2264 If provided, this parameter will override values (default or not) for the
2265 following parameters: ``delimiter``, ``doublequote``, ``escapechar``,
2266 ``skipinitialspace``, ``quotechar``, and ``quoting``. If it is necessary to
2267 override values, a ``ParserWarning`` will be issued. See ``csv.Dialect``
2268 documentation for more details.
2269 on_bad_lines : {'error', 'warn', 'skip'} or Callable, default 'error'
2270 Specifies what to do upon encountering a bad
2271 line (a line with too many fields).
2272 Allowed values are:
2274 - ``'error'``, raise an Exception when a bad line is encountered.
2275 - ``'warn'``, raise a warning when a bad line is encountered and
2276 skip that line.
2277 - ``'skip'``, skip bad lines without raising or warning when they
2278 are encountered.
2279 - Callable, function that will process a single bad line.
2280 - With ``engine='python'``, function with signature
2281 ``(bad_line: list[str]) -> list[str] | None``.
2282 ``bad_line`` is a list of strings split by the ``sep``.
2283 If the function returns ``None``, the bad line will be ignored.
2284 If the function returns a new ``list`` of strings with more elements than
2285 expected, a ``ParserWarning`` will be emitted while
2286 dropping extra elements.
2287 - With ``engine='pyarrow'``, function with signature
2288 as described in pyarrow documentation: `invalid_row_handler
2289 <https://arrow.apache.org/docs/
2290 python/generated/pyarrow.csv.ParseOptions.html
2291 #pyarrow.csv.ParseOptions.invalid_row_handler>`_.
2293 .. versionadded:: 2.2.0
2295 Callable for ``engine='pyarrow'``
2297 low_memory : bool, default True
2298 Internally process the file in chunks, resulting in lower memory use
2299 while parsing, but possibly mixed type inference. Because each chunk
2300 is type-inferred independently, the same literal can be parsed as an
2301 ``int`` in one chunk and a ``str`` in another; after concatenation the
2302 column has ``object`` dtype and contains both representations, so
2303 comparisons such as ``df["col"] == 12345`` or
2304 ``df["col"] == "12345"`` will match only a subset of the rows holding
2305 that value. A :class:`~pandas.errors.DtypeWarning` is emitted when
2306 this occurs. To ensure no mixed types either set ``False``, or specify
2307 the type with the ``dtype`` parameter.
2308 Note that the entire file is read into a single :class:`~pandas.DataFrame`
2309 regardless, use the ``chunksize`` or ``iterator`` parameter
2310 to return the data in
2311 chunks. (Only valid with C parser).
2312 memory_map : bool, default False
2313 If a filepath is provided for ``filepath_or_buffer``, map the file object
2314 directly onto memory and access the data directly from there. Using this
2315 option can improve performance because there is no longer any I/O overhead.
2316 float_precision : {'high', 'legacy', 'round_trip'}, optional
2317 Specifies which converter the C engine should use for floating-point
2318 values. The options are ``None`` or ``'high'`` for the ordinary converter,
2319 ``'legacy'`` for the original lower precision pandas converter, and
2320 ``'round_trip'`` for the round-trip converter.
2322 .. deprecated:: 3.1.0
2323 All float precision modes now use the same converter.
2324 The ``float_precision`` argument will be removed in a future version.
2326 storage_options : dict, optional
2327 Extra options that make sense for a particular storage connection, e.g.
2328 host, port, username, password, etc. For HTTP(S) URLs the key-value pairs
2329 are forwarded to ``urllib.request.Request`` as header options. For other
2330 URLs (e.g. starting with "s3://", and "gcs://") the key-value pairs are
2331 forwarded to ``fsspec.open``. Please see ``fsspec`` and ``urllib`` for more
2332 details, and for more examples on storage options refer `here
2333 <https://pandas.pydata.org/docs/user_guide/io.html?
2334 highlight=storage_options#reading-writing-remote-files>`_.
2336 dtype_backend : {'numpy_nullable', 'pyarrow'}
2337 Back-end data type applied to the resultant :class:`DataFrame`
2338 (still experimental). If not specified, the default behavior
2339 is to not use nullable data types. If specified, the behavior
2340 is as follows:
2342 * ``"numpy_nullable"``: returns nullable-dtype-backed :class:`DataFrame`
2343 * ``"pyarrow"``: returns pyarrow-backed nullable
2344 :class:`ArrowDtype` :class:`DataFrame`
2346 .. versionadded:: 2.0
2348 Returns
2349 -------
2350 DataFrame or TextFileReader
2351 A comma-separated values (csv) file is returned as two-dimensional
2352 data structure with labeled axes.
2354 See Also
2355 --------
2356 DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
2357 read_csv : Read a comma-separated values (csv) file into DataFrame.
2358 read_fwf : Read a table of fixed-width formatted lines into DataFrame.
2360 Notes
2361 -----
2362 Column labels read from a header row are always strings. To use column
2363 labels of another type, pass them explicitly with ``names`` and set
2364 ``header=0`` when the file contains a header row.
2366 Examples
2367 --------
2368 >>> pd.read_table("data.csv") # doctest: +SKIP
2369 Name Value
2370 0 foo 1
2371 1 bar 2
2372 2 #baz 3
2374 Index and header can be specified via the `index_col` and `header` arguments.
2376 >>> pd.read_table("data.csv", header=None) # doctest: +SKIP
2377 0 1
2378 0 Name Value
2379 1 foo 1
2380 2 bar 2
2381 3 #baz 3
2383 >>> pd.read_table("data.csv", index_col="Value") # doctest: +SKIP
2384 Name
2385 Value
2386 1 foo
2387 2 bar
2388 3 #baz
2390 Column types are inferred but can be explicitly specified using the dtype argument.
2392 >>> pd.read_table("data.csv", dtype={"Value": float}) # doctest: +SKIP
2393 Name Value
2394 0 foo 1.0
2395 1 bar 2.0
2396 2 #baz 3.0
2398 True, False, and NA values, and thousands separators have defaults,
2399 but can be explicitly specified, too. Supply the values you would like
2400 as strings or lists of strings!
2402 >>> pd.read_table("data.csv", na_values=["foo", "bar"]) # doctest: +SKIP
2403 Name Value
2404 0 NaN 1
2405 1 NaN 2
2406 2 #baz 3
2408 Comment lines in the input file can be skipped using the `comment` argument.
2410 >>> pd.read_table("data.csv", comment="#") # doctest: +SKIP
2411 Name Value
2412 0 foo 1
2413 1 bar 2
2415 By default, columns with dates will be read as ``object`` rather than ``datetime``.
2417 >>> df = pd.read_table("tmp.csv") # doctest: +SKIP
2419 >>> df # doctest: +SKIP
2420 col 1 col 2 col 3
2421 0 10 10/04/2018 Sun 15 Jan 2023
2422 1 20 15/04/2018 Fri 12 May 2023
2424 >>> df.dtypes # doctest: +SKIP
2425 col 1 int64
2426 col 2 object
2427 col 3 object
2428 dtype: object
2430 Specific columns can be parsed as dates by using the `parse_dates` and
2431 `date_format` arguments.
2433 >>> df = pd.read_table(
2434 ... "tmp.csv",
2435 ... parse_dates=[1, 2],
2436 ... date_format={"col 2": "%d/%m/%Y", "col 3": "%a %d %b %Y"},
2437 ... ) # doctest: +SKIP
2439 >>> df.dtypes # doctest: +SKIP
2440 col 1 int64
2441 col 2 datetime64[ns]
2442 col 3 datetime64[ns]
2443 dtype: object
2444 """
2445 # locals() should never be modified
2446 kwds = locals().copy()
2447 del kwds["filepath_or_buffer"]
2448 del kwds["sep"]
2450 kwds_defaults = _refine_defaults_read(
2451 dialect,
2452 delimiter,
2453 engine,
2454 sep,
2455 on_bad_lines,
2456 names,
2457 defaults={"delimiter": "\t"},
2458 dtype_backend=dtype_backend,
2459 lineterminator=lineterminator,
2460 )
2461 kwds.update(kwds_defaults)
2463 return _read(filepath_or_buffer, kwds)
2466@overload
2467def read_fwf(
2468 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
2469 *,
2470 colspecs: Sequence[tuple[int, int]] | str | None = ...,
2471 widths: Sequence[int] | None = ...,
2472 infer_nrows: int = ...,
2473 iterator: Literal[True],
2474 chunksize: int | None = ...,
2475 **kwds: Unpack[_read_shared[HashableT]],
2476) -> TextFileReader: ...
2479@overload
2480def read_fwf(
2481 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
2482 *,
2483 colspecs: Sequence[tuple[int, int]] | str | None = ...,
2484 widths: Sequence[int] | None = ...,
2485 infer_nrows: int = ...,
2486 iterator: bool = ...,
2487 chunksize: int,
2488 **kwds: Unpack[_read_shared[HashableT]],
2489) -> TextFileReader: ...
2492@overload
2493def read_fwf(
2494 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
2495 *,
2496 colspecs: Sequence[tuple[int, int]] | str | None = ...,
2497 widths: Sequence[int] | None = ...,
2498 infer_nrows: int = ...,
2499 iterator: Literal[False] = ...,
2500 chunksize: None = ...,
2501 **kwds: Unpack[_read_shared[HashableT]],
2502) -> DataFrame: ...
2505@set_module("pandas")
2506def read_fwf(
2507 filepath_or_buffer: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str],
2508 *,
2509 colspecs: Sequence[tuple[int, int]] | str | None = "infer",
2510 widths: Sequence[int] | None = None,
2511 infer_nrows: int = 100,
2512 iterator: bool = False,
2513 chunksize: int | None = None,
2514 **kwds: Unpack[_read_shared[HashableT]],
2515) -> DataFrame | TextFileReader:
2516 r"""
2517 Read a table of fixed-width formatted lines into DataFrame.
2519 Also supports optionally iterating or breaking of the file
2520 into chunks.
2522 Additional help can be found in the `online docs for IO Tools
2523 <https://pandas.pydata.org/pandas-docs/stable/user_guide/io.html>`_.
2525 Parameters
2526 ----------
2527 filepath_or_buffer : str, path object, or file-like object
2528 Any valid string path is acceptable. The string could be a URL. Valid
2529 URL schemes include http, ftp, s3, and file. For file URLs, a host is
2530 expected. A local file could be: ``file://localhost/path/to/table.csv``.
2532 Certain URL schemes may require additional packages. For example, S3
2533 URLs require the ``s3fs`` library. See
2534 :ref:`install.optional_dependencies` for a full list.
2536 If you want to pass in a path object, pandas accepts any ``os.PathLike``.
2538 By file-like object, we refer to objects with a ``read()`` method that
2539 accepts an optional size argument, such as a file handle (e.g. via
2540 builtin ``open`` function) or ``StringIO``.
2541 colspecs : list of tuple (int, int) or 'infer'. optional
2542 A list of tuples giving the extents of the fixed-width
2543 fields of each line as half-open intervals (i.e., [from, to] ).
2544 String value 'infer' can be used to instruct the parser to try
2545 detecting the column specifications from the first 100 rows of
2546 the data which are not being skipped via skiprows (default='infer').
2547 widths : list of int, optional
2548 A list of field widths which can be used instead of 'colspecs' if
2549 the intervals are contiguous.
2550 infer_nrows : int, default 100
2551 The number of rows to consider when letting the parser determine the
2552 ``colspecs``. Pass ``np.inf`` to infer column specifications from the
2553 entire file, which is useful when columns vary in width and a value
2554 wider than anything in the first ``infer_nrows`` rows would otherwise
2555 be truncated.
2556 iterator : bool, default False
2557 Return ``TextFileReader`` object for iteration or getting chunks with
2558 ``get_chunk()``.
2559 chunksize : int, optional
2560 Number of lines to read from the file per chunk.
2561 **kwds : optional
2562 Optional keyword arguments can be passed to ``TextFileReader``.
2564 Returns
2565 -------
2566 DataFrame or TextFileReader
2567 A comma-separated values (csv) file is returned as two-dimensional
2568 data structure with labeled axes.
2570 See Also
2571 --------
2572 DataFrame.to_csv : Write DataFrame to a comma-separated values (csv) file.
2573 read_csv : Read a comma-separated values (csv) file into DataFrame.
2575 Examples
2576 --------
2577 >>> pd.read_fwf("data.csv") # doctest: +SKIP
2578 """
2579 # Check input arguments.
2580 if colspecs is None and widths is None:
2581 raise ValueError("Must specify either colspecs or widths")
2582 if colspecs not in (None, "infer") and widths is not None:
2583 raise ValueError("You must specify only one of 'widths' and 'colspecs'")
2585 # Compute 'colspecs' from 'widths', if specified.
2586 if widths is not None:
2587 colspecs, col = [], 0
2588 for w in widths:
2589 colspecs.append((col, col + w))
2590 col += w
2592 # for mypy
2593 assert colspecs is not None
2595 # GH#40830
2596 # Ensure length of `colspecs` matches length of `names`
2597 names = kwds.get("names")
2598 if names is not None and names is not lib.no_default:
2599 if len(names) != len(colspecs) and colspecs != "infer":
2600 # need to check len(index_col) as it might contain
2601 # unnamed indices, in which case it's name is not required
2602 len_index = 0
2603 if kwds.get("index_col") is not None:
2604 index_col: Any = kwds.get("index_col")
2605 if index_col is not False:
2606 if not is_list_like(index_col):
2607 len_index = 1
2608 else:
2609 # for mypy: handled in the if-branch
2610 assert index_col is not lib.no_default
2612 len_index = len(index_col)
2613 if kwds.get("usecols") is None and len(names) + len_index != len(colspecs):
2614 # If usecols is used colspec may be longer than names
2615 raise ValueError("Length of colspecs must match length of names")
2617 check_dtype_backend(kwds.setdefault("dtype_backend", lib.no_default))
2618 return _read(
2619 filepath_or_buffer,
2620 kwds
2621 | {
2622 "colspecs": colspecs,
2623 "infer_nrows": infer_nrows,
2624 "engine": "python-fwf",
2625 "iterator": iterator,
2626 "chunksize": chunksize,
2627 },
2628 )
2631class TextFileReader(abc.Iterator):
2632 """
2633 Iterator over chunks of a delimited text file.
2635 Returned by :func:`read_csv`, :func:`read_table`, and :func:`read_fwf`
2636 when ``iterator=True`` or ``chunksize`` is given. Not intended to be
2637 constructed directly by users.
2639 A ``TextFileReader`` can be iterated over to yield successive chunks as
2640 :class:`DataFrame` objects, used as a context manager to ensure the
2641 underlying file handle is closed, or queried explicitly via
2642 :meth:`~TextFileReader.read` and :meth:`~TextFileReader.get_chunk`.
2644 Passed ``dialect`` overrides any of the related parser options.
2646 Parameters
2647 ----------
2648 f : str, path object, or file-like object
2649 Source to read from. Accepts the same inputs as :func:`read_csv`.
2650 engine : {'c', 'python', 'pyarrow', 'python-fwf'}, optional
2651 Parser engine to use. If not specified, defaults to ``'python'``.
2652 **kwds
2653 Any keyword argument accepted by :func:`read_csv`, :func:`read_table`,
2654 or :func:`read_fwf`.
2656 See Also
2657 --------
2658 read_csv : Read a comma-separated values (csv) file into a DataFrame.
2659 read_table : Read general delimited file into a DataFrame.
2660 read_fwf : Read a table of fixed-width formatted lines into a DataFrame.
2662 Examples
2663 --------
2664 >>> with pd.read_csv("data.csv", chunksize=1000) as reader: # doctest: +SKIP
2665 ... for chunk in reader:
2666 ... process(chunk)
2668 >>> with pd.read_csv("data.csv", iterator=True) as reader: # doctest: +SKIP
2669 ... first = reader.get_chunk(5)
2670 """
2672 def __init__(
2673 self,
2674 f: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str] | list,
2675 engine: CSVEngine | None = None,
2676 **kwds,
2677 ) -> None:
2678 if engine is not None:
2679 engine_specified = True
2680 else:
2681 engine = "python"
2682 engine_specified = False
2683 self.engine = engine
2684 self._engine_specified = kwds.get("engine_specified", engine_specified)
2686 # before _validate_skipfooter, which reads `iterator` for truthiness
2687 _validate_bool_kwargs(kwds)
2688 _validate_skipfooter(kwds)
2690 dialect = _extract_dialect(kwds)
2691 if dialect is not None:
2692 if engine == "pyarrow":
2693 raise ValueError(
2694 "The 'dialect' option is not supported with the 'pyarrow' engine"
2695 )
2696 kwds = _merge_with_dialect_properties(dialect, kwds)
2697 # again, for the values the dialect supplied
2698 _validate_bool_kwargs(kwds)
2700 if kwds.get("header", "infer") == "infer":
2701 kwds["header"] = 0 if kwds.get("names") is None else None
2703 self.orig_options = kwds
2705 # miscellanea
2706 self._currow = 0
2708 options = self._get_options_with_defaults(engine)
2709 options["storage_options"] = kwds.get("storage_options", None)
2711 self.chunksize = options.pop("chunksize", None)
2712 self.nrows = options.pop("nrows", None)
2714 self._check_file_or_buffer(f, engine)
2715 self.options, self.engine = self._clean_options(options, engine)
2717 if "has_index_names" in kwds:
2718 self.options["has_index_names"] = kwds["has_index_names"]
2720 self.handles: IOHandles | None = None
2721 self._engine = self._make_engine(f, self.engine)
2723 def close(self) -> None:
2724 """
2725 Close the underlying file handle and parser engine.
2727 Called automatically when the ``TextFileReader`` is used as a context
2728 manager.
2730 See Also
2731 --------
2732 TextFileReader.read : Read rows from the file into a DataFrame.
2733 TextFileReader.get_chunk : Read the next chunk of rows.
2735 Examples
2736 --------
2737 >>> reader = pd.read_csv("data.csv", chunksize=5) # doctest: +SKIP
2738 >>> reader.close() # doctest: +SKIP
2739 """
2740 if self.handles is not None:
2741 self.handles.close()
2742 self._engine.close()
2744 def _get_options_with_defaults(self, engine: CSVEngine) -> dict[str, Any]:
2745 kwds = self.orig_options
2747 options = {}
2748 default: object | None
2750 for argname, default in parser_defaults.items():
2751 value = kwds.get(argname, default)
2753 # see gh-12935
2754 if (
2755 engine == "pyarrow"
2756 and argname in _pyarrow_unsupported
2757 and value != default
2758 and value != getattr(value, "value", default)
2759 ):
2760 raise ValueError(
2761 f"The {argname!r} option is not supported with the 'pyarrow' engine"
2762 )
2763 options[argname] = value
2765 for argname, default in _c_parser_defaults.items():
2766 if argname in kwds:
2767 value = kwds[argname]
2769 if engine != "c" and value != default:
2770 # TODO: Refactor this logic, its pretty convoluted
2771 if "python" in engine and argname not in _python_unsupported:
2772 pass
2773 elif "pyarrow" in engine and argname not in _pyarrow_unsupported:
2774 pass
2775 else:
2776 raise ValueError(
2777 f"The {argname!r} option is not supported with the "
2778 f"{engine!r} engine"
2779 )
2780 else:
2781 value = default
2782 options[argname] = value
2784 if engine == "python-fwf":
2785 for argname, default in _fwf_defaults.items():
2786 options[argname] = kwds.get(argname, default)
2788 return options
2790 def _check_file_or_buffer(self, f, engine: CSVEngine) -> None:
2791 if hasattr(f, "encoding"):
2792 file_encoding = f.encoding
2793 orig_reader_enc = self.orig_options.get("encoding", None)
2794 any_none = file_encoding is None or orig_reader_enc is None
2795 if file_encoding != orig_reader_enc and not any_none:
2796 file_path = getattr(f, "name", None)
2797 raise ValueError(
2798 f"The specified reader encoding {orig_reader_enc} is different "
2799 f"from the encoding {file_encoding} of file {file_path}."
2800 )
2802 def _clean_options(
2803 self, options: dict[str, Any], engine: CSVEngine
2804 ) -> tuple[dict[str, Any], CSVEngine]:
2805 result = options.copy()
2807 fallback_reason = None
2809 # C engine not supported yet
2810 if engine == "c":
2811 if options["skipfooter"] > 0:
2812 fallback_reason = "the 'c' engine does not support skipfooter"
2813 engine = "python"
2815 sep = options["delimiter"]
2817 if sep is None:
2818 # sniffing the separator with csv.Sniffer is python-engine only
2819 if engine in ("c", "pyarrow"):
2820 fallback_reason = f"the '{engine}' engine does not support sep=None"
2821 engine = "python"
2822 elif len(sep) > 1:
2823 if engine == "c" and sep == r"\s+":
2824 # delim_whitespace passed on to pandas._libs.parsers.TextReader
2825 result["delim_whitespace"] = True
2826 del result["delimiter"]
2827 elif engine not in ("python", "python-fwf"):
2828 # wait until regex engine integrated
2829 fallback_reason = (
2830 f"the '{engine}' engine does not support "
2831 "separators > 1 char, including regex separators"
2832 )
2833 engine = "python"
2834 else:
2835 # The C engine always tokenizes a utf-8 byte stream: text handles
2836 # are re-encoded to utf-8 before reaching it, whatever `encoding`
2837 # says. So the separator has to be a single utf-8 byte; the
2838 # platform's filesystem encoding has nothing to do with it.
2839 encodeable = True
2840 try:
2841 if len(sep.encode("utf-8")) > 1:
2842 encodeable = False
2843 except UnicodeEncodeError:
2844 # e.g. a lone surrogate
2845 encodeable = False
2846 if not encodeable and engine not in ("python", "python-fwf"):
2847 fallback_reason = (
2848 "the separator encoded in utf-8 "
2849 f"is > 1 char long, and the '{engine}' engine "
2850 "does not support such separators"
2851 )
2852 engine = "python"
2854 quotechar = options["quotechar"]
2855 if quotechar is not None and isinstance(quotechar, (str, bytes)):
2856 if (
2857 len(quotechar) == 1
2858 and ord(quotechar) > 127
2859 and engine not in ("python", "python-fwf")
2860 ):
2861 fallback_reason = (
2862 "ord(quotechar) > 127, meaning the "
2863 "quotechar is larger than one byte, "
2864 f"and the '{engine}' engine does not support such quotechars"
2865 )
2866 engine = "python"
2868 if fallback_reason and self._engine_specified:
2869 raise ValueError(fallback_reason)
2871 if engine == "c":
2872 for arg in _c_unsupported:
2873 del result[arg]
2875 if "python" in engine:
2876 for arg in _python_unsupported:
2877 if fallback_reason and result[arg] != _c_parser_defaults.get(arg):
2878 raise ValueError(
2879 "Falling back to the 'python' engine because "
2880 f"{fallback_reason}, but this causes {arg!r} to be "
2881 "ignored as it is not supported by the 'python' engine."
2882 )
2883 del result[arg]
2885 if fallback_reason:
2886 warnings.warn(
2887 (
2888 "Falling back to the 'python' engine because "
2889 f"{fallback_reason}; you can avoid this warning by specifying "
2890 "engine='python'."
2891 ),
2892 ParserWarning,
2893 stacklevel=find_stack_level(),
2894 )
2896 index_col = options["index_col"]
2897 names = options["names"]
2898 converters = options["converters"]
2899 na_values = options["na_values"]
2900 skiprows = options["skiprows"]
2902 validate_header_arg(options["header"])
2904 if index_col is True:
2905 raise ValueError("The value of index_col couldn't be 'True'")
2906 if is_index_col(index_col):
2907 if not isinstance(index_col, (list, tuple, np.ndarray)):
2908 index_col = [index_col]
2909 result["index_col"] = index_col
2911 names = list(names) if names is not None else names
2913 # type conversion-related
2914 if converters is not None:
2915 if not isinstance(converters, dict):
2916 raise TypeError(
2917 "Type converters must be a dict or subclass, "
2918 f"input was a {type(converters).__name__}"
2919 )
2920 else:
2921 converters = {}
2923 # Converting values to NA
2924 keep_default_na = options["keep_default_na"]
2925 floatify = engine != "pyarrow"
2926 na_values, na_fvalues = _clean_na_values(
2927 na_values, keep_default_na, floatify=floatify
2928 )
2930 # handle skiprows; this is internally handled by the
2931 # c-engine, so only need for python and pyarrow parsers
2932 if engine == "pyarrow":
2933 if not is_integer(skiprows) and skiprows is not None:
2934 # pyarrow expects skiprows to be passed as an integer
2935 raise ValueError(
2936 "skiprows argument must be an integer when using engine='pyarrow'"
2937 )
2938 else:
2939 if is_integer(skiprows):
2940 skiprows = range(skiprows)
2941 if skiprows is None:
2942 skiprows = set()
2943 elif not callable(skiprows):
2944 skiprows = set(skiprows)
2946 # put stuff back
2947 result["names"] = names
2948 result["converters"] = converters
2949 result["na_values"] = na_values
2950 result["na_fvalues"] = na_fvalues
2951 result["skiprows"] = skiprows
2953 return result, engine
2955 def __next__(self) -> DataFrame:
2956 try:
2957 return self.get_chunk()
2958 except StopIteration:
2959 self.close()
2960 raise
2962 def _make_engine(
2963 self,
2964 f: FilePath | ReadCsvBuffer[bytes] | ReadCsvBuffer[str] | list | IO,
2965 engine: CSVEngine = "c",
2966 ) -> ParserBase:
2967 mapping: dict[str, type[ParserBase]] = {
2968 "c": CParserWrapper,
2969 "python": PythonParser,
2970 "pyarrow": ArrowParserWrapper,
2971 "python-fwf": FixedWidthFieldParser,
2972 }
2974 if engine not in mapping:
2975 raise ValueError(
2976 f"Unknown engine: {engine} (valid options are {mapping.keys()})"
2977 )
2978 if not isinstance(f, list):
2979 # open file here
2980 # The c and pyarrow engines can decode utf-8 bytes natively;
2981 # wrapping in TextIOWrapper makes them slower, especially for
2982 # memory_map and remote file-like objects (GH#46823).
2983 is_text = engine in ("python", "python-fwf") or (
2984 engine == "c"
2985 and self.options.get("encoding", "utf-8") not in (None, "utf-8")
2986 )
2987 mode = "r" if is_text else "rb"
2988 self.handles = get_handle(
2989 f,
2990 mode,
2991 encoding=self.options.get("encoding", None),
2992 compression=self.options.get("compression", None),
2993 memory_map=self.options.get("memory_map", False),
2994 is_text=is_text,
2995 errors=self.options.get("encoding_errors", "strict"),
2996 storage_options=self.options.get("storage_options", None),
2997 )
2998 assert self.handles is not None
2999 f = self.handles.handle
3001 elif engine != "python":
3002 msg = f"Invalid file path or buffer object type: {type(f)}"
3003 raise ValueError(msg)
3005 try:
3006 return mapping[engine](f, **self.options)
3007 except Exception:
3008 if self.handles is not None:
3009 self.handles.close()
3010 raise
3012 def _failover_to_python(self) -> None:
3013 raise AbstractMethodError(self)
3015 def read(self, nrows: int | None = None) -> DataFrame:
3016 """
3017 Read rows from the file into a :class:`DataFrame`.
3019 Advances the internal cursor by the number of rows returned, so
3020 successive calls yield later rows of the file. Used with readers
3021 created via :func:`read_csv` or :func:`read_table` with
3022 ``iterator=True`` or ``chunksize`` set.
3024 Parameters
3025 ----------
3026 nrows : int, optional
3027 Number of rows to read. If ``None``, read until end of file.
3028 Ignored when the underlying engine is ``"pyarrow"``, which always
3029 reads the full file.
3031 Returns
3032 -------
3033 DataFrame
3034 The parsed rows.
3036 See Also
3037 --------
3038 TextFileReader.get_chunk : Read the next chunk of rows.
3039 read_csv : Read a comma-separated values (csv) file into a DataFrame.
3041 Examples
3042 --------
3043 >>> with pd.read_csv("data.csv", iterator=True) as reader: # doctest: +SKIP
3044 ... df = reader.read()
3045 """
3046 if self.engine == "pyarrow":
3047 try:
3048 # error: "ParserBase" has no attribute "read"
3049 df = self._engine.read() # type: ignore[attr-defined]
3050 except Exception:
3051 self.close()
3052 raise
3053 else:
3054 nrows = validate_integer("nrows", nrows)
3055 try:
3056 # error: "ParserBase" has no attribute "read"
3057 (
3058 index,
3059 columns,
3060 col_dict,
3061 ) = self._engine.read( # type: ignore[attr-defined]
3062 nrows
3063 )
3064 except Exception:
3065 self.close()
3066 raise
3068 if index is None:
3069 if col_dict:
3070 # Any column is actually fine:
3071 new_rows = len(next(iter(col_dict.values())))
3072 index = RangeIndex(self._currow, self._currow + new_rows)
3073 else:
3074 new_rows = 0
3075 else:
3076 new_rows = len(index)
3078 if hasattr(self, "orig_options"):
3079 dtype_arg = self.orig_options.get("dtype", None)
3080 else:
3081 dtype_arg = None
3083 if isinstance(dtype_arg, dict):
3084 dtype = defaultdict(lambda: None) # type: ignore[var-annotated]
3085 dtype.update(dtype_arg)
3086 elif dtype_arg is not None and pandas_dtype(dtype_arg) in (
3087 np.str_,
3088 np.object_,
3089 ):
3090 dtype = defaultdict(lambda: dtype_arg)
3091 else:
3092 dtype = None
3094 if dtype is not None:
3095 new_col_dict = {}
3096 for k, v in col_dict.items():
3097 d = (
3098 dtype[k]
3099 if pandas_dtype(dtype[k]) in (np.str_, np.object_)
3100 else None
3101 )
3102 new_col_dict[k] = Series(v, index=index, dtype=d, copy=False)
3103 else:
3104 new_col_dict = col_dict
3106 df = DataFrame(
3107 new_col_dict,
3108 columns=columns,
3109 index=index,
3110 copy=False,
3111 )
3113 self._currow += new_rows
3114 return df
3116 def get_chunk(self, size: int | None = None) -> DataFrame:
3117 """
3118 Read the next chunk of rows from the file.
3120 Convenience wrapper around :meth:`TextFileReader.read` that defaults
3121 ``size`` to the ``chunksize`` passed to :func:`read_csv` or
3122 :func:`read_table`, and respects a configured ``nrows`` limit by
3123 raising ``StopIteration`` once it has been reached.
3125 Parameters
3126 ----------
3127 size : int, optional
3128 Number of rows to read in this chunk. Defaults to the ``chunksize``
3129 passed when the reader was created.
3131 Returns
3132 -------
3133 DataFrame
3134 The next chunk of parsed rows.
3136 Raises
3137 ------
3138 StopIteration
3139 If ``nrows`` was set and has already been reached.
3141 See Also
3142 --------
3143 TextFileReader.read : Read rows from the file into a DataFrame.
3144 read_csv : Read a comma-separated values (csv) file into a DataFrame.
3146 Examples
3147 --------
3148 >>> with pd.read_csv("data.csv", iterator=True) as reader: # doctest: +SKIP
3149 ... first_five = reader.get_chunk(5)
3150 """
3151 if size is None:
3152 size = self.chunksize
3153 if self.nrows is not None:
3154 if self._currow >= self.nrows:
3155 raise StopIteration
3156 if size is None:
3157 size = self.nrows - self._currow
3158 else:
3159 size = min(size, self.nrows - self._currow)
3160 return self.read(nrows=size)
3162 def __enter__(self) -> Self:
3163 return self
3165 def __exit__(
3166 self,
3167 exc_type: type[BaseException] | None,
3168 exc_value: BaseException | None,
3169 traceback: TracebackType | None,
3170 ) -> None:
3171 self.close()
3174def TextParser(*args, **kwds) -> TextFileReader:
3175 """
3176 Converts lists of lists/tuples into DataFrames with proper type inference
3177 and optional (e.g. string to datetime) conversion. Also enables iterating
3178 lazily over chunks of large files
3180 Parameters
3181 ----------
3182 data : file-like object or list
3183 delimiter : separator character to use
3184 dialect : str or csv.Dialect class / instance, optional
3185 Ignored if delimiter is longer than 1 character
3186 names : sequence, default
3187 header : int, default 0
3188 Row to use to parse column labels. Defaults to the first row. Prior
3189 rows will be discarded
3190 index_col : int or list, optional
3191 Column or columns to use as the (possibly hierarchical) index
3192 has_index_names: bool, default False
3193 True if the cols defined in index_col have an index name and are
3194 not in the header.
3195 na_values : scalar, str, list-like, or dict, optional
3196 Additional strings to recognize as NA/NaN.
3197 keep_default_na : bool, default True
3198 thousands : str, optional
3199 Thousands separator
3200 comment : str, optional
3201 Comment out remainder of line
3202 parse_dates : bool, default False
3203 date_format : str or dict of column -> format, default ``None``
3205 .. versionadded:: 2.0.0
3206 skiprows : list of integers
3207 Row numbers to skip
3208 skipfooter : int
3209 Number of line at bottom of file to skip
3210 converters : dict, optional
3211 Dict of functions for converting values in certain columns. Keys can
3212 either be integers or column labels, values are functions that take one
3213 input argument, the cell (not column) content, and return the
3214 transformed content.
3215 encoding : str, optional
3216 Encoding to use for UTF when reading/writing (ex. 'utf-8')
3217 float_precision : str, optional
3218 Specifies which converter the C engine should use for floating-point
3219 values. The options are `None` or `high` for the ordinary converter,
3220 `legacy` for the original lower precision pandas converter, and
3221 `round_trip` for the round-trip converter.
3223 .. deprecated:: 3.1.0
3224 All float precision modes now use the same converter.
3225 The ``float_precision`` argument will be removed in a future version.
3226 """
3227 kwds["engine"] = "python"
3228 return TextFileReader(*args, **kwds)
3231def _clean_na_values(na_values, keep_default_na: bool = True, floatify: bool = True):
3232 na_fvalues: set | dict
3233 if na_values is None:
3234 if keep_default_na:
3235 na_values = STR_NA_VALUES
3236 else:
3237 na_values = set()
3238 na_fvalues = set()
3239 elif isinstance(na_values, dict):
3240 old_na_values = na_values.copy()
3241 na_values = {} # Prevent aliasing.
3243 # Convert the values in the na_values dictionary
3244 # into array-likes for further use. This is also
3245 # where we append the default NaN values, provided
3246 # that `keep_default_na=True`.
3247 for k, v in old_na_values.items():
3248 if not is_list_like(v):
3249 v = [v]
3251 if keep_default_na:
3252 v = set(v) | STR_NA_VALUES
3254 na_values[k] = _stringify_na_values(v, floatify)
3255 na_fvalues = {k: _floatify_na_values(v) for k, v in na_values.items()}
3256 else:
3257 if not is_list_like(na_values):
3258 na_values = [na_values]
3259 na_values = _stringify_na_values(na_values, floatify)
3260 if keep_default_na:
3261 na_values = na_values | STR_NA_VALUES
3263 na_fvalues = _floatify_na_values(na_values)
3265 return na_values, na_fvalues
3268def _floatify_na_values(na_values) -> set[float]:
3269 # create float versions of the na_values
3270 result = set()
3271 for v in na_values:
3272 try:
3273 v = float(v)
3274 if not np.isnan(v):
3275 result.add(v)
3276 except (TypeError, ValueError, OverflowError):
3277 pass
3278 return result
3281def _stringify_na_values(na_values, floatify: bool) -> set[str | float]:
3282 """return a stringified and numeric for these values"""
3283 result: list[str | float] = []
3284 for x in na_values:
3285 result.append(str(x))
3286 result.append(x)
3287 try:
3288 v = float(x)
3290 # we are like 999 here
3291 if v == int(v):
3292 v = int(v)
3293 result.append(f"{v}.0")
3294 result.append(str(v))
3296 if floatify:
3297 result.append(v)
3298 except (TypeError, ValueError, OverflowError):
3299 pass
3300 if floatify:
3301 try:
3302 result.append(int(x))
3303 except (TypeError, ValueError, OverflowError):
3304 pass
3305 return set(result)
3308def _refine_defaults_read(
3309 dialect: str | csv.Dialect | type[csv.Dialect] | None,
3310 delimiter: str | lib.NoDefault | None,
3311 engine: CSVEngine | None,
3312 sep: str | lib.NoDefault | None,
3313 on_bad_lines: str | Callable,
3314 names: Sequence[Hashable] | lib.NoDefault | None,
3315 defaults: dict[str, Any],
3316 dtype_backend: DtypeBackend | lib.NoDefault,
3317 lineterminator: str | bytes | None,
3318):
3319 """Validate/refine default values of input parameters of read_csv, read_table.
3321 Parameters
3322 ----------
3323 dialect : str or csv.Dialect
3324 If provided, this parameter will override values (default or not) for the
3325 following parameters: `delimiter`, `doublequote`, `escapechar`,
3326 `skipinitialspace`, `quotechar`, and `quoting`. If it is necessary to
3327 override values, a ParserWarning will be issued. See csv.Dialect
3328 documentation for more details.
3329 delimiter : str or object
3330 Alias for sep, taking the same sentinel value when not provided.
3331 engine : {'c', 'python'}
3332 Parser engine to use. The C engine is faster while the python engine is
3333 currently more feature-complete.
3334 sep : str or object
3335 A delimiter provided by the user (str) or a sentinel value, i.e.
3336 pandas._libs.lib.no_default.
3337 on_bad_lines : str, callable
3338 An option for handling bad lines or a sentinel value(None).
3339 names : array-like, optional
3340 List of column names to use. If the file contains a header row,
3341 then you should explicitly pass ``header=0`` to override the column names.
3342 Duplicates in this list are not allowed.
3343 defaults: dict
3344 Default values of input parameters.
3345 lineterminator : str, bytes or None
3346 Line terminator passed by the user, if any.
3348 Returns
3349 -------
3350 kwds : dict
3351 Input parameters with correct values.
3352 """
3353 # fix types for sep, delimiter to Union(str, Any)
3354 delim_default = defaults["delimiter"]
3355 kwds: dict[str, Any] = {}
3356 # gh-23761
3357 #
3358 # When a dialect is passed, it overrides any of the overlapping
3359 # parameters passed in directly. We don't want to warn if the
3360 # default parameters were passed in (since it probably means
3361 # that the user didn't pass them in explicitly in the first place).
3362 #
3363 # "delimiter" is the annoying corner case because we alias it to
3364 # "sep" before doing comparison to the dialect values later on.
3365 # Thus, we need a flag to indicate that we need to "override"
3366 # the comparison to dialect values by checking if default values
3367 # for BOTH "delimiter" and "sep" were provided.
3368 if dialect is not None:
3369 kwds["sep_override"] = delimiter is lib.no_default and (
3370 sep is lib.no_default or sep == delim_default
3371 )
3373 if delimiter is not lib.no_default and sep is not lib.no_default:
3374 raise ValueError("Specified a sep and a delimiter; you can only specify one.")
3376 kwds["names"] = None if names is lib.no_default else names
3378 # Alias sep -> delimiter. Both use lib.no_default rather than None as the
3379 # "not passed" sentinel, so an explicit None means sniff (GH#47024).
3380 if delimiter is lib.no_default:
3381 delimiter = sep
3383 if isinstance(lineterminator, (bytes, bytearray)):
3384 # the C engine accepts these; compare on the character it will use
3385 lineterminator = lineterminator.decode("latin-1")
3387 # GH#43528, GH#51801: the C engine silently mis-parses these, as the field
3388 # separator is consumed as a line terminator before it can split a field.
3389 # A custom lineterminator takes over that role, leaving "\n"/"\r" free to
3390 # separate fields.
3391 if delimiter in ("\n", "\r") and lineterminator in (None, delimiter):
3392 raise ValueError(
3393 f"Specified {delimiter!r} as separator or delimiter, but a line "
3394 f"terminator cannot be used as a separator. To parse {delimiter!r} "
3395 f"as a separator, pass a lineterminator other than {delimiter!r} "
3396 "(engine='c' only).",
3397 )
3399 if delimiter is lib.no_default:
3400 # assign default separator value
3401 kwds["delimiter"] = delim_default
3402 else:
3403 kwds["delimiter"] = delimiter
3405 if engine is not None:
3406 kwds["engine_specified"] = True
3407 else:
3408 kwds["engine"] = "c"
3409 kwds["engine_specified"] = False
3411 if on_bad_lines == "error":
3412 kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.BLHM_ERROR
3413 elif on_bad_lines == "warn":
3414 kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.BLHM_WARN
3415 elif on_bad_lines == "skip":
3416 kwds["on_bad_lines"] = ParserBase.BadLineHandleMethod.BLHM_SKIP
3417 elif callable(on_bad_lines):
3418 if engine not in ["python", "pyarrow"]:
3419 raise ValueError(
3420 "on_bad_line can only be a callable function "
3421 "if engine='python' or 'pyarrow'"
3422 )
3423 kwds["on_bad_lines"] = on_bad_lines
3424 else:
3425 raise ValueError(f"Argument {on_bad_lines} is invalid for on_bad_lines")
3427 check_dtype_backend(dtype_backend)
3429 kwds["dtype_backend"] = dtype_backend
3431 return kwds
3434def _extract_dialect(
3435 kwds: dict[str, str | csv.Dialect | type[csv.Dialect]],
3436) -> csv.Dialect | type[csv.Dialect] | None:
3437 """
3438 Extract concrete csv dialect instance.
3440 Returns
3441 -------
3442 csv.Dialect or None
3443 """
3444 if kwds.get("dialect") is None:
3445 return None
3447 dialect = kwds["dialect"]
3448 if isinstance(dialect, str) and dialect in csv.list_dialects():
3449 # get_dialect is typed to return a `_csv.Dialect` for some reason in typeshed
3450 tdialect = cast("csv.Dialect", csv.get_dialect(dialect))
3451 _validate_dialect(tdialect)
3453 else:
3454 _validate_dialect(dialect)
3455 tdialect = cast("csv.Dialect", dialect)
3457 return tdialect
3460MANDATORY_DIALECT_ATTRS = (
3461 "delimiter",
3462 "doublequote",
3463 "escapechar",
3464 "skipinitialspace",
3465 "quotechar",
3466 "quoting",
3467)
3470def _validate_dialect(dialect: csv.Dialect | type[csv.Dialect] | str) -> None:
3471 """
3472 Validate csv dialect instance.
3474 Raises
3475 ------
3476 ValueError
3477 If incorrect dialect is provided.
3478 """
3479 for param in MANDATORY_DIALECT_ATTRS:
3480 if not hasattr(dialect, param):
3481 raise ValueError(f"Invalid dialect {dialect} provided")
3484def _merge_with_dialect_properties(
3485 dialect: csv.Dialect | type[csv.Dialect],
3486 defaults: dict[str, Any],
3487) -> dict[str, Any]:
3488 """
3489 Merge default kwargs in TextFileReader with dialect parameters.
3491 Parameters
3492 ----------
3493 dialect : csv.Dialect
3494 Concrete csv dialect. See csv.Dialect documentation for more details.
3495 defaults : dict
3496 Keyword arguments passed to TextFileReader.
3498 Returns
3499 -------
3500 kwds : dict
3501 Updated keyword arguments, merged with dialect parameters.
3502 """
3503 kwds = defaults.copy()
3505 for param in MANDATORY_DIALECT_ATTRS:
3506 dialect_val = getattr(dialect, param)
3508 parser_default = parser_defaults[param]
3509 provided = kwds.get(param, parser_default)
3511 # Messages for conflicting values between the dialect
3512 # instance and the actual parameters provided.
3513 conflict_msgs = []
3515 # Don't warn if the default parameter was passed in,
3516 # even if it conflicts with the dialect (gh-23761).
3517 if provided not in (parser_default, dialect_val):
3518 msg = (
3519 f"Conflicting values for '{param}': '{provided}' was "
3520 f"provided, but the dialect specifies '{dialect_val}'. "
3521 "Using the dialect-specified value."
3522 )
3524 # Annoying corner case for not warning about
3525 # conflicts between dialect and delimiter parameter.
3526 # Refer to the outer "_read_" function for more info.
3527 if not (param == "delimiter" and kwds.pop("sep_override", False)):
3528 conflict_msgs.append(msg)
3530 if conflict_msgs:
3531 warnings.warn(
3532 "\n\n".join(conflict_msgs), ParserWarning, stacklevel=find_stack_level()
3533 )
3534 kwds[param] = dialect_val
3535 return kwds
3538def _validate_skipfooter(kwds: dict[str, Any]) -> None:
3539 """
3540 Check whether skipfooter is compatible with other kwargs in TextFileReader.
3542 Parameters
3543 ----------
3544 kwds : dict
3545 Keyword arguments passed to TextFileReader.
3547 Raises
3548 ------
3549 ValueError
3550 If skipfooter is not compatible with other parameters.
3551 """
3552 if kwds.get("skipfooter"):
3553 if kwds.get("iterator") or kwds.get("chunksize"):
3554 raise ValueError("'skipfooter' not supported for iteration")
3555 if kwds.get("nrows"):
3556 raise ValueError("'skipfooter' not supported with 'nrows'")