/src/duckdb/extension/parquet/include/parquet_column_schema.hpp
Line | Count | Source |
1 | | //===----------------------------------------------------------------------===// |
2 | | // DuckDB |
3 | | // |
4 | | // parquet_column_schema.hpp |
5 | | // |
6 | | // |
7 | | //===----------------------------------------------------------------------===// |
8 | | #pragma once |
9 | | |
10 | | #include <stdint.h> |
11 | | #include <string> |
12 | | |
13 | | #include "duckdb.hpp" |
14 | | #include "parquet_types.h" |
15 | | #include "duckdb/common/optional_idx.hpp" |
16 | | #include "duckdb/common/string.hpp" |
17 | | #include "duckdb/common/typedefs.hpp" |
18 | | #include "duckdb/common/types.hpp" |
19 | | #include "duckdb/common/unique_ptr.hpp" |
20 | | #include "duckdb/common/vector.hpp" |
21 | | #include "duckdb/storage/statistics/base_statistics.hpp" |
22 | | |
23 | | namespace duckdb { |
24 | | |
25 | | using namespace duckdb_parquet; // NOLINT |
26 | | |
27 | | using duckdb_parquet::ConvertedType; |
28 | | using duckdb_parquet::FieldRepetitionType; |
29 | | using duckdb_parquet::SchemaElement; |
30 | | |
31 | | using duckdb_parquet::FileMetaData; |
32 | | struct ParquetOptions; |
33 | | |
34 | | enum class ParquetColumnSchemaType { COLUMN, FILE_ROW_NUMBER, EXPRESSION, VARIANT, GEOMETRY }; |
35 | | |
36 | | enum class ParquetExtraTypeInfo { |
37 | | NONE, |
38 | | IMPALA_TIMESTAMP, |
39 | | UNIT_NS, |
40 | | UNIT_MS, |
41 | | UNIT_MICROS, |
42 | | DECIMAL_BYTE_ARRAY, |
43 | | DECIMAL_INT32, |
44 | | DECIMAL_INT64, |
45 | | FLOAT16 |
46 | | }; |
47 | | |
48 | | struct ParquetColumnSchema { |
49 | | public: |
50 | 0 | ParquetColumnSchema() = default; |
51 | 0 | ParquetColumnSchema(ParquetColumnSchema &&other) = default; |
52 | 0 | ParquetColumnSchema(const ParquetColumnSchema &other) = default; |
53 | 0 | ParquetColumnSchema &operator=(ParquetColumnSchema &&other) = default; |
54 | | |
55 | | public: |
56 | | //! Writer constructors |
57 | | static ParquetColumnSchema FromLogicalType(const string &name, const LogicalType &type, idx_t max_define, |
58 | | idx_t max_repeat, idx_t column_index, |
59 | | duckdb_parquet::FieldRepetitionType::type repetition_type, |
60 | | bool allow_geometry, |
61 | | ParquetColumnSchemaType schema_type = ParquetColumnSchemaType::COLUMN); |
62 | | |
63 | | public: |
64 | | //! Reader constructors |
65 | | static ParquetColumnSchema FromSchemaElement(const SchemaElement &element, idx_t max_define, idx_t max_repeat, |
66 | | idx_t schema_index, idx_t column_index, ParquetColumnSchemaType type, |
67 | | const ParquetOptions &options); |
68 | | static ParquetColumnSchema FromParentSchema(ParquetColumnSchema parent, LogicalType result_type, |
69 | | ParquetColumnSchemaType schema_type); |
70 | | static ParquetColumnSchema FromChildSchemas(const string &name, const LogicalType &type, idx_t max_define, |
71 | | idx_t max_repeat, idx_t schema_index, idx_t column_index, |
72 | | vector<ParquetColumnSchema> &&children, |
73 | | ParquetColumnSchemaType schema_type = ParquetColumnSchemaType::COLUMN); |
74 | | static ParquetColumnSchema FileRowNumber(); |
75 | | |
76 | | public: |
77 | | unique_ptr<BaseStatistics> Stats(const FileMetaData &file_meta_data, const ParquetOptions &parquet_options, |
78 | | idx_t row_group_idx_p, const vector<duckdb_parquet::ColumnChunk> &columns) const; |
79 | | |
80 | | public: |
81 | | void SetSchemaIndex(idx_t schema_idx); |
82 | | |
83 | | public: |
84 | | string name; |
85 | | idx_t max_define; |
86 | | idx_t max_repeat; |
87 | | //! Populated by FinalizeSchema if used in the parquet_writer path |
88 | | optional_idx schema_index; |
89 | | idx_t column_index; |
90 | | ParquetColumnSchemaType schema_type; |
91 | | LogicalType type; |
92 | | optional_idx parent_schema_index; |
93 | | uint32_t type_length = 0; |
94 | | uint32_t type_scale = 0; |
95 | | duckdb_parquet::Type::type parquet_type = duckdb_parquet::Type::INT32; |
96 | | ParquetExtraTypeInfo type_info = ParquetExtraTypeInfo::NONE; |
97 | | vector<ParquetColumnSchema> children; |
98 | | optional_idx field_id; |
99 | | //! Whether a column is nullable or not |
100 | | duckdb_parquet::FieldRepetitionType::type repetition_type = duckdb_parquet::FieldRepetitionType::OPTIONAL; |
101 | | //! Whether the column can be recognized as a GEOMETRY type |
102 | | bool allow_geometry = false; |
103 | | }; |
104 | | |
105 | | } // namespace duckdb |