/src/duckdb/extension/parquet/reader/struct_column_reader.cpp
Line | Count | Source |
1 | | #include <stdint.h> |
2 | | #include <algorithm> |
3 | | #include <stdexcept> |
4 | | #include <utility> |
5 | | #include <vector> |
6 | | |
7 | | #include "duckdb/common/vector/struct_vector.hpp" |
8 | | #include "reader/struct_column_reader.hpp" |
9 | | #include "column_reader.hpp" |
10 | | #include "duckdb/common/assert.hpp" |
11 | | #include "duckdb/common/exception.hpp" |
12 | | #include "duckdb/common/optional_idx.hpp" |
13 | | #include "duckdb/common/typedefs.hpp" |
14 | | #include "duckdb/common/types.hpp" |
15 | | #include "duckdb/common/types/validity_mask.hpp" |
16 | | #include "duckdb/common/types/vector.hpp" |
17 | | #include "duckdb/common/unique_ptr.hpp" |
18 | | #include "duckdb/common/vector.hpp" |
19 | | #include "duckdb/common/vector/constant_vector.hpp" |
20 | | #include "duckdb/common/vector/flat_vector.hpp" |
21 | | |
22 | | namespace duckdb_apache { |
23 | | namespace thrift { |
24 | | namespace protocol { |
25 | | class TProtocol; |
26 | | } // namespace protocol |
27 | | } // namespace thrift |
28 | | } // namespace duckdb_apache |
29 | | namespace duckdb_parquet { |
30 | | class ColumnChunk; |
31 | | } // namespace duckdb_parquet |
32 | | |
33 | | namespace duckdb { |
34 | | class ParquetReader; |
35 | | class ThriftFileTransport; |
36 | | struct ParquetColumnSchema; |
37 | | |
38 | | //===--------------------------------------------------------------------===// |
39 | | // Struct Column Reader |
40 | | //===--------------------------------------------------------------------===// |
41 | | StructColumnReader::StructColumnReader(const ParquetReader &reader, const ParquetColumnSchema &schema, |
42 | | vector<unique_ptr<ColumnReader>> child_readers_p) |
43 | 0 | : ColumnReader(reader, schema), child_readers(std::move(child_readers_p)) { |
44 | 0 | D_ASSERT(Type().InternalType() == PhysicalType::STRUCT); |
45 | 0 | for (auto &child : child_readers) { |
46 | 0 | if (child) { |
47 | 0 | child->SetParent(*this); |
48 | 0 | } |
49 | 0 | } |
50 | 0 | } |
51 | | |
52 | 0 | ColumnReader &StructColumnReader::GetChildReader(idx_t child_idx) { |
53 | 0 | if (!child_readers[child_idx]) { |
54 | 0 | throw InternalException("StructColumnReader::GetChildReader(%d) - but this child reader is not set", child_idx); |
55 | 0 | } |
56 | 0 | return *child_readers[child_idx].get(); |
57 | 0 | } |
58 | | |
59 | | void StructColumnReader::InitializeRead(idx_t row_group_idx_p, const vector<ColumnChunk> &columns, |
60 | 0 | TProtocol &protocol_p) { |
61 | 0 | for (auto &child : child_readers) { |
62 | 0 | if (!child) { |
63 | 0 | continue; |
64 | 0 | } |
65 | 0 | child->InitializeRead(row_group_idx_p, columns, protocol_p); |
66 | 0 | } |
67 | 0 | } |
68 | | |
69 | 0 | idx_t StructColumnReader::Read(ColumnReaderInput &input, Vector &result) { |
70 | 0 | auto &struct_entries = StructVector::GetEntries(result); |
71 | 0 | D_ASSERT(StructType::GetChildTypes(Type()).size() == struct_entries.size()); |
72 | |
|
73 | 0 | if (pending_skips > 0) { |
74 | 0 | throw InternalException("StructColumnReader cannot have pending skips"); |
75 | 0 | } |
76 | 0 | auto &num_values = input.num_values; |
77 | 0 | auto &define_out = input.define_out; |
78 | 0 | auto &repeat_out = input.repeat_out; |
79 | | |
80 | | // If the child reader values are all valid, "define_out" may not be initialized at all |
81 | | // So, we just initialize them to all be valid beforehand |
82 | 0 | std::fill_n(define_out, num_values, MaxDefine()); |
83 | |
|
84 | 0 | optional_idx read_count; |
85 | 0 | for (idx_t i = 0; i < child_readers.size(); i++) { |
86 | 0 | auto &child = child_readers[i]; |
87 | 0 | auto &target_vector = struct_entries[i]; |
88 | 0 | if (!child) { |
89 | | // if we are not scanning this vector - set it to NULL |
90 | 0 | ConstantVector::SetNull(target_vector, count_t(num_values)); |
91 | 0 | continue; |
92 | 0 | } |
93 | 0 | ColumnReaderInput child_input(num_values, define_out, repeat_out); |
94 | 0 | auto child_num_values = child->Read(child_input, target_vector); |
95 | 0 | if (!read_count.IsValid()) { |
96 | 0 | read_count = child_num_values; |
97 | 0 | } else if (read_count.GetIndex() != child_num_values) { |
98 | 0 | throw std::runtime_error("Struct child row count mismatch"); |
99 | 0 | } |
100 | 0 | } |
101 | 0 | if (!read_count.IsValid()) { |
102 | 0 | read_count = num_values; |
103 | 0 | } |
104 | | // set the validity mask for this level |
105 | 0 | auto &validity = FlatVector::ValidityMutable(result); |
106 | 0 | for (idx_t i = 0; i < read_count.GetIndex(); i++) { |
107 | 0 | if (define_out[i] < MaxDefine()) { |
108 | 0 | validity.SetInvalid(i); |
109 | 0 | } |
110 | 0 | } |
111 | | // set the parent struct's size so it matches its children (each child reader sets its own size) |
112 | 0 | FlatVector::SetSize(result, count_t(read_count.GetIndex())); |
113 | |
|
114 | 0 | return read_count.GetIndex(); |
115 | 0 | } |
116 | | |
117 | 0 | void StructColumnReader::Skip(idx_t num_values) { |
118 | 0 | for (auto &child : child_readers) { |
119 | 0 | if (!child) { |
120 | 0 | continue; |
121 | 0 | } |
122 | 0 | child->Skip(num_values); |
123 | 0 | } |
124 | 0 | } |
125 | | |
126 | 0 | void StructColumnReader::RegisterPrefetch(ThriftFileTransport &transport, bool allow_merge) { |
127 | 0 | for (auto &child : child_readers) { |
128 | 0 | if (!child) { |
129 | 0 | continue; |
130 | 0 | } |
131 | 0 | child->RegisterPrefetch(transport, allow_merge); |
132 | 0 | } |
133 | 0 | } |
134 | | |
135 | 0 | uint64_t StructColumnReader::TotalCompressedSize() { |
136 | 0 | uint64_t size = 0; |
137 | 0 | for (auto &child : child_readers) { |
138 | 0 | if (!child) { |
139 | 0 | continue; |
140 | 0 | } |
141 | 0 | size += child->TotalCompressedSize(); |
142 | 0 | } |
143 | 0 | return size; |
144 | 0 | } |
145 | | |
146 | 0 | static bool TypeHasExactRowCount(const LogicalType &type) { |
147 | 0 | switch (type.id()) { |
148 | 0 | case LogicalTypeId::LIST: |
149 | 0 | case LogicalTypeId::MAP: |
150 | 0 | return false; |
151 | 0 | case LogicalTypeId::STRUCT: |
152 | 0 | for (auto &kv : StructType::GetChildTypes(type)) { |
153 | 0 | if (TypeHasExactRowCount(kv.second)) { |
154 | 0 | return true; |
155 | 0 | } |
156 | 0 | } |
157 | 0 | return false; |
158 | 0 | default: |
159 | 0 | return true; |
160 | 0 | } |
161 | 0 | } |
162 | | |
163 | 0 | idx_t StructColumnReader::GroupRowsAvailable() { |
164 | 0 | for (auto &child : child_readers) { |
165 | 0 | if (!child) { |
166 | 0 | continue; |
167 | 0 | } |
168 | 0 | if (TypeHasExactRowCount(child->Type())) { |
169 | 0 | return child->GroupRowsAvailable(); |
170 | 0 | } |
171 | 0 | } |
172 | 0 | for (auto &child : child_readers) { |
173 | 0 | if (!child) { |
174 | 0 | continue; |
175 | 0 | } |
176 | 0 | return child->GroupRowsAvailable(); |
177 | 0 | } |
178 | 0 | throw InternalException("No projected columns in struct?"); |
179 | 0 | } |
180 | | |
181 | | } // namespace duckdb |