/src/tdengine/include/common/tdataformat.h
Line | Count | Source |
1 | | /* |
2 | | * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com> |
3 | | * |
4 | | * This program is free software: you can use, redistribute, and/or modify |
5 | | * it under the terms of the GNU Affero General Public License, version 3 |
6 | | * or later ("AGPL"), as published by the Free Software Foundation. |
7 | | * |
8 | | * This program is distributed in the hope that it will be useful, but WITHOUT |
9 | | * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or |
10 | | * FITNESS FOR A PARTICULAR PURPOSE. |
11 | | * |
12 | | * You should have received a copy of the GNU Affero General Public License |
13 | | * along with this program. If not, see <http://www.gnu.org/licenses/>. |
14 | | */ |
15 | | |
16 | | #ifndef _TD_COMMON_DATA_FORMAT_H_ |
17 | | #define _TD_COMMON_DATA_FORMAT_H_ |
18 | | |
19 | | #include "os.h" |
20 | | #include "talgo.h" |
21 | | #include "tarray.h" |
22 | | #include "tbuffer.h" |
23 | | #include "tencode.h" |
24 | | #include "tsimplehash.h" |
25 | | #include "ttypes.h" |
26 | | #include "tutil.h" |
27 | | |
28 | | #ifdef __cplusplus |
29 | | extern "C" { |
30 | | #endif |
31 | | |
32 | | typedef struct SSchema SSchema; |
33 | | typedef struct SSchema2 SSchema2; |
34 | | typedef struct SSchemaExt SSchemaExt; |
35 | | typedef struct SSchemaRsma SSchemaRsma; |
36 | | typedef struct STColumn STColumn; |
37 | | typedef struct STSchema STSchema; |
38 | | typedef struct SRSchema SRSchema; |
39 | | typedef struct SValue SValue; |
40 | | typedef struct SColVal SColVal; |
41 | | typedef struct SRow SRow; |
42 | | typedef struct SRowIter SRowIter; |
43 | | typedef struct STagVal STagVal; |
44 | | typedef struct STag STag; |
45 | | typedef struct SColData SColData; |
46 | | typedef struct SBlobSet SBlobSet; |
47 | | |
48 | | typedef struct SRowKey SRowKey; |
49 | | typedef struct SValueColumn SValueColumn; |
50 | | typedef struct SRowBuildScanInfo SRowBuildScanInfo; |
51 | | |
52 | | #define ROW_BUILD_NONE ((uint8_t)0x1) |
53 | 0 | #define ROW_BUILD_UPDATE ((uint8_t)0x2) |
54 | | #define ROW_BUILD_MERGE ((uint8_t)0x4) |
55 | | |
56 | | typedef struct SBlobValOffset SBlobValOffset; |
57 | | struct SColumnDataAgg; |
58 | | typedef struct SColumnDataAgg *SColumnDataAggPtr; |
59 | | |
60 | 0 | #define HAS_NONE ((uint8_t)0x1) |
61 | 0 | #define HAS_NULL ((uint8_t)0x2) |
62 | 0 | #define HAS_VALUE ((uint8_t)0x4) |
63 | | #define HAS_BLOB ((uint8_t)0x8) |
64 | | |
65 | | // bitmap ================================ |
66 | | const static uint8_t BIT1_MAP[8] = {0b11111110, 0b11111101, 0b11111011, 0b11110111, |
67 | | 0b11101111, 0b11011111, 0b10111111, 0b01111111}; |
68 | | |
69 | | const static uint8_t BIT2_MAP[4] = {0b11111100, 0b11110011, 0b11001111, 0b00111111}; |
70 | | |
71 | 0 | #define ONE ((uint8_t)1) |
72 | 0 | #define THREE ((uint8_t)3) |
73 | 0 | #define DIV_8(i) ((i) >> 3) |
74 | 0 | #define MOD_8(i) ((i)&7) |
75 | 0 | #define DIV_4(i) ((i) >> 2) |
76 | 0 | #define MOD_4(i) ((i)&3) |
77 | 0 | #define MOD_4_TIME_2(i) (MOD_4(i) << 1) |
78 | 0 | #define BIT1_SIZE(n) (DIV_8((n)-1) + 1) |
79 | 0 | #define BIT2_SIZE(n) (DIV_4((n)-1) + 1) |
80 | 0 | #define SET_BIT1(p, i, v) ((p)[DIV_8(i)] = (p)[DIV_8(i)] & BIT1_MAP[MOD_8(i)] | ((v) << MOD_8(i))) |
81 | | #define SET_BIT1_EX(p, i, v) \ |
82 | 0 | do { \ |
83 | 0 | if (MOD_8(i) == 0) { \ |
84 | 0 | (p)[DIV_8(i)] = 0; \ |
85 | 0 | } \ |
86 | 0 | SET_BIT1(p, i, v); \ |
87 | 0 | } while (0) |
88 | 0 | #define GET_BIT1(p, i) (((p)[DIV_8(i)] >> MOD_8(i)) & ONE) |
89 | 0 | #define SET_BIT2(p, i, v) ((p)[DIV_4(i)] = (p)[DIV_4(i)] & BIT2_MAP[MOD_4(i)] | ((v) << MOD_4_TIME_2(i))) |
90 | | #define SET_BIT2_EX(p, i, v) \ |
91 | 0 | do { \ |
92 | 0 | if (MOD_4(i) == 0) { \ |
93 | 0 | (p)[DIV_4(i)] = 0; \ |
94 | 0 | } \ |
95 | 0 | SET_BIT2(p, i, v); \ |
96 | 0 | } while (0) |
97 | 0 | #define GET_BIT2(p, i) (((p)[DIV_4(i)] >> MOD_4_TIME_2(i)) & THREE) |
98 | | |
99 | | // SColVal ================================ |
100 | 0 | #define CV_FLAG_VALUE ((int8_t)0x0) |
101 | 0 | #define CV_FLAG_NONE ((int8_t)0x1) |
102 | 0 | #define CV_FLAG_NULL ((int8_t)0x2) |
103 | | |
104 | 0 | #define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .flag = CV_FLAG_NONE, .value = {.type = (TYPE)}}) |
105 | 0 | #define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .flag = CV_FLAG_NULL, .value = {.type = (TYPE)}}) |
106 | 0 | #define COL_VAL_VALUE(CID, V) ((SColVal){.cid = (CID), .flag = CV_FLAG_VALUE, .value = (V)}) |
107 | | |
108 | 0 | #define COL_VAL_IS_NONE(CV) ((CV)->flag == CV_FLAG_NONE) |
109 | 0 | #define COL_VAL_IS_NULL(CV) ((CV)->flag == CV_FLAG_NULL) |
110 | 0 | #define COL_VAL_IS_VALUE(CV) ((CV)->flag == CV_FLAG_VALUE) |
111 | | |
112 | | // Strategies of merging rows with |
113 | | // same pk in single insert batch. |
114 | | typedef enum { |
115 | | PREFER_NON_NULL = 0, // choose latest non-null value for each column |
116 | | KEEP_CONSISTENCY = 1 // choose latest row |
117 | | } ERowMergeStrategy; |
118 | 0 | #define BSE_SEQUECE_SIZE sizeof(uint64_t) |
119 | | |
120 | | enum { TSDB_DATA_BLOB_VALUE = 0x1, TSDB_DATA_BLOB_EMPTY_VALUE = 0x2, TSDB_DATA_BLOB_NULL_VALUE = 0x4 }; |
121 | | |
122 | | #define tRowGetKey(_pRow, _pKey) \ |
123 | 0 | do { \ |
124 | 0 | (_pKey)->ts = taosGetInt64Aligned(&((_pRow)->ts)); \ |
125 | 0 | (_pKey)->numOfPKs = 0; \ |
126 | 0 | if ((_pRow)->numOfPKs > 0) { \ |
127 | 0 | tRowGetPrimaryKey((_pRow), (_pKey)); \ |
128 | 0 | } \ |
129 | 0 | } while (0) |
130 | | |
131 | | // SValueColumn ================================ |
132 | | typedef struct { |
133 | | int8_t cmprAlg; // filled by caller |
134 | | int8_t type; |
135 | | int32_t dataOriginalSize; |
136 | | int32_t dataCompressedSize; |
137 | | int32_t offsetOriginalSize; |
138 | | int32_t offsetCompressedSize; |
139 | | } SValueColumnCompressInfo; |
140 | | |
141 | | int32_t tValueColumnInit(SValueColumn *valCol); |
142 | | void tValueColumnDestroy(SValueColumn *valCol); |
143 | | void tValueColumnClear(SValueColumn *valCol); |
144 | | int32_t tValueColumnAppend(SValueColumn *valCol, const SValue *value); |
145 | | int32_t tValueColumnUpdate(SValueColumn *valCol, int32_t idx, const SValue *value); |
146 | | int32_t tValueColumnGet(SValueColumn *valCol, int32_t idx, SValue *value); |
147 | | int32_t tValueColumnCompress(SValueColumn *valCol, SValueColumnCompressInfo *info, SBuffer *output, SBuffer *assist); |
148 | | int32_t tValueColumnDecompress(void *input, const SValueColumnCompressInfo *compressInfo, SValueColumn *valCol, |
149 | | SBuffer *buffer); |
150 | | int32_t tValueColumnCompressInfoEncode(const SValueColumnCompressInfo *compressInfo, SBuffer *buffer); |
151 | | int32_t tValueColumnCompressInfoDecode(SBufferReader *reader, SValueColumnCompressInfo *compressInfo); |
152 | | int32_t tValueCompare(const SValue *tv1, const SValue *tv2); |
153 | | |
154 | | // SRow ================================ |
155 | | int32_t tRowBuild(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow, SRowBuildScanInfo *pScanInfo); |
156 | | int32_t tRowBuildWithBlob(SArray *aColVal, const STSchema *pTSchema, SRow **ppRow, SBlobSet *pBlobSet, |
157 | | SRowBuildScanInfo *sinfo); |
158 | | int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal); |
159 | | |
160 | | typedef struct { |
161 | | uint64_t offset; |
162 | | uint32_t len; |
163 | | uint32_t dataOffset; |
164 | | int8_t nextRow; |
165 | | int8_t type; |
166 | | } SBlobValue; |
167 | | |
168 | | typedef struct { |
169 | | uint64_t seq; |
170 | | uint32_t seqOffsetInRow; |
171 | | void *data; |
172 | | int32_t len; |
173 | | int8_t type; |
174 | | } SBlobItem; |
175 | | int32_t tBlobSetCreate(int64_t cap, int8_t type, SBlobSet **ppBlobSet); |
176 | | int32_t tBlobSetPush(SBlobSet *pBlobSet, SBlobItem *pBlobItem, uint64_t *seq, int8_t nextRow); |
177 | | int32_t tBlobSetUpdate(SBlobSet *pBlobSet, uint64_t seq, SBlobItem *pBlobItem); |
178 | | int32_t tBlobSetGet(SBlobSet *pBlobSet, uint64_t seq, SBlobItem *pItem); |
179 | | void tBlobSetDestroy(SBlobSet *pBlowRow); |
180 | | int32_t tBlobSetSize(SBlobSet *pBlobSet); |
181 | | void tBlobSetSwap(SBlobSet *p1, SBlobSet *p2); |
182 | | // int32_t tBlobRowEnd(SBlobSet *pBlobSet); |
183 | | // int32_t tBlobSetRebuild(SBlobSet *pBlobSet, int32_t srow, int32_t nrow, SBlobSet **pNew); |
184 | | |
185 | | int32_t tRowGetBlobSeq(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal, uint64_t *seq); |
186 | | void tRowDestroy(SRow *pRow); |
187 | | int32_t tRowSort(SArray *aRowP); |
188 | | int32_t tRowMerge(SArray *aRowP, STSchema *pTSchema, ERowMergeStrategy strategy); |
189 | | int32_t tRowUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aColData, int32_t nColData, int32_t flag); |
190 | | void tRowGetPrimaryKey(SRow *pRow, SRowKey *key); |
191 | | int32_t tRowKeyCompare(const SRowKey *key1, const SRowKey *key2); |
192 | | void tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc); |
193 | | int32_t tRowSortWithBlob(SArray *aRowP, STSchema *pTSchema, SBlobSet *pBlobSet); |
194 | | int32_t tRowMergeWithBlob(SArray *pRow, STSchema *pTSchema, SBlobSet *pBlobSet, int8_t flag); |
195 | | |
196 | | // SRowIter ================================ |
197 | | int32_t tRowIterOpen(SRow *pRow, STSchema *pTSchema, SRowIter **ppIter); |
198 | | void tRowIterClose(SRowIter **ppIter); |
199 | | SColVal *tRowIterNext(SRowIter *pIter); |
200 | | |
201 | | // STag ================================ |
202 | | int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); |
203 | | int32_t tTagNameCompare(const void *a, const void *b); |
204 | | void tTagFree(STag *pTag); |
205 | | bool tTagIsJson(const void *pTag); |
206 | | bool tTagIsJsonNull(void *tagVal); |
207 | | bool tTagGet(const STag *pTag, STagVal *pTagVal); |
208 | | char *tTagValToData(const STagVal *pTagVal, bool isJson); |
209 | | int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); |
210 | | int32_t tDecodeTag(SDecoder *pDecoder, STag **ppTag); |
211 | | int32_t tTagToValArray(const STag *pTag, SArray **ppArray); |
212 | | void destroyTagVal(void *pTag); |
213 | | void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove |
214 | | int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf, void *charsetCxt); |
215 | | void destroyColVal(void *p); |
216 | | |
217 | | // SColData ================================ |
218 | | typedef struct { |
219 | | uint32_t cmprAlg; // filled by caller |
220 | | int8_t columnFlag; |
221 | | int8_t flag; |
222 | | int8_t dataType; |
223 | | int16_t columnId; |
224 | | int32_t numOfData; |
225 | | int32_t bitmapOriginalSize; |
226 | | int32_t bitmapCompressedSize; |
227 | | int32_t offsetOriginalSize; |
228 | | int32_t offsetCompressedSize; |
229 | | int32_t dataOriginalSize; |
230 | | int32_t dataCompressedSize; |
231 | | } SColDataCompressInfo; |
232 | | |
233 | | typedef void *(*xMallocFn)(void *, int32_t); |
234 | | typedef int32_t (*checkWKBGeometryFn)(const unsigned char *geoWKB, size_t nGeom); |
235 | | typedef int32_t (*initGeosFn)(); |
236 | | |
237 | | void tColDataDestroy(void *ph); |
238 | | void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t cflag); |
239 | | void tColDataClear(SColData *pColData); |
240 | | void tColDataDeepClear(SColData *pColData); |
241 | | int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); |
242 | | int32_t tColDataUpdateValue(SColData *pColData, SColVal *pColVal, bool forward); |
243 | | int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); |
244 | | uint8_t tColDataGetBitValue(const SColData *pColData, int32_t iVal); |
245 | | int32_t tColDataCopy(SColData *pColDataFrom, SColData *pColData, xMallocFn xMalloc, void *arg); |
246 | | void tColDataArrGetRowKey(SColData *aColData, int32_t nColData, int32_t iRow, SRowKey *key); |
247 | | |
248 | | extern void (*tColDataCalcSMA[])(SColData *pColData, SColumnDataAggPtr pAggs); |
249 | | |
250 | | int32_t tColDataCompress(SColData *colData, SColDataCompressInfo *info, SBuffer *output, SBuffer *assist); |
251 | | int32_t tColDataDecompress(void *input, SColDataCompressInfo *info, SColData *colData, SBuffer *assist); |
252 | | |
253 | | // for stmt bind |
254 | | int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind, int32_t buffMaxLen, initGeosFn igeos, |
255 | | checkWKBGeometryFn cgeos); |
256 | | int32_t tColDataSortMerge(SArray **arr); |
257 | | int32_t tColDataSortMergeWithBlob(SArray **arr, SBlobSet *pBlob); |
258 | | |
259 | | // for raw block |
260 | | int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, |
261 | | char *data); |
262 | | |
263 | | int32_t tColDataAddValueByDataBlockWithBlob(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, |
264 | | char *lengthOrbitmap, char *data, void *pBlobSet); |
265 | | // for encode/decode |
266 | | int32_t tEncodeColData(uint8_t version, SEncoder *pEncoder, SColData *pColData); |
267 | | int32_t tDecodeColData(uint8_t version, SDecoder *pDecoder, SColData *pColData, bool jump); |
268 | | int32_t tEncodeRow(SEncoder *pEncoder, SRow *pRow); |
269 | | int32_t tDecodeRow(SDecoder *pDecoder, SRow **ppRow); |
270 | | |
271 | | int32_t tEncodeBlobSet(SEncoder *pEncoder, SBlobSet *pRow); |
272 | | int32_t tDecodeBlobSet(SDecoder *pDecoder, SBlobSet **pBlobSet); |
273 | | |
274 | | // STRUCT ================================ |
275 | | struct STColumn { |
276 | | col_id_t colId; |
277 | | int8_t type; |
278 | | int8_t flags; |
279 | | int32_t bytes; |
280 | | int32_t offset; |
281 | | }; |
282 | | |
283 | | struct STSchema { |
284 | | int32_t numOfCols; |
285 | | int32_t version; |
286 | | int32_t flen; |
287 | | int32_t tlen; |
288 | | STColumn columns[]; |
289 | | }; |
290 | | |
291 | | struct SRSchema { |
292 | | char tbName[TSDB_TABLE_NAME_LEN]; |
293 | | int8_t tbType; |
294 | | int32_t rowSize; |
295 | | int32_t maxBufRows; |
296 | | tb_uid_t tbUid; |
297 | | int64_t interval[2]; |
298 | | func_id_t *funcIds; |
299 | | STSchema *tSchema; |
300 | | void *extSchema; // SExtSchema, for decimal type |
301 | | }; |
302 | | |
303 | 0 | static FORCE_INLINE void tFreeSRSchema(SRSchema **rSchema) { |
304 | 0 | if (rSchema && *rSchema) { |
305 | 0 | taosMemoryFreeClear((*rSchema)->funcIds); |
306 | 0 | taosMemoryFreeClear((*rSchema)->tSchema); |
307 | 0 | taosMemoryFreeClear((*rSchema)->extSchema); |
308 | 0 | taosMemoryFreeClear(*rSchema); |
309 | 0 | } |
310 | 0 | } Unexecuted instantiation: sql-fuzzer.c:tFreeSRSchema Unexecuted instantiation: parser.c:tFreeSRSchema Unexecuted instantiation: parTranslater.c:tFreeSRSchema Unexecuted instantiation: parUtil.c:tFreeSRSchema Unexecuted instantiation: parAstParser.c:tFreeSRSchema Unexecuted instantiation: parAuthenticator.c:tFreeSRSchema Unexecuted instantiation: parCalcConst.c:tFreeSRSchema Unexecuted instantiation: parInsertSql.c:tFreeSRSchema Unexecuted instantiation: parInsertUtil.c:tFreeSRSchema Unexecuted instantiation: taos_lemon_sql.tab.c:tFreeSRSchema Unexecuted instantiation: parAstCreater.c:tFreeSRSchema Unexecuted instantiation: parInsertSml.c:tFreeSRSchema Unexecuted instantiation: planner.c:tFreeSRSchema Unexecuted instantiation: planLogicCreater.c:tFreeSRSchema Unexecuted instantiation: planOptimizer.c:tFreeSRSchema Unexecuted instantiation: planPhysiCreater.c:tFreeSRSchema Unexecuted instantiation: planScaleOut.c:tFreeSRSchema Unexecuted instantiation: planSpliter.c:tFreeSRSchema Unexecuted instantiation: planUtil.c:tFreeSRSchema Unexecuted instantiation: planValidator.c:tFreeSRSchema Unexecuted instantiation: catalog.c:tFreeSRSchema Unexecuted instantiation: ctgAsync.c:tFreeSRSchema Unexecuted instantiation: ctgCache.c:tFreeSRSchema Unexecuted instantiation: ctgDbg.c:tFreeSRSchema Unexecuted instantiation: ctgRemote.c:tFreeSRSchema Unexecuted instantiation: ctgRent.c:tFreeSRSchema Unexecuted instantiation: ctgUtil.c:tFreeSRSchema Unexecuted instantiation: extConnector.c:tFreeSRSchema Unexecuted instantiation: functionMgt.c:tFreeSRSchema Unexecuted instantiation: tudf.c:tFreeSRSchema Unexecuted instantiation: builtins.c:tFreeSRSchema Unexecuted instantiation: builtinsimpl.c:tFreeSRSchema Unexecuted instantiation: thistogram.c:tFreeSRSchema Unexecuted instantiation: tpercentile.c:tFreeSRSchema Unexecuted instantiation: tavgfunction.c:tFreeSRSchema Unexecuted instantiation: tminmax.c:tFreeSRSchema Unexecuted instantiation: tminmaxavx.c:tFreeSRSchema Unexecuted instantiation: filter.c:tFreeSRSchema Unexecuted instantiation: scalar.c:tFreeSRSchema Unexecuted instantiation: sclfunc.c:tFreeSRSchema Unexecuted instantiation: sclvector.c:tFreeSRSchema Unexecuted instantiation: nodesCloneFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesCodeFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesEqualFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesMatchFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesMsgFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesRemotePlanToSQL.c:tFreeSRSchema Unexecuted instantiation: nodesToSQLFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesTraverseFuncs.c:tFreeSRSchema Unexecuted instantiation: nodesUtilFuncs.c:tFreeSRSchema Unexecuted instantiation: extTypeMap.c:tFreeSRSchema Unexecuted instantiation: queryUtil.c:tFreeSRSchema Unexecuted instantiation: querymsg.c:tFreeSRSchema Unexecuted instantiation: trans.c:tFreeSRSchema Unexecuted instantiation: transCli.c:tFreeSRSchema Unexecuted instantiation: transComm.c:tFreeSRSchema Unexecuted instantiation: transSasl.c:tFreeSRSchema Unexecuted instantiation: transSvr.c:tFreeSRSchema Unexecuted instantiation: transTLS.c:tFreeSRSchema Unexecuted instantiation: thttp.c:tFreeSRSchema Unexecuted instantiation: geomFunc.c:tFreeSRSchema Unexecuted instantiation: tconfig.c:tFreeSRSchema Unexecuted instantiation: tdigest.c:tFreeSRSchema Unexecuted instantiation: tlog.c:tFreeSRSchema Unexecuted instantiation: tmempool.c:tFreeSRSchema Unexecuted instantiation: tpagedbuf.c:tFreeSRSchema Unexecuted instantiation: mpDirect.c:tFreeSRSchema Unexecuted instantiation: systable.c:tFreeSRSchema Unexecuted instantiation: tdatablock.c:tFreeSRSchema Unexecuted instantiation: tdataformat.c:tFreeSRSchema Unexecuted instantiation: tglobal.c:tFreeSRSchema Unexecuted instantiation: tmisce.c:tFreeSRSchema Unexecuted instantiation: tname.c:tFreeSRSchema Unexecuted instantiation: tpriv.c:tFreeSRSchema Unexecuted instantiation: ttime.c:tFreeSRSchema Unexecuted instantiation: tvariant.c:tFreeSRSchema Unexecuted instantiation: streamMsg.c:tFreeSRSchema Unexecuted instantiation: streamWindowPlan.c:tFreeSRSchema Unexecuted instantiation: tmsg.c:tFreeSRSchema Unexecuted instantiation: xnode.c:tFreeSRSchema Unexecuted instantiation: tencrypt.c:tFreeSRSchema Unexecuted instantiation: streamJson.c:tFreeSRSchema Unexecuted instantiation: decimal.c:tFreeSRSchema |
311 | | |
312 | | /* |
313 | | * 1. Tuple format: |
314 | | * SRow + [(type, offset) * numOfPKs +] [bit map +] fix-length data + [var-length data] |
315 | | * |
316 | | * 2. K-V format: |
317 | | * SRow + [(type, offset) * numOfPKs +] offset array + ([-]cid [+ data]) * numColsNotNone |
318 | | */ |
319 | | struct SRow { |
320 | | uint8_t flag; |
321 | | uint8_t numOfPKs; |
322 | | uint16_t sver; |
323 | | uint32_t len; |
324 | | TSKEY ts; |
325 | | uint8_t data[]; |
326 | | }; |
327 | | |
328 | | struct SBlobSet { |
329 | | int8_t type; |
330 | | int8_t rowType; |
331 | | SHashObj *pSeqToffset; |
332 | | int64_t seq; |
333 | | int64_t len; |
334 | | int32_t cap; |
335 | | uint8_t compress; |
336 | | SArray *pSeqTable; |
337 | | |
338 | | SArray *pSet; |
339 | | uint8_t *data; |
340 | | }; |
341 | | |
342 | | typedef struct { |
343 | | int8_t type; |
344 | | uint32_t offset; |
345 | | } SPrimaryKeyIndex; |
346 | | |
347 | | #define DATUM_MAX_SIZE 16 |
348 | | |
349 | | struct SValue { |
350 | | union { |
351 | | int64_t val; |
352 | | uint8_t *pData; |
353 | | }; |
354 | | uint32_t nData; |
355 | | int8_t type; |
356 | | }; |
357 | | |
358 | | struct SBlobValOffset { |
359 | | uint64_t seq; |
360 | | uint32_t offset; |
361 | | uint32_t rowNum; |
362 | | }; |
363 | | #define VALUE_GET_DATUM(pVal, type) \ |
364 | 0 | (IS_VAR_DATA_TYPE(type) || type == TSDB_DATA_TYPE_DECIMAL) ? (pVal)->pData : (void *)&(pVal)->val |
365 | | |
366 | 0 | #define VALUE_GET_TRIVIAL_DATUM(pVal) ((pVal)->val) |
367 | 0 | #define VALUE_SET_TRIVIAL_DATUM(pVal, v) (pVal)->val = v |
368 | | |
369 | | void valueSetDatum(SValue *pVal, int8_t type, void *pDatum, uint32_t len); |
370 | | void valueCloneDatum(SValue *pDst, const SValue *pSrc, int8_t type); |
371 | | void valueClearDatum(SValue *pVal, int8_t type); |
372 | | |
373 | | #define TD_MAX_PK_COLS 2 |
374 | | struct SRowKey { |
375 | | TSKEY ts; |
376 | | uint8_t numOfPKs; |
377 | | SValue pks[TD_MAX_PK_COLS]; |
378 | | }; |
379 | | |
380 | | struct SColVal { |
381 | | int16_t cid; |
382 | | int8_t flag; |
383 | | SValue value; |
384 | | }; |
385 | | |
386 | | struct SColData { |
387 | | int16_t cid; |
388 | | int8_t type; |
389 | | int8_t cflag; |
390 | | int32_t numOfNone; // # of none |
391 | | int32_t numOfNull; // # of null |
392 | | int32_t numOfValue; // # of vale |
393 | | int32_t nVal; |
394 | | int8_t flag; |
395 | | uint8_t *pBitMap; |
396 | | int32_t *aOffset; |
397 | | int32_t nData; |
398 | | uint8_t *pData; |
399 | | }; |
400 | | |
401 | | #pragma pack(push, 1) |
402 | | struct STagVal { |
403 | | // char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta |
404 | | union { |
405 | | int16_t cid; |
406 | | char *pKey; |
407 | | }; |
408 | | int8_t type; |
409 | | union { |
410 | | int64_t i64; |
411 | | struct { |
412 | | uint32_t nData; |
413 | | uint8_t *pData; |
414 | | }; |
415 | | }; |
416 | | }; |
417 | | |
418 | 0 | #define TD_TAG_JSON ((int8_t)0x40) // distinguish JSON string and JSON value with the highest bit |
419 | 0 | #define TD_TAG_LARGE ((int8_t)0x20) |
420 | | struct STag { |
421 | | int8_t flags; |
422 | | int16_t len; |
423 | | int16_t nTag; |
424 | | int32_t ver; |
425 | | int8_t idx[]; |
426 | | }; |
427 | | #pragma pack(pop) |
428 | | |
429 | | #if 1 //================================================================================================================================================ |
430 | | // Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap. |
431 | | #define TD_SUPPORT_BITMAP |
432 | | |
433 | | #define STR_TO_VARSTR(x, str) \ |
434 | 0 | do { \ |
435 | 0 | VarDataLenT __len = (VarDataLenT)strlen(str); \ |
436 | 0 | *(VarDataLenT *)(x) = __len; \ |
437 | 0 | (void)memcpy(varDataVal(x), (str), __len); \ |
438 | 0 | } while (0); |
439 | | |
440 | | #define STR_WITH_MAXSIZE_TO_VARSTR(x, str, _maxs) \ |
441 | 0 | do { \ |
442 | 0 | char *_e = stpncpy(varDataVal(x), (str), (_maxs)-VARSTR_HEADER_SIZE); \ |
443 | 0 | varDataSetLen(x, (_e - (x)-VARSTR_HEADER_SIZE)); \ |
444 | 0 | } while (0) |
445 | | |
446 | | #define STR_WITH_SIZE_TO_VARSTR(x, str, _size) \ |
447 | | do { \ |
448 | | *(VarDataLenT *)(x) = (VarDataLenT)(_size); \ |
449 | | (void)memcpy(varDataVal(x), (str), (_size)); \ |
450 | | } while (0); |
451 | | |
452 | | // STSchema ================================ |
453 | | STSchema *tBuildTSchema(SSchema *aSchema, int32_t numOfCols, int32_t version); |
454 | | #define tDestroyTSchema(pTSchema) \ |
455 | 0 | do { \ |
456 | 0 | if (pTSchema) { \ |
457 | 0 | taosMemoryFree(pTSchema); \ |
458 | 0 | pTSchema = NULL; \ |
459 | 0 | } \ |
460 | 0 | } while (0) |
461 | | const STColumn *tTSchemaSearchColumn(const STSchema *pTSchema, int16_t cid); |
462 | | |
463 | | struct SValueColumn { |
464 | | int8_t type; |
465 | | uint32_t numOfValues; |
466 | | SBuffer data; |
467 | | SBuffer offsets; |
468 | | }; |
469 | | |
470 | | typedef struct { |
471 | | int32_t dataType; // filled by caller |
472 | | uint32_t cmprAlg; // filled by caller |
473 | | int32_t originalSize; // filled by caller |
474 | | int32_t compressedSize; |
475 | | } SCompressInfo; |
476 | | |
477 | | int32_t tCompressData(void *input, // input |
478 | | SCompressInfo *info, // compress info |
479 | | void *output, // output |
480 | | int32_t outputSize, // output size |
481 | | SBuffer *buffer // assistant buffer provided by caller, can be NULL |
482 | | ); |
483 | | int32_t tDecompressData(void *input, // input |
484 | | const SCompressInfo *info, // compress info |
485 | | void *output, // output |
486 | | int32_t outputSize, // output size |
487 | | SBuffer *buffer // assistant buffer provided by caller, can be NULL |
488 | | ); |
489 | | int32_t tCompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist); |
490 | | int32_t tDecompressDataToBuffer(void *input, SCompressInfo *info, SBuffer *output, SBuffer *assist); |
491 | | |
492 | | typedef struct { |
493 | | int32_t columnId; |
494 | | int32_t type; |
495 | | TAOS_MULTI_BIND *bind; |
496 | | } SBindInfo; |
497 | | int32_t tRowBuildFromBind(SBindInfo *infos, int32_t numOfInfos, bool infoSorted, const STSchema *pTSchema, |
498 | | SArray *rowArray, bool *pOrdered, bool *pDupTs); |
499 | | |
500 | | // stmt2 binding |
501 | | int32_t tColDataAddValueByBind2(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen); |
502 | | |
503 | | int32_t tColDataAddValueByBind2WithGeos(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, |
504 | | initGeosFn igeos, checkWKBGeometryFn cgeos); |
505 | | |
506 | | int32_t tColDataAddValueByBind2WithBlob(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, |
507 | | SBlobSet *pBlobSet); |
508 | | |
509 | | int32_t tColDataAddValueByBind2WithDecimal(SColData *pColData, TAOS_STMT2_BIND *pBind, int32_t buffMaxLen, |
510 | | uint8_t precision, uint8_t scale); |
511 | | |
512 | | typedef struct { |
513 | | int32_t columnId; |
514 | | int32_t type; |
515 | | int32_t bytes; |
516 | | STypeMod typeMod; |
517 | | TAOS_STMT2_BIND *bind; |
518 | | |
519 | | } SBindInfo2; |
520 | | |
521 | | int32_t tRowBuildFromBind2(SBindInfo2 *infos, int32_t numOfInfos, SSHashObj *parsedCols, bool infoSorted, |
522 | | const STSchema *pTSchema, const SSchemaExt *pSchemaExt, SArray *rowArray, bool *pOrdered, |
523 | | bool *pDupTs); |
524 | | |
525 | | int32_t tRowBuildFromBind2WithBlob(SBindInfo2 *infos, int32_t numOfInfos, SSHashObj *parsedCols, bool infoSorted, |
526 | | const STSchema *pTSchema, const SSchemaExt *pSchemaExt, SArray *rowArray, |
527 | | bool *pOrdered, bool *pDupTs, SBlobSet *pBlobSet); |
528 | | |
529 | | struct SRowBuildScanInfo { |
530 | | int32_t numOfNone; |
531 | | int32_t numOfNull; |
532 | | int32_t numOfValue; |
533 | | int32_t numOfPKs; |
534 | | int8_t flag; |
535 | | |
536 | | // tuple |
537 | | int8_t tupleFlag; |
538 | | SPrimaryKeyIndex tupleIndices[TD_MAX_PK_COLS]; |
539 | | int32_t tuplePKSize; // primary key size |
540 | | int32_t tupleBitmapSize; // bitmap size |
541 | | int32_t tupleFixedSize; // fixed part size |
542 | | int32_t tupleVarSize; // var part size |
543 | | int32_t tupleRowSize; |
544 | | |
545 | | // key-value |
546 | | int8_t kvFlag; |
547 | | SPrimaryKeyIndex kvIndices[TD_MAX_PK_COLS]; |
548 | | int32_t kvMaxOffset; |
549 | | int32_t kvPKSize; // primary key size |
550 | | int32_t kvIndexSize; // offset array size |
551 | | int32_t kvPayloadSize; // payload size |
552 | | int32_t kvRowSize; |
553 | | |
554 | | int8_t hasBlob; |
555 | | int8_t scanType; |
556 | | }; |
557 | | |
558 | | int8_t schemaHasBlob(const STSchema *pSchema); |
559 | | #endif |
560 | | |
561 | | // Append a tag name and its column ID into a tag-name array whose elemSize is |
562 | | // TSDB_COL_NAME_LEN + sizeof(col_id_t). Using taosArrayPush(arr, name) would |
563 | | // copy elemSize bytes from a TSDB_COL_NAME_LEN-byte source, causing an |
564 | | // out-of-bounds read. This helper uses taosArrayReserve to allocate the slot |
565 | | // first, then writes the two fields separately. |
566 | 0 | static FORCE_INLINE int32_t insTagNameAppend(SArray *pTagNames, const char *name, col_id_t colId) { |
567 | 0 | void *slot = taosArrayReserve(pTagNames, 1); |
568 | 0 | if (NULL == slot) return terrno; |
569 | 0 | memcpy(slot, name, TSDB_COL_NAME_LEN); |
570 | 0 | *(col_id_t *)POINTER_SHIFT(slot, TSDB_COL_NAME_LEN) = colId; |
571 | 0 | return TSDB_CODE_SUCCESS; |
572 | 0 | } Unexecuted instantiation: sql-fuzzer.c:insTagNameAppend Unexecuted instantiation: parser.c:insTagNameAppend Unexecuted instantiation: parTranslater.c:insTagNameAppend Unexecuted instantiation: parUtil.c:insTagNameAppend Unexecuted instantiation: parAstParser.c:insTagNameAppend Unexecuted instantiation: parAuthenticator.c:insTagNameAppend Unexecuted instantiation: parCalcConst.c:insTagNameAppend Unexecuted instantiation: parInsertSql.c:insTagNameAppend Unexecuted instantiation: parInsertUtil.c:insTagNameAppend Unexecuted instantiation: taos_lemon_sql.tab.c:insTagNameAppend Unexecuted instantiation: parAstCreater.c:insTagNameAppend Unexecuted instantiation: parInsertSml.c:insTagNameAppend Unexecuted instantiation: planner.c:insTagNameAppend Unexecuted instantiation: planLogicCreater.c:insTagNameAppend Unexecuted instantiation: planOptimizer.c:insTagNameAppend Unexecuted instantiation: planPhysiCreater.c:insTagNameAppend Unexecuted instantiation: planScaleOut.c:insTagNameAppend Unexecuted instantiation: planSpliter.c:insTagNameAppend Unexecuted instantiation: planUtil.c:insTagNameAppend Unexecuted instantiation: planValidator.c:insTagNameAppend Unexecuted instantiation: catalog.c:insTagNameAppend Unexecuted instantiation: ctgAsync.c:insTagNameAppend Unexecuted instantiation: ctgCache.c:insTagNameAppend Unexecuted instantiation: ctgDbg.c:insTagNameAppend Unexecuted instantiation: ctgRemote.c:insTagNameAppend Unexecuted instantiation: ctgRent.c:insTagNameAppend Unexecuted instantiation: ctgUtil.c:insTagNameAppend Unexecuted instantiation: extConnector.c:insTagNameAppend Unexecuted instantiation: functionMgt.c:insTagNameAppend Unexecuted instantiation: tudf.c:insTagNameAppend Unexecuted instantiation: builtins.c:insTagNameAppend Unexecuted instantiation: builtinsimpl.c:insTagNameAppend Unexecuted instantiation: thistogram.c:insTagNameAppend Unexecuted instantiation: tpercentile.c:insTagNameAppend Unexecuted instantiation: tavgfunction.c:insTagNameAppend Unexecuted instantiation: tminmax.c:insTagNameAppend Unexecuted instantiation: tminmaxavx.c:insTagNameAppend Unexecuted instantiation: filter.c:insTagNameAppend Unexecuted instantiation: scalar.c:insTagNameAppend Unexecuted instantiation: sclfunc.c:insTagNameAppend Unexecuted instantiation: sclvector.c:insTagNameAppend Unexecuted instantiation: nodesCloneFuncs.c:insTagNameAppend Unexecuted instantiation: nodesCodeFuncs.c:insTagNameAppend Unexecuted instantiation: nodesEqualFuncs.c:insTagNameAppend Unexecuted instantiation: nodesMatchFuncs.c:insTagNameAppend Unexecuted instantiation: nodesMsgFuncs.c:insTagNameAppend Unexecuted instantiation: nodesRemotePlanToSQL.c:insTagNameAppend Unexecuted instantiation: nodesToSQLFuncs.c:insTagNameAppend Unexecuted instantiation: nodesTraverseFuncs.c:insTagNameAppend Unexecuted instantiation: nodesUtilFuncs.c:insTagNameAppend Unexecuted instantiation: extTypeMap.c:insTagNameAppend Unexecuted instantiation: queryUtil.c:insTagNameAppend Unexecuted instantiation: querymsg.c:insTagNameAppend Unexecuted instantiation: trans.c:insTagNameAppend Unexecuted instantiation: transCli.c:insTagNameAppend Unexecuted instantiation: transComm.c:insTagNameAppend Unexecuted instantiation: transSasl.c:insTagNameAppend Unexecuted instantiation: transSvr.c:insTagNameAppend Unexecuted instantiation: transTLS.c:insTagNameAppend Unexecuted instantiation: thttp.c:insTagNameAppend Unexecuted instantiation: geomFunc.c:insTagNameAppend Unexecuted instantiation: tconfig.c:insTagNameAppend Unexecuted instantiation: tdigest.c:insTagNameAppend Unexecuted instantiation: tlog.c:insTagNameAppend Unexecuted instantiation: tmempool.c:insTagNameAppend Unexecuted instantiation: tpagedbuf.c:insTagNameAppend Unexecuted instantiation: mpDirect.c:insTagNameAppend Unexecuted instantiation: systable.c:insTagNameAppend Unexecuted instantiation: tdatablock.c:insTagNameAppend Unexecuted instantiation: tdataformat.c:insTagNameAppend Unexecuted instantiation: tglobal.c:insTagNameAppend Unexecuted instantiation: tmisce.c:insTagNameAppend Unexecuted instantiation: tname.c:insTagNameAppend Unexecuted instantiation: tpriv.c:insTagNameAppend Unexecuted instantiation: ttime.c:insTagNameAppend Unexecuted instantiation: tvariant.c:insTagNameAppend Unexecuted instantiation: streamMsg.c:insTagNameAppend Unexecuted instantiation: streamWindowPlan.c:insTagNameAppend Unexecuted instantiation: tmsg.c:insTagNameAppend Unexecuted instantiation: xnode.c:insTagNameAppend Unexecuted instantiation: tencrypt.c:insTagNameAppend Unexecuted instantiation: streamJson.c:insTagNameAppend Unexecuted instantiation: decimal.c:insTagNameAppend |
573 | | |
574 | | #ifdef __cplusplus |
575 | | } |
576 | | #endif |
577 | | |
578 | | #endif /*_TD_COMMON_DATA_FORMAT_H_*/ |