chore: solve conflicts

This commit is contained in:
kailixu 2024-04-17 10:52:34 +08:00
commit 74afb8deae
183 changed files with 38936 additions and 6357 deletions

View File

@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2
date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 600 ''' + extra_param + '''
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + '''
'''
}
}

View File

@ -57,6 +57,7 @@ extern "C" {
#define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs"
#define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_INS_TABLE_ENCRYPTIONS "ins_encryptions"
#define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas"

View File

@ -414,6 +414,18 @@ typedef struct STUidTagInfo {
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime);
#define TSMA_RES_STB_POSTFIX "_tsma_res_stb_"
#define MD5_OUTPUT_LEN 32
#define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname
static inline bool isTsmaResSTb(const char* stbName) {
const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX);
if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) {
return true;
}
return false;
}
#ifdef __cplusplus
}
#endif

View File

@ -37,6 +37,7 @@ typedef struct SBlockOrderInfo {
#define NBIT (3u)
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
#define CharPos(r_) ((r_) >> NBIT)
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
@ -192,12 +193,14 @@ int32_t getJsonValueLen(const char* data);
int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue);
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows);
int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
uint32_t numOfRows, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
const SColumnInfoData* pSource, int32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo);
int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows);
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, bool asc);
@ -210,6 +213,8 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows);
void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows);
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
int32_t pageSize);
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
@ -234,7 +239,10 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
void blockDataCleanup(SSDataBlock* pDataBlock);
void blockDataReset(SSDataBlock* pDataBlock);
void blockDataEmpty(SSDataBlock* pDataBlock);
int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
bool clearPayload);
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize);

View File

@ -221,6 +221,8 @@ extern char tsSmlTsDefaultName[];
extern int32_t tmqMaxTopicNum;
extern int32_t tmqRowSize;
extern int32_t tsMaxTsmaNum;
extern int32_t tsMaxTsmaCalcDelay;
// wal
extern int64_t tsWalFsyncDataSizeLimit;

View File

@ -110,6 +110,7 @@ enum {
HEARTBEAT_KEY_TMQ,
HEARTBEAT_KEY_DYN_VIEW,
HEARTBEAT_KEY_VIEWINFO,
HEARTBEAT_KEY_TSMA,
};
typedef enum _mgmt_table {
@ -145,6 +146,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_TSMAS,
TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_GRANTS_FULL,
@ -246,7 +248,9 @@ typedef enum ENodeType {
QUERY_NODE_EVENT_WINDOW,
QUERY_NODE_HINT,
QUERY_NODE_VIEW,
QUERY_NODE_WINDOW_OFFSET,
QUERY_NODE_COUNT_WINDOW,
QUERY_NODE_TSMA_OPTIONS,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
@ -369,6 +373,10 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_GRANTS_LOGS_STMT,
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_ENCRYPTIONS_STMT,
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -441,9 +449,9 @@ typedef enum ENodeType {
} ENodeType;
typedef struct {
int32_t vgId;
char* dbFName;
char* tbName;
int32_t vgId;
const char* dbFName;
const char* tbName;
} SBuildTableInput;
typedef struct {
@ -1400,8 +1408,14 @@ int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp
void tFreeSDnodeListRsp(SDnodeListRsp* pRsp);
typedef struct {
SUseDbRsp* useDbRsp;
SDbCfgRsp* cfgRsp;
SArray* pTsmas; // SArray<STableTSMAInfo*>
} STableTSMAInfoRsp;
typedef struct {
SUseDbRsp* useDbRsp;
SDbCfgRsp* cfgRsp;
STableTSMAInfoRsp* pTsmaRsp;
int32_t dbTsmaVersion;
} SDbHbRsp;
typedef struct {
@ -2603,6 +2617,7 @@ typedef struct {
SArray* pVgroupVerList;
// 3.3.0.0
SArray* pCols; // array of SField
int64_t smaId;
} SCMCreateStreamReq;
typedef struct {
@ -3567,6 +3582,10 @@ typedef struct {
char* ast;
int64_t deleteMark;
int64_t lastTs;
int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0
SArray* pVgroupVerList;
int8_t recursiveTsma;
char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma
} SMCreateSmaReq;
int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
@ -4190,6 +4209,104 @@ typedef struct {
int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp);
int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
void tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name
bool fetchingWithTsmaName; // if we are fetching with tsma name
}STableTSMAInfoReq;
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq);
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq);
typedef struct {
int32_t funcId;
col_id_t colId;
} STableTSMAFuncInfo;
typedef struct {
char name[TSDB_TABLE_NAME_LEN];
uint64_t tsmaId;
char targetTb[TSDB_TABLE_NAME_LEN];
char targetDbFName[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t suid;
uint64_t destTbUid;
uint64_t dbId;
int32_t version;
int64_t interval;
int8_t unit;
SArray* pFuncs; // SArray<STableTSMAFuncInfo>
SArray* pTags; // SArray<SSchema>
SArray* pUsedCols; // SArray<SSchema>
char* ast;
int64_t streamUid;
int64_t reqTs;
int64_t rspTs;
int64_t delayDuration; // ms
bool fillHistoryFinished;
} STableTSMAInfo;
int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAInfoRsp* pRsp);
int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRsp* pRsp);
int32_t tCloneTbTSMAInfo(STableTSMAInfo* pInfo, STableTSMAInfo** pRes);
void tFreeTableTSMAInfo(void* p);
void tFreeAndClearTableTSMAInfo(void* p);
void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp* pRsp);
#define STSMAHbRsp STableTSMAInfoRsp
#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp
#define tDeserializeTSMAHbRsp tDeserializeTableTSMAInfoRsp
#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp
typedef struct SStreamProgressReq {
int64_t streamId;
int32_t vgId;
int32_t fetchIdx;
int32_t subFetchIdx;
} SStreamProgressReq;
int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProgressReq* pReq);
int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgressReq* pReq);
typedef struct SStreamProgressRsp {
int64_t streamId;
int32_t vgId;
bool fillHisFinished;
int64_t progressDelay;
int32_t fetchIdx;
int32_t subFetchIdx;
} SStreamProgressRsp;
int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProgressRsp* pRsp);
int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgressRsp* pRsp);
typedef struct SDropCtbWithTsmaSingleVgReq {
SVgroupInfo vgInfo;
SArray* pTbs; // SVDropTbReq
} SMDropTbReqsOnSingleVg;
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder* pEncoder, const SMDropTbReqsOnSingleVg* pReq);
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* pReq);
void tFreeSMDropTbReqOnSingleVg(void* p);
typedef struct SDropTbsReq {
SArray* pVgReqs; // SMDropTbReqsOnSingleVg
} SMDropTbsReq;
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq);
int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq);
void tFreeSMDropTbsReq(void*);
typedef struct SVFetchTtlExpiredTbsRsp {
SArray* pExpiredTbs; // SVDropTbReq
int32_t vgId;
} SVFetchTtlExpiredTbsRsp;
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredTbsRsp* pRsp);
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp* pRsp);
void tFreeFetchTtlExpiredTbsRsp(void* p);
#pragma pack(pop)

View File

@ -225,6 +225,12 @@
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TSMA, "create-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TSMA, "drop-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STB_DROP, "drop-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_TSMA, "get-table-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TSMA, "get-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TB_WITH_TSMA, "drop-tb-with-tsma", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8
@ -269,6 +275,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
@ -312,6 +319,8 @@
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESUME, "stream-task-resume", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8
@ -359,6 +368,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8

View File

@ -79,7 +79,7 @@ int64_t taosTimeGetIntervalEnd(int64_t ts, const SInterval* pInterval);
int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, int32_t order);
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision);
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow);
int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce();
@ -88,6 +88,7 @@ char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);

View File

@ -209,174 +209,185 @@
#define TK_QTAGS 191
#define TK_AS 192
#define TK_SYSTEM 193
#define TK_INDEX 194
#define TK_FUNCTION 195
#define TK_INTERVAL 196
#define TK_COUNT 197
#define TK_LAST_ROW 198
#define TK_META 199
#define TK_ONLY 200
#define TK_TOPIC 201
#define TK_CONSUMER 202
#define TK_GROUP 203
#define TK_DESC 204
#define TK_DESCRIBE 205
#define TK_RESET 206
#define TK_QUERY 207
#define TK_CACHE 208
#define TK_EXPLAIN 209
#define TK_ANALYZE 210
#define TK_VERBOSE 211
#define TK_NK_BOOL 212
#define TK_RATIO 213
#define TK_NK_FLOAT 214
#define TK_OUTPUTTYPE 215
#define TK_AGGREGATE 216
#define TK_BUFSIZE 217
#define TK_LANGUAGE 218
#define TK_REPLACE 219
#define TK_STREAM 220
#define TK_INTO 221
#define TK_PAUSE 222
#define TK_RESUME 223
#define TK_TRIGGER 224
#define TK_AT_ONCE 225
#define TK_WINDOW_CLOSE 226
#define TK_IGNORE 227
#define TK_EXPIRED 228
#define TK_FILL_HISTORY 229
#define TK_UPDATE 230
#define TK_SUBTABLE 231
#define TK_UNTREATED 232
#define TK_KILL 233
#define TK_CONNECTION 234
#define TK_TRANSACTION 235
#define TK_BALANCE 236
#define TK_VGROUP 237
#define TK_LEADER 238
#define TK_MERGE 239
#define TK_REDISTRIBUTE 240
#define TK_SPLIT 241
#define TK_DELETE 242
#define TK_INSERT 243
#define TK_NK_BIN 244
#define TK_NK_HEX 245
#define TK_NULL 246
#define TK_NK_QUESTION 247
#define TK_NK_ALIAS 248
#define TK_NK_ARROW 249
#define TK_ROWTS 250
#define TK_QSTART 251
#define TK_QEND 252
#define TK_QDURATION 253
#define TK_WSTART 254
#define TK_WEND 255
#define TK_WDURATION 256
#define TK_IROWTS 257
#define TK_ISFILLED 258
#define TK_CAST 259
#define TK_NOW 260
#define TK_TODAY 261
#define TK_TIMEZONE 262
#define TK_CLIENT_VERSION 263
#define TK_SERVER_VERSION 264
#define TK_SERVER_STATUS 265
#define TK_CURRENT_USER 266
#define TK_CASE 267
#define TK_WHEN 268
#define TK_THEN 269
#define TK_ELSE 270
#define TK_BETWEEN 271
#define TK_IS 272
#define TK_NK_LT 273
#define TK_NK_GT 274
#define TK_NK_LE 275
#define TK_NK_GE 276
#define TK_NK_NE 277
#define TK_MATCH 278
#define TK_NMATCH 279
#define TK_CONTAINS 280
#define TK_IN 281
#define TK_JOIN 282
#define TK_INNER 283
#define TK_SELECT 284
#define TK_NK_HINT 285
#define TK_DISTINCT 286
#define TK_WHERE 287
#define TK_PARTITION 288
#define TK_BY 289
#define TK_SESSION 290
#define TK_STATE_WINDOW 291
#define TK_EVENT_WINDOW 292
#define TK_COUNT_WINDOW 293
#define TK_SLIDING 294
#define TK_FILL 295
#define TK_VALUE 296
#define TK_VALUE_F 297
#define TK_NONE 298
#define TK_PREV 299
#define TK_NULL_F 300
#define TK_LINEAR 301
#define TK_NEXT 302
#define TK_HAVING 303
#define TK_RANGE 304
#define TK_EVERY 305
#define TK_ORDER 306
#define TK_SLIMIT 307
#define TK_SOFFSET 308
#define TK_LIMIT 309
#define TK_OFFSET 310
#define TK_ASC 311
#define TK_NULLS 312
#define TK_ABORT 313
#define TK_AFTER 314
#define TK_ATTACH 315
#define TK_BEFORE 316
#define TK_BEGIN 317
#define TK_BITAND 318
#define TK_BITNOT 319
#define TK_BITOR 320
#define TK_BLOCKS 321
#define TK_CHANGE 322
#define TK_COMMA 323
#define TK_CONCAT 324
#define TK_CONFLICT 325
#define TK_COPY 326
#define TK_DEFERRED 327
#define TK_DELIMITERS 328
#define TK_DETACH 329
#define TK_DIVIDE 330
#define TK_DOT 331
#define TK_EACH 332
#define TK_FAIL 333
#define TK_FILE 334
#define TK_FOR 335
#define TK_GLOB 336
#define TK_ID 337
#define TK_IMMEDIATE 338
#define TK_IMPORT 339
#define TK_INITIALLY 340
#define TK_INSTEAD 341
#define TK_ISNULL 342
#define TK_MODULES 343
#define TK_NK_BITNOT 344
#define TK_NK_SEMI 345
#define TK_NOTNULL 346
#define TK_OF 347
#define TK_PLUS 348
#define TK_PRIVILEGE 349
#define TK_RAISE 350
#define TK_RESTRICT 351
#define TK_ROW 352
#define TK_SEMI 353
#define TK_STAR 354
#define TK_STATEMENT 355
#define TK_STRICT 356
#define TK_STRING 357
#define TK_TIMES 358
#define TK_VALUES 359
#define TK_VARIABLE 360
#define TK_WAL 361
#define TK_TSMA 194
#define TK_INTERVAL 195
#define TK_RECURSIVE 196
#define TK_TSMAS 197
#define TK_FUNCTION 198
#define TK_INDEX 199
#define TK_COUNT 200
#define TK_LAST_ROW 201
#define TK_META 202
#define TK_ONLY 203
#define TK_TOPIC 204
#define TK_CONSUMER 205
#define TK_GROUP 206
#define TK_DESC 207
#define TK_DESCRIBE 208
#define TK_RESET 209
#define TK_QUERY 210
#define TK_CACHE 211
#define TK_EXPLAIN 212
#define TK_ANALYZE 213
#define TK_VERBOSE 214
#define TK_NK_BOOL 215
#define TK_RATIO 216
#define TK_NK_FLOAT 217
#define TK_OUTPUTTYPE 218
#define TK_AGGREGATE 219
#define TK_BUFSIZE 220
#define TK_LANGUAGE 221
#define TK_REPLACE 222
#define TK_STREAM 223
#define TK_INTO 224
#define TK_PAUSE 225
#define TK_RESUME 226
#define TK_TRIGGER 227
#define TK_AT_ONCE 228
#define TK_WINDOW_CLOSE 229
#define TK_IGNORE 230
#define TK_EXPIRED 231
#define TK_FILL_HISTORY 232
#define TK_UPDATE 233
#define TK_SUBTABLE 234
#define TK_UNTREATED 235
#define TK_KILL 236
#define TK_CONNECTION 237
#define TK_TRANSACTION 238
#define TK_BALANCE 239
#define TK_VGROUP 240
#define TK_LEADER 241
#define TK_MERGE 242
#define TK_REDISTRIBUTE 243
#define TK_SPLIT 244
#define TK_DELETE 245
#define TK_INSERT 246
#define TK_NK_BIN 247
#define TK_NK_HEX 248
#define TK_NULL 249
#define TK_NK_QUESTION 250
#define TK_NK_ALIAS 251
#define TK_NK_ARROW 252
#define TK_ROWTS 253
#define TK_QSTART 254
#define TK_QEND 255
#define TK_QDURATION 256
#define TK_WSTART 257
#define TK_WEND 258
#define TK_WDURATION 259
#define TK_IROWTS 260
#define TK_ISFILLED 261
#define TK_CAST 262
#define TK_NOW 263
#define TK_TODAY 264
#define TK_TIMEZONE 265
#define TK_CLIENT_VERSION 266
#define TK_SERVER_VERSION 267
#define TK_SERVER_STATUS 268
#define TK_CURRENT_USER 269
#define TK_CASE 270
#define TK_WHEN 271
#define TK_THEN 272
#define TK_ELSE 273
#define TK_BETWEEN 274
#define TK_IS 275
#define TK_NK_LT 276
#define TK_NK_GT 277
#define TK_NK_LE 278
#define TK_NK_GE 279
#define TK_NK_NE 280
#define TK_MATCH 281
#define TK_NMATCH 282
#define TK_CONTAINS 283
#define TK_IN 284
#define TK_JOIN 285
#define TK_INNER 286
#define TK_LEFT 287
#define TK_RIGHT 288
#define TK_OUTER 289
#define TK_SEMI 290
#define TK_ANTI 291
#define TK_ASOF 292
#define TK_WINDOW 293
#define TK_WINDOW_OFFSET 294
#define TK_JLIMIT 295
#define TK_SELECT 296
#define TK_NK_HINT 297
#define TK_DISTINCT 298
#define TK_WHERE 299
#define TK_PARTITION 300
#define TK_BY 301
#define TK_SESSION 302
#define TK_STATE_WINDOW 303
#define TK_EVENT_WINDOW 304
#define TK_COUNT_WINDOW 305
#define TK_SLIDING 306
#define TK_FILL 307
#define TK_VALUE 308
#define TK_VALUE_F 309
#define TK_NONE 310
#define TK_PREV 311
#define TK_NULL_F 312
#define TK_LINEAR 313
#define TK_NEXT 314
#define TK_HAVING 315
#define TK_RANGE 316
#define TK_EVERY 317
#define TK_ORDER 318
#define TK_SLIMIT 319
#define TK_SOFFSET 320
#define TK_LIMIT 321
#define TK_OFFSET 322
#define TK_ASC 323
#define TK_NULLS 324
#define TK_ABORT 325
#define TK_AFTER 326
#define TK_ATTACH 327
#define TK_BEFORE 328
#define TK_BEGIN 329
#define TK_BITAND 330
#define TK_BITNOT 331
#define TK_BITOR 332
#define TK_BLOCKS 333
#define TK_CHANGE 334
#define TK_COMMA 335
#define TK_CONCAT 336
#define TK_CONFLICT 337
#define TK_COPY 338
#define TK_DEFERRED 339
#define TK_DELIMITERS 340
#define TK_DETACH 341
#define TK_DIVIDE 342
#define TK_DOT 343
#define TK_EACH 344
#define TK_FAIL 345
#define TK_FILE 346
#define TK_FOR 347
#define TK_GLOB 348
#define TK_ID 349
#define TK_IMMEDIATE 350
#define TK_IMPORT 351
#define TK_INITIALLY 352
#define TK_INSTEAD 353
#define TK_ISNULL 354
#define TK_MODULES 355
#define TK_NK_BITNOT 356
#define TK_NK_SEMI 357
#define TK_NOTNULL 358
#define TK_OF 359
#define TK_PLUS 360
#define TK_PRIVILEGE 361
#define TK_RAISE 362
#define TK_RESTRICT 363
#define TK_ROW 364
#define TK_STAR 365
#define TK_STATEMENT 366
#define TK_STRICT 367
#define TK_STRING 368
#define TK_TIMES 369
#define TK_VALUES 370
#define TK_VARIABLE 371
#define TK_WAL 372
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601
@ -390,6 +401,8 @@
#define TK_PARTITION_FIRST 609
#define TK_PARA_TABLES_SORT 610
#define TK_SMALLDATA_TS_SORT 611
#define TK_HASH_JOIN 612
#define TK_SKIP_TSMA 613
#define TK_NK_NIL 65535

View File

@ -94,6 +94,8 @@ typedef struct SCatalogReq {
SArray* pTableCfg; // element is SNAME
SArray* pTableTag; // element is SNAME
SArray* pView; // element is STablesReq
SArray* pTableTSMAs; // element is STablesReq
SArray* pTSMAs; // element is STablesReq
bool qNodeRequired; // valid qnode
bool dNodeRequired; // valid dnode
bool svrVerRequired;
@ -122,6 +124,8 @@ typedef struct SMetaData {
SArray* pTableTag; // pRes = SArray<STagVal>*
SArray* pDnodeList; // pRes = SArray<SEpSet>*
SArray* pView; // pRes = SViewMeta*
SArray* pTableTsmas; // pRes = SArray<STableTSMAInfo*>
SArray* pTsmas; // pRes = SArray<STableTSMAInfo*>
SMetaRes* pSvrVer; // pRes = char*
} SMetaData;
@ -130,9 +134,11 @@ typedef struct SCatalogCfg {
uint32_t maxViewCacheNum;
uint32_t maxDBCacheNum;
uint32_t maxUserCacheNum;
uint32_t maxTSMACacheNum;
uint32_t dbRentSec;
uint32_t stbRentSec;
uint32_t viewRentSec;
uint32_t tsmaRentSec;
} SCatalogCfg;
typedef struct SSTableVersion {
@ -152,6 +158,7 @@ typedef struct SDbCacheInfo {
int32_t cfgVersion;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
int64_t stateTs;
int32_t tsmaVersion;
} SDbCacheInfo;
typedef struct SDynViewVersion {
@ -167,6 +174,14 @@ typedef struct SViewVersion {
int32_t version;
} SViewVersion;
typedef struct STSMAVersion {
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
char name[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t tsmaId;
int32_t version;
} STSMAVersion;
typedef struct STbSVersion {
char* tbFName;
@ -342,6 +357,8 @@ int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* n
int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num);
int32_t catalogGetExpiredTsmas(SCatalog* pCtg, STSMAVersion** tsmas, uint32_t* num);
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg);
int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo);
@ -388,6 +405,16 @@ int32_t ctgdEnableDebug(char* option, bool enable);
int32_t ctgdHandleDbgCommand(char* command);
int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma, int32_t tsmaVersion);
int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma);
int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma);
int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma);
/**
* Destroy catalog and relase all resources
*/

View File

@ -88,6 +88,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_LTRIM,
FUNCTION_TYPE_RTRIM,
FUNCTION_TYPE_SUBSTR,
FUNCTION_TYPE_MD5,
// conversion function
FUNCTION_TYPE_CAST = 2000,
@ -164,6 +165,18 @@ typedef enum EFunctionType {
FUNCTION_TYPE_STDDEV_MERGE,
FUNCTION_TYPE_IRATE_PARTIAL,
FUNCTION_TYPE_IRATE_MERGE,
FUNCTION_TYPE_AVG_STATE,
FUNCTION_TYPE_AVG_STATE_MERGE,
FUNCTION_TYPE_FIRST_STATE,
FUNCTION_TYPE_FIRST_STATE_MERGE,
FUNCTION_TYPE_LAST_STATE,
FUNCTION_TYPE_LAST_STATE_MERGE,
FUNCTION_TYPE_SPREAD_STATE,
FUNCTION_TYPE_SPREAD_STATE_MERGE,
FUNCTION_TYPE_STDDEV_STATE,
FUNCTION_TYPE_STDDEV_STATE_MERGE,
FUNCTION_TYPE_HYPERLOGLOG_STATE,
FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE,
// geometry functions
FUNCTION_TYPE_GEOM_FROM_TEXT = 4250,
@ -238,6 +251,7 @@ bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId);
bool fmIsGroupKeyFunc(int32_t funcId);
bool fmIsBlockDistFunc(int32_t funcId);
bool fmIsIgnoreNullFunc(int32_t funcId);
bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId);
@ -270,6 +284,13 @@ bool fmIsInvertible(int32_t funcId);
char* fmGetFuncName(int32_t funcId);
bool fmIsTSMASupportedFunc(func_id_t funcId);
int32_t fmCreateStateFuncs(SNodeList* pFuncs);
int32_t fmCreateStateMergeFuncs(SNodeList* pFuncs);
int32_t fmGetFuncId(const char* name);
bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId);
bool fmIsCountLikeFunc(int32_t funcId);
#ifdef __cplusplus
}
#endif

View File

@ -215,11 +215,13 @@ typedef struct SDropTableClause {
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists;
SArray* pTsmas;
} SDropTableClause;
typedef struct SDropTableStmt {
ENodeType type;
SNodeList* pTables;
bool withTsma;
} SDropTableStmt;
typedef struct SDropSuperTableStmt {
@ -592,6 +594,34 @@ typedef struct SSplitVgroupStmt {
int32_t vgId;
} SSplitVgroupStmt;
typedef struct STSMAOptions {
ENodeType type;
SNodeList* pFuncs;
SNodeList* pCols;
SNode* pInterval;
uint8_t tsPrecision;
bool recursiveTsma; // true if create recursive tsma
} STSMAOptions;
typedef struct SCreateTSMAStmt {
ENodeType type;
bool ignoreExists;
char tsmaName[TSDB_TABLE_NAME_LEN];
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; // base tb name or base tsma name
char originalTbName[TSDB_TABLE_NAME_LEN];
STSMAOptions* pOptions;
SNode* pPrevQuery;
SMCreateSmaReq* pReq;
} SCreateTSMAStmt;
typedef struct SDropTSMAStmt {
ENodeType type;
bool ignoreNotExists;
char dbName[TSDB_DB_NAME_LEN];
char tsmaName[TSDB_TABLE_NAME_LEN];
} SDropTSMAStmt;
#ifdef __cplusplus
}
#endif

View File

@ -123,6 +123,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListMakeStrictAppendList(SNodeList** pTarget, SNodeList* pSrc);
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode);
int32_t nodesListMakePushFront(SNodeList** pList, SNode* pNode);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
@ -169,6 +170,7 @@ int32_t nodesMsgToNode(const char* pStr, int32_t len, SNode** pNode);
int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len);
char* nodesGetNameFromColumnNode(SNode* pNode);
int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots);
void nodesSortList(SNodeList** pList, int32_t (*)(SNode* pNode1, SNode* pNode2));
#ifdef __cplusplus
}

View File

@ -66,6 +66,7 @@ typedef struct SLogicNode {
EOrder inputTsOrder;
EOrder outputTsOrder;
bool forceCreateNonBlockingOptr; // true if the operator can use non-blocking(pipeline) mode
bool splitDone;
} SLogicNode;
typedef enum EScanType {
@ -108,6 +109,9 @@ typedef struct SScanLogicNode {
int8_t igExpired;
int8_t igCheckUpdate;
SArray* pSmaIndexes;
SArray* pTsmas;
SArray* pTsmaTargetTbVgInfo;
SArray* pTsmaTargetTbInfo;
SNodeList* pGroupTags;
bool groupSort;
SNodeList* pTags; // for create stream
@ -123,20 +127,38 @@ typedef struct SScanLogicNode {
SArray* pFuncTypes; // for last, last_row
bool paraTablesSort; // for table merge scan
bool smallDataTsSort; // disable row id sort for table merge scan
bool needSplit;
} SScanLogicNode;
typedef struct SJoinLogicNode {
SLogicNode node;
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
EJoinAlgorithm joinAlgo;
SNode* addPrimEqCond;
SNode* pPrimKeyEqCond;
SNode* pColEqCond;
SNode* pColOnCond;
SNode* pTagEqCond;
SNode* pTagOnCond;
SNode* pOtherOnCond;
SNode* pFullOnCond; // except prim eq cond
SNodeList* pLeftEqNodes;
SNodeList* pRightEqNodes;
bool allEqTags;
bool isSingleTableJoin;
bool hasSubQuery;
bool isLowLevelJoin;
bool seqWinGroup;
bool grpJoin;
bool hashJoinHint;
// FOR HASH JOIN
int32_t timeRangeTarget; //table onCond filter
STimeWindow timeRange; //table onCond filter
SNode* pLeftOnCond; //table onCond filter
SNode* pRightOnCond; //table onCond filter
} SJoinLogicNode;
typedef struct SAggLogicNode {
@ -151,6 +173,7 @@ typedef struct SAggLogicNode {
bool isGroupTb;
bool isPartTb; // true if partition keys has tbname
bool hasGroup;
SNodeList *pTsmaSubplans;
} SAggLogicNode;
typedef struct SProjectLogicNode {
@ -234,7 +257,9 @@ typedef struct SMergeLogicNode {
SNodeList* pMergeKeys;
SNodeList* pInputs;
int32_t numOfChannels;
int32_t numOfSubplans;
int32_t srcGroupId;
int32_t srcEndGroupId;
bool colsMerge;
bool needSort;
bool groupSort;
@ -287,6 +312,7 @@ typedef struct SWindowLogicNode {
bool isPartTb;
int64_t windowCount;
int64_t windowSliding;
SNodeList* pTsmaSubplans;
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -480,26 +506,52 @@ typedef struct SInterpFuncPhysiNode {
} SInterpFuncPhysiNode;
typedef struct SSortMergeJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pOtherOnCond;
SNodeList* pTargets;
SPhysiNode node;
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
int32_t asofOpType;
SNode* leftPrimExpr;
SNode* rightPrimExpr;
int32_t leftPrimSlotId;
int32_t rightPrimSlotId;
SNodeList* pEqLeft;
SNodeList* pEqRight;
SNode* pPrimKeyCond; //remove
SNode* pColEqCond; //remove
SNode* pColOnCond;
SNode* pFullOnCond;
SNodeList* pTargets;
SQueryStat inputStat[2];
bool seqWinGroup;
bool grpJoin;
} SSortMergeJoinPhysiNode;
typedef struct SHashJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNodeList* pOnLeft;
SNodeList* pOnRight;
SNode* pFilterConditions;
SNodeList* pTargets;
SQueryStat inputStat[2];
SPhysiNode node;
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
SNodeList* pOnLeft;
SNodeList* pOnRight;
SNode* leftPrimExpr;
SNode* rightPrimExpr;
int32_t leftPrimSlotId;
int32_t rightPrimSlotId;
int32_t timeRangeTarget; //table onCond filter
STimeWindow timeRange; //table onCond filter
SNode* pLeftOnCond; //table onCond filter
SNode* pRightOnCond; //table onCond filter
SNode* pFullOnCond; //preFilter
SNodeList* pTargets;
SQueryStat inputStat[2];
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pTagEqCond;
// only in planner internal
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pTagEqCond;
} SHashJoinPhysiNode;
typedef struct SGroupCachePhysiNode {
@ -533,6 +585,7 @@ typedef struct SAggPhysiNode {
SNodeList* pAggFuncs;
bool mergeDataBlock;
bool groupKeyOptimized;
bool hasCountLikeFunc;
} SAggPhysiNode;
typedef struct SDownstreamSourceNode {
@ -562,7 +615,9 @@ typedef struct SMergePhysiNode {
SNodeList* pMergeKeys;
SNodeList* pTargets;
int32_t numOfChannels;
int32_t numOfSubplans;
int32_t srcGroupId;
int32_t srcEndGroupId;
bool groupSort;
bool ignoreGroupId;
bool inputWithGroupId;

View File

@ -23,6 +23,7 @@ extern "C" {
#include "nodes.h"
#include "tmsg.h"
#include "tvariant.h"
#include "tsimplehash.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) \
@ -79,6 +80,7 @@ typedef struct SColumnNode {
uint16_t projIdx; // the idx in project list, start from 1
EColumnType colType; // column or tag
bool hasIndex;
bool isPrimTs;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
char tableAlias[TSDB_TABLE_NAME_LEN];
@ -103,10 +105,16 @@ typedef struct STargetNode {
SNode* pExpr;
} STargetNode;
#define VALUE_FLAG_IS_DURATION (1 << 0)
#define VALUE_FLAG_IS_TIME_OFFSET (1 << 1)
#define IS_DURATION_VAL(_flag) ((_flag) & VALUE_FLAG_IS_DURATION)
#define IS_TIME_OFFSET_VAL(_flag) ((_flag) & VALUE_FLAG_IS_TIME_OFFSET)
typedef struct SValueNode {
SExprNode node; // QUERY_NODE_VALUE
char* literal;
bool isDuration;
int32_t flag;
bool translate;
bool notReserved;
bool isNull;
@ -133,6 +141,8 @@ typedef enum EHintOption {
HINT_PARTITION_FIRST,
HINT_PARA_TABLES_SORT,
HINT_SMALLDATA_TS_SORT,
HINT_HASH_JOIN,
HINT_SKIP_TSMA,
} EHintOption;
typedef struct SHintNode {
@ -168,6 +178,8 @@ typedef struct SFunctionNode {
int32_t udfBufSize;
bool hasPk;
int32_t pkBytes;
bool hasOriginalFunc;
int32_t originalFuncId;
} SFunctionNode;
typedef struct STableNode {
@ -181,6 +193,11 @@ typedef struct STableNode {
struct STableMeta;
typedef struct STsmaTargetCTbInfo {
char tableName[TSDB_TABLE_NAME_LEN]; // child table or normal table name
uint64_t uid;
} STsmaTargetTbInfo;
typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta;
@ -189,6 +206,9 @@ typedef struct SRealTableNode {
double ratio;
SArray* pSmaIndexes;
int8_t cacheLastMode;
SArray* pTsmas;
SArray* tsmaTargetTbVgInfo; // SArray<SVgroupsInfo*>, used for child table or normal table only
SArray* tsmaTargetTbInfo; // SArray<STsmaTargetTbInfo>, used for child table or normal table only
} SRealTableNode;
typedef struct STempTableNode {
@ -206,12 +226,31 @@ typedef struct SViewNode {
int8_t cacheLastMode;
} SViewNode;
#define JOIN_JLIMIT_MAX_VALUE 1024
#define IS_INNER_NONE_JOIN(_type, _stype) ((_type) == JOIN_TYPE_INNER && (_stype) == JOIN_STYPE_NONE)
#define IS_SEMI_JOIN(_stype) ((_stype) == JOIN_STYPE_SEMI)
#define IS_WINDOW_JOIN(_stype) ((_stype) == JOIN_STYPE_WIN)
#define IS_ASOF_JOIN(_stype) ((_stype) == JOIN_STYPE_ASOF)
typedef enum EJoinType {
JOIN_TYPE_INNER = 1,
JOIN_TYPE_INNER = 0,
JOIN_TYPE_LEFT,
JOIN_TYPE_RIGHT,
JOIN_TYPE_FULL,
JOIN_TYPE_MAX_VALUE
} EJoinType;
typedef enum EJoinSubType {
JOIN_STYPE_NONE = 0,
JOIN_STYPE_OUTER,
JOIN_STYPE_SEMI,
JOIN_STYPE_ANTI,
JOIN_STYPE_ASOF,
JOIN_STYPE_WIN,
JOIN_STYPE_MAX_VALUE
} EJoinSubType;
typedef enum EJoinAlgorithm {
JOIN_ALGO_UNKNOWN = 0,
JOIN_ALGO_MERGE,
@ -223,13 +262,18 @@ typedef enum EDynQueryType {
} EDynQueryType;
typedef struct SJoinTableNode {
STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType;
bool hasSubQuery;
bool isLowLevelJoin;
SNode* pLeft;
SNode* pRight;
SNode* pOnCond;
STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
SNode* addPrimCond;
bool hasSubQuery;
bool isLowLevelJoin;
SNode* pParent;
SNode* pLeft;
SNode* pRight;
SNode* pOnCond;
} SJoinTableNode;
typedef enum EGroupingSetType { GP_TYPE_NORMAL = 1 } EGroupingSetType;
@ -305,6 +349,7 @@ typedef enum EFillMode {
typedef enum ETimeLineMode {
TIME_LINE_NONE = 1,
TIME_LINE_BLOCK,
TIME_LINE_MULTI,
TIME_LINE_GLOBAL,
} ETimeLineMode;
@ -338,6 +383,13 @@ typedef struct SCaseWhenNode {
SNodeList* pWhenThenList;
} SCaseWhenNode;
typedef struct SWindowOffsetNode {
ENodeType type; // QUERY_NODE_WINDOW_OFFSET
SNode* pStartOffset; // SValueNode
SNode* pEndOffset; // SValueNode
} SWindowOffsetNode;
typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT
bool isDistinct;
@ -362,6 +414,7 @@ typedef struct SSelectStmt {
uint8_t precision;
int32_t selectFuncNum;
int32_t returnRows; // EFuncReturnRows
ETimeLineMode timeLineCurMode;
ETimeLineMode timeLineResMode;
bool isEmptyResult;
bool isSubquery;
@ -385,6 +438,7 @@ typedef struct SSelectStmt {
bool onlyHasKeepOrderFunc;
bool groupSort;
bool tagScan;
bool joinContains;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
@ -400,6 +454,7 @@ typedef struct SSetOperator {
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
ETimeLineMode timeLineResMode;
bool joinContains;
} SSetOperator;
typedef enum ESqlClause {
@ -537,12 +592,15 @@ typedef struct SQuery {
bool stableQuery;
} SQuery;
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId);
@ -579,6 +637,12 @@ const char* logicConditionTypeStr(ELogicConditionType type);
bool nodesIsStar(SNode* pNode);
bool nodesIsTableStar(SNode* pNode);
char* getJoinTypeString(EJoinType type);
char* getJoinSTypeString(EJoinSubType type);
char* getFullJoinTypeString(EJoinType type, EJoinSubType stype);
int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc);
#ifdef __cplusplus
}
#endif

View File

@ -158,6 +158,7 @@ int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
void destoryCatalogReq(SCatalogReq *pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr);
#ifdef __cplusplus
}

View File

@ -272,6 +272,7 @@ void initQueryModuleMsgHandle();
const SSchema* tGetTbnameColumnSchema();
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
int32_t getAsofJoinReverseOp(EOperatorType op);
int32_t queryCreateCTableMetaFromMsg(STableMetaRsp* msg, SCTableMeta* pMeta);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
@ -330,7 +331,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
(_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW)
(_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW || \
(_type) == TDMT_MND_CREATE_TSMA || (_type) == TDMT_MND_DROP_TSMA || (_type) == TDMT_MND_DROP_TB_WITH_TSMA)
#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
(SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \

View File

@ -31,6 +31,15 @@ enum {
FLT_OPTION_NEED_UNIQE = 4,
};
typedef enum EConditionType {
COND_TYPE_PRIMARY_KEY = 1,
COND_TYPE_TAG_INDEX,
COND_TYPE_TAG,
COND_TYPE_NORMAL
} EConditionType;
#define FILTER_RESULT_ALL_QUALIFIED 0x1
#define FILTER_RESULT_NONE_QUALIFIED 0x2
#define FILTER_RESULT_PARTIAL_QUALIFIED 0x3
@ -54,6 +63,8 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg,
/* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,
SNode **pOtherCond);
bool filterIsMultiTableColsCond(SNode *pCond);
EConditionType filterClassifyCondition(SNode *pNode);
#ifdef __cplusplus
}

View File

@ -72,6 +72,7 @@ int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t md5Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
/* Conversion functions */
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -82,6 +83,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int64_t offsetFromTz(char *timezone, int64_t factor);
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

View File

@ -247,11 +247,11 @@ typedef enum {
TASK_SCANHISTORY_CONT = 0x1,
TASK_SCANHISTORY_QUIT = 0x2,
TASK_SCANHISTORY_REXEC = 0x3,
} EScanHistoryRet;
} EScanHistoryCode;
typedef struct {
EScanHistoryRet ret;
int32_t idleTime;
EScanHistoryCode ret;
int32_t idleTime;
} SScanhistoryDataInfo;
typedef struct {
@ -811,7 +811,7 @@ int32_t streamSendCheckRsp(const SStreamMeta* pMeta, const SStreamTaskCheckReq*
int32_t streamProcessCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp);
int32_t streamLaunchFillHistoryTask(SStreamTask* pTask);
int32_t streamStartScanHistoryAsync(SStreamTask* pTask, int8_t igUntreated);
int32_t streamReExecScanHistoryFuture(SStreamTask* pTask, int32_t idleDuration);
int32_t streamExecScanHistoryInFuture(SStreamTask* pTask, int32_t idleDuration);
bool streamHistoryTaskSetVerRangeStep2(SStreamTask* pTask, int64_t latestVer);
int32_t streamQueueGetNumOfItems(const SStreamQueue* pQueue);

View File

@ -31,7 +31,7 @@ extern "C" {
#define TSWAP(a, b) \
do { \
char *__tmp = alloca(sizeof(a)); \
char *__tmp = (char*)alloca(sizeof(a)); \
memcpy(__tmp, &(a), sizeof(a)); \
memcpy(&(a), &(b), sizeof(a)); \
memcpy(&(b), __tmp, sizeof(a)); \

View File

@ -432,6 +432,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481)
#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482)
#define TSDB_CODE_MND_INVALID_DROP_TSMA TAOS_DEF_ERROR_CODE(0, 0x0485)
#define TSDB_CODE_MND_MAX_TSMA_NUM_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0x0486)
// mnode-tag-indxe
@ -556,6 +558,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_QWORKER_QUIT TAOS_DEF_ERROR_CODE(0, 0x0730)
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
// grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
@ -771,12 +774,18 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E)
#define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F)
#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670)
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2671)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2673)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x2674)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x2675)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x2676)
#define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2671)
#define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_INVALID_WIN_OFFSET_UNIT TAOS_DEF_ERROR_CODE(0, 0x2673)
#define TSDB_CODE_PAR_VALID_PRIM_TS_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x2674)
#define TSDB_CODE_PAR_ORDERBY_UNKNOWN_EXPR TAOS_DEF_ERROR_CODE(0, 0x2675)
#define TSDB_CODE_PAR_NOT_WIN_FUNC TAOS_DEF_ERROR_CODE(0, 0x2676)
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2677)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2678)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2679)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x267A)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x267B)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x267C)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner
@ -825,6 +834,12 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3103)
#define TSDB_CODE_TSMA_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x3104)
#define TSDB_CODE_TSMA_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x3105)
#define TSDB_CODE_TSMA_INVALID_TB TAOS_DEF_ERROR_CODE(0, 0x3106)
#define TSDB_CODE_TSMA_INVALID_INTERVAL TAOS_DEF_ERROR_CODE(0, 0x3107)
#define TSDB_CODE_TSMA_INVALID_FUNC_PARAM TAOS_DEF_ERROR_CODE(0, 0x3108)
#define TSDB_CODE_TSMA_UNSUPPORTED_FUNC TAOS_DEF_ERROR_CODE(0, 0x3109)
#define TSDB_CODE_TSMA_MUST_BE_DROPPED TAOS_DEF_ERROR_CODE(0, 0x3110)
#define TSDB_CODE_TSMA_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x3111)
//rsma
#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150)

View File

@ -149,7 +149,7 @@ typedef enum EOperatorType {
OP_TYPE_BIT_OR,
// binary comparison operator
OP_TYPE_GREATER_THAN = 40,
OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION
OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL,
@ -170,6 +170,7 @@ typedef enum EOperatorType {
OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN,
OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION
// json operator
OP_TYPE_JSON_GET_VALUE = 150,

View File

@ -0,0 +1,219 @@
#!/bin/bash
log_date_time=`date +%m%d%Y-%H%M`
log=${log_date_time}.log
dataDir=/data
logDir=/data/taos/log
echo "#########collect sysinfo############"
cat /etc/os-release
free -g
cat /proc/cpuinfo |grep processor
uname -a
df -h
lsblk
blkid
cat /etc/fstab
mount
cat /etc/sysctl.conf
cat /etc/security/limits.conf
sysctl -p
######check hostname #####
echo "#########hostname"
hostname
hostname -f
echo "#########/etc/hostname"
cat /etc/hostname
echo "#########/etc/hosts"
cat /etc/hosts
echo "#########Step 1############"
echo "$#"
######single node without config file#####
if [ "$#" -eq 0 ] ;then
echo "#########This is for single node !!"
echo "#########Please provide config.ini file for cluster installation !!"
first_ep=`hostname -f`:6030
second_ep=""
fi
echo "#########Step 2############"
######use config file to configre cluster!#####
echo $1
config_file=$1
if [ "$#" -eq 1 ] ;then
echo "#########Use $1 file to configure cluster."
echo "#########The configure of cluster"
cat $config_file
first_ep=`sed -n '1p' $config_file |awk '{print $2}'`:6030
second_ep=`sed -n '2p' $config_file |awk '{print $2}'`:6030
fi
echo "#########Step 3############"
################################################################
echo "#########create data directory"
mkdir -p ${dataDir}/taos/{data,core,tmp,soft}
mkdir -p ${logDir}
chmod 777 ${dataDir}/taos/tmp
chmod 777 ${logDir}
echo "#########install suggest package and disable firewall"
os=$(cat /etc/os-release| grep PRETTY_NAME | awk '{print $1}'|awk -F '=' '{print $2}' | sed 's/"//g')
if [ $os = 'Ubuntu' ]
then
echo "This is Ununtu!"
#apt install screen tmux gdb fio iperf3 sysstat net-tools ntp tree wget
ufw status
ufw stop
ufw disable
elif [ $os = 'CentOS' -o $os = 'Red' ]
then
echo "This is Centos/Red"
#yum install -y screen tmux gdb fio iperf3 sysstat net-tools ntp tree wget
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
else
echo "####$os"
fi
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
echo "#########system tunning"
echo "fs.nr_open = 10485760" >>/etc/sysctl.conf
echo "net.core.somaxconn=10240" >> /etc/sysctl.conf
echo "net.core.netdev_max_backlog=20480" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_syn_backlog=10240" >> /etc/sysctl.conf
echo "net.ipv4.tcp_retries2=5" >> /etc/sysctl.conf
echo "net.ipv4.tcp_syn_retries=2" >> /etc/sysctl.conf
echo "net.ipv4.tcp_synack_retries=2" >> /etc/sysctl.conf
echo "net.ipv4.tcp_tw_reuse=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_tw_recycle=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_keepalive_time=600" >> /etc/sysctl.conf
echo "net.ipv4.tcp_abort_on_overflow=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_tw_buckets=5000" >> /etc/sysctl.conf
echo "net.ipv4.ip_local_port_range=10000 60999" >> /etc/sysctl.conf
echo "* soft ntaosc 65536" >>/etc/security/limits.conf
echo "* soft nofile 1048576" >>/etc/security/limits.conf
echo "* soft stack 65536" >>/etc/security/limits.conf
echo "* hard ntaosc 65536" >>/etc/security/limits.conf
echo "* hard nofile 1048576" >>/etc/security/limits.conf
echo "* hard stack 65536" >>/etc/security/limits.conf
echo "root soft ntaosc 65536" >>/etc/security/limits.conf
echo "root soft nofile 1048576" >>/etc/security/limits.conf
echo "root soft stack 65536" >>/etc/security/limits.conf
echo "root hard ntaosc 65536" >>/etc/security/limits.conf
echo "root hard nofile 1048576" >>/etc/security/limits.conf
echo "root hard stack 65536" >>/etc/security/limits.conf
echo "ulimit -c unlimited" >>/etc/profile
echo "kernel.core_pattern=${dataDir}/taos/core/core-%e-%p" >>/etc/sysctl.conf
sysctl -p
date
####to do check date timezone and ntp########
#cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
#NTP
#SWAP
#reboot
echo "#########install TDengine"
./install.sh -e no
pwd
echo "#########Create /etc/taos/taos.cfg"
mv /etc/taos/taos.cfg /etc/taos/taos.cfg.bak
cat /proc/cpuinfo | grep "processor" | wc -l
num=`cat /proc/cpuinfo | grep "processor" | wc -l`
cpunum=$(expr $num + $num)
host_name=`hostname -f`
#####vi /etc/taos/taos.cfg
echo "firstEp $first_ep
secondEp $second_ep
fqdn $host_name
supportVnodes $cpunum
logDir $logDir
dataDir ${dataDir}/taos/data
tempDir ${dataDir}/taos/tmp
keepColumnName 0
maxNumOfDistinctRes 10000000
timezone UTC-8
locale en_US.UTF-8
charset UTF-8
maxShellConns 100000
maxConnections 100000
audit 0
auditFqdn localhost
monitor 1
monitorFqdn localhost
logKeepDays 10
debugflag 131
shellActivityTimer 120
numOfRpcSessions 30000" >> /etc/taos/taos.cfg
########################
########################tunning taosadapter and taosx log directory
sed -i "/^\#*path/c\path\ =\ \"${logDir}\"" /etc/taos/taosadapter.toml
sed -i "/^\#*logs_home/c\logs_home\ =\ \"${logDir}\"" /etc/taos/taosx.toml
##########################single nodes
echo "#########Start toasd"
./start-all.sh
date
if [ "$#" -eq 0 ] ;then
sleep 5
taos -s "show dnodes;"
taos -s "show cluster\G;"
echo "########"
taosd -k
curl -u root:taosdata ${host_name}:6041/rest/sql -d "select server_version()"
echo -e "\r"
echo "#########Installation completed########"
echo "#########Suggest reboot OS##########"
date
exit
fi
##########################use $config_file to config cluster
if test "$host_name"x = `sed -n '1p' $config_file |awk '{print $2}'`x ;then
echo "#########Add dnodes and mnodes"
while read ip_address host_name
do
echo "$host_name"
taos -s "create dnode '$host_name:6030';"
sleep 5
done <$config_file
fi
host_name=`hostname -f`
if test "$host_name"x = `sed -n '3p' $config_file |awk '{print $2}'`x ;then
echo "#########Add mnodes"
taos -s "create mnode on dnode 2;"
sleep 5
taos -s "create mnode on dnode 3;"
fi
sleep 5
taos -s "show dnodes;"
taos -s "show mnodes;"
taos -s "show grants\G;"
taos -s "show cluster\G;"
echo "########"
taosd -k
echo "#########Test TDengine"
curl -u root:taosdata ${host_name}:6041/rest/sql -d "select server_version()"
echo -e "\r"
echo "#########Installation completed########"
echo "#########Suggest reboot OS##########"
date
cd ..

View File

@ -234,6 +234,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
catalogUpdateDbCfg(pCatalog, rsp->cfgRsp->db, rsp->cfgRsp->dbId, rsp->cfgRsp);
rsp->cfgRsp = NULL;
}
if (rsp->pTsmaRsp) {
if (rsp->pTsmaRsp->pTsmas) {
for (int32_t i = 0; i < rsp->pTsmaRsp->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
catalogAsyncUpdateTSMA(pCatalog, &pTsma, rsp->dbTsmaVersion);
}
taosArrayClear(rsp->pTsmaRsp->pTsmas);
}
}
}
_return:
@ -327,6 +336,33 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal
return TSDB_CODE_SUCCESS;
}
static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog* pCatalog) {
int32_t code = 0;
STSMAHbRsp hbRsp = {0};
if (tDeserializeTSMAHbRsp(value, valueLen, &hbRsp)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int32_t numOfTsma = taosArrayGetSize(hbRsp.pTsmas);
for (int32_t i = 0; i < numOfTsma; ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
if (!pTsmaInfo->pFuncs) {
tscDebug("hb to remove tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
catalogRemoveTSMA(pCatalog, pTsmaInfo);
tFreeAndClearTableTSMAInfo(pTsmaInfo);
} else {
tscDebug("hb to update tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
catalogUpdateTSMA(pCatalog, &pTsmaInfo);
tFreeAndClearTableTSMAInfo(pTsmaInfo);
}
}
taosArrayDestroy(hbRsp.pTsmas);
return TSDB_CODE_SUCCESS;
}
static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
for (int32_t i = 0; i < kvNum; ++i) {
@ -379,6 +415,13 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p
break;
}
#endif
case HEARTBEAT_KEY_TSMA: {
if (kv->valueLen <= 0 || !kv->value) {
tscError("Invalid tsma info, len: %d, value: %p", kv->valueLen, kv->value);
}
hbprocessTSMARsp(kv->value, kv->valueLen, pCatalog);
break;
}
default:
tscError("invalid hb key type:%d", kv->key);
break;
@ -763,6 +806,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
db->cfgVersion = htonl(db->cfgVersion);
db->numOfTable = htonl(db->numOfTable);
db->stateTs = htobe64(db->stateTs);
db->tsmaVersion = htonl(db->tsmaVersion);
}
SKv kv = {
@ -871,6 +915,39 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
return TSDB_CODE_SUCCESS;
}
int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, SClientHbReq* pReq) {
int32_t code = 0;
uint32_t tsmaNum = 0;
STSMAVersion *tsmas = NULL;
code = catalogGetExpiredTsmas(pCatalog, &tsmas, &tsmaNum);
if (code) {
taosMemoryFree(tsmas);
return code;
}
if (tsmaNum <= 0) {
taosMemoryFree(tsmas);
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < tsmaNum; ++i) {
STSMAVersion* tsma = &tsmas[i];
tsma->dbId = htobe64(tsma->dbId);
tsma->tsmaId = htobe64(tsma->tsmaId);
tsma->version = htonl(tsma->version);
}
tscDebug("hb got %d expred tsmas, valueLen: %lu", tsmaNum, sizeof(STSMAVersion) * tsmaNum);
if (!pReq->info) {
pReq->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
}
SKv kv = {.key = HEARTBEAT_KEY_TSMA, .valueLen = sizeof(STSMAVersion) * tsmaNum, .value = tsmas};
taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
return TSDB_CODE_SUCCESS;
}
int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) {
SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
@ -935,6 +1012,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return code;
}
#endif
code = hbGetExpiredTSMAInfo(connKey, pCatalog, req);
} else {
req->app.appId = 0;
}

View File

@ -1041,6 +1041,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId);
removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
restartAsyncQuery(pRequest, code);
return;
}

View File

@ -936,6 +936,8 @@ int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL);
pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL);
pTarget->pView = taosArrayDup(pSrc->pView, NULL);
pTarget->pTableTSMAs = taosArrayDup(pSrc->pTableTSMAs, NULL);
pTarget->pTSMAs = taosArrayDup(pSrc->pTSMAs, NULL);
pTarget->qNodeRequired = pSrc->qNodeRequired;
pTarget->dNodeRequired = pSrc->dNodeRequired;
pTarget->svrVerRequired = pSrc->svrVerRequired;

View File

@ -373,6 +373,18 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema tsmaSchema[] = {
{.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_stb", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "interval", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "func_list", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema userGrantsFullSchema[] = {
{.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@ -432,6 +444,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true},
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_ENCRYPTIONS, encryptionsSchema, tListLen(encryptionsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
};
static const SSysDbTableSchema connectionsSchema[] = {

View File

@ -191,26 +191,29 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
}
size_t start = 1;
// the first item
memcpy(pColumnInfoData->pData, pData, itemLen);
int32_t t = 0;
int32_t count = log(numOfRows) / log(2);
uint32_t startOffset = (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) ? pColumnInfoData->varmeta.length : (currentRow * itemLen);
// the first item
memcpy(pColumnInfoData->pData + startOffset, pData, itemLen);
while (t < count) {
int32_t xlen = 1 << t;
memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData,
memcpy(pColumnInfoData->pData + start * itemLen + startOffset,
pColumnInfoData->pData + startOffset,
xlen * itemLen);
t += 1;
start += xlen;
}
// the tail part
if (numOfRows > start) {
memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData,
memcpy(pColumnInfoData->pData + start * itemLen + startOffset,
pColumnInfoData->pData + startOffset,
(numOfRows - start) * itemLen);
}
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
for (int32_t i = 0; i < numOfRows; ++i) {
pColumnInfoData->varmeta.offset[i + currentRow] = pColumnInfoData->varmeta.length + i * itemLen;
@ -226,9 +229,13 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
bool trimValue) {
int32_t len = pColumnInfoData->info.bytes;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
len = varDataTLen(pData);
if (pColumnInfoData->varmeta.allocLen < (numOfRows + currentRow) * len) {
int32_t code = colDataReserve(pColumnInfoData, (numOfRows + currentRow) * len);
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
len = getJsonValueLen(pData);
} else {
len = varDataTLen(pData);
}
if (pColumnInfoData->varmeta.allocLen < (numOfRows * len + pColumnInfoData->varmeta.length)) {
int32_t code = colDataReserve(pColumnInfoData, (numOfRows * len + pColumnInfoData->varmeta.length));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -239,10 +246,12 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
}
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows) {
pColumnInfoData->hasNull = true;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
memset(&pColumnInfoData->varmeta.offset[currentRow], -1, sizeof(int32_t) * numOfRows);
} else {
if (numOfRows < sizeof(char) * 2) {
if (numOfRows < 16) {
for (int32_t i = 0; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
}
@ -256,9 +265,10 @@ void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
}
}
memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, (numOfRows - i) / sizeof(char));
i += (numOfRows - i) / sizeof(char) * sizeof(char);
int32_t bytes = (numOfRows - i) / 8;
memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, bytes);
i += bytes * 8;
for (; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
}
@ -481,6 +491,111 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
return 0;
}
int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows) {
if (pDst->info.type != pSrc->info.type || pDst->info.bytes != pSrc->info.bytes || pSrc->reassigned) {
return TSDB_CODE_FAILED;
}
if (numOfRows <= 0) {
return numOfRows;
}
if (IS_VAR_DATA_TYPE(pDst->info.type)) {
int32_t allLen = 0;
void* srcAddr = NULL;
if (pSrc->hasNull) {
for (int32_t i = 0; i < numOfRows; ++i) {
if (colDataIsNull_var(pSrc, srcIdx + i)) {
pDst->varmeta.offset[dstIdx + i] = -1;
pDst->hasNull = true;
continue;
}
char* pData = colDataGetVarData(pSrc, srcIdx + i);
if (NULL == srcAddr) {
srcAddr = pData;
}
int32_t dataLen = 0;
if (pSrc->info.type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
pDst->varmeta.offset[dstIdx + i] = pDst->varmeta.length + allLen;
allLen += dataLen;
}
} else {
for (int32_t i = 0; i < numOfRows; ++i) {
char* pData = colDataGetVarData(pSrc, srcIdx + i);
int32_t dataLen = 0;
if (pSrc->info.type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
pDst->varmeta.offset[dstIdx + i] = pDst->varmeta.length + allLen;
allLen += dataLen;
}
}
if (allLen > 0) {
// copy data
if (pDst->varmeta.allocLen < pDst->varmeta.length + allLen) {
char* tmp = taosMemoryRealloc(pDst->pData, pDst->varmeta.length + allLen);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pDst->pData = tmp;
pDst->varmeta.allocLen = pDst->varmeta.length + allLen;
}
if (pSrc->hasNull) {
memcpy(pDst->pData + pDst->varmeta.length, srcAddr, allLen);
} else {
memcpy(pDst->pData + pDst->varmeta.length, colDataGetVarData(pSrc, srcIdx), allLen);
}
pDst->varmeta.length = pDst->varmeta.length + allLen;
}
} else {
if (pSrc->hasNull) {
if (BitPos(dstIdx) == BitPos(srcIdx)) {
for (int32_t i = 0; i < numOfRows; ++i) {
if (0 == BitPos(dstIdx) && (i + (1 << NBIT) <= numOfRows)) {
BMCharPos(pDst->nullbitmap, dstIdx + i) = BMCharPos(pSrc->nullbitmap, srcIdx + i);
if (BMCharPos(pDst->nullbitmap, dstIdx + i)) {
pDst->hasNull = true;
}
i += (1 << NBIT) - 1;
} else {
if (colDataIsNull_f(pSrc->nullbitmap, srcIdx + i)) {
colDataSetNull_f(pDst->nullbitmap, dstIdx + i);
pDst->hasNull = true;
} else {
colDataClearNull_f(pDst->nullbitmap, dstIdx + i);
}
}
}
} else {
for (int32_t i = 0; i < numOfRows; ++i) {
if (colDataIsNull_f(pSrc->nullbitmap, srcIdx + i)) {
colDataSetNull_f(pDst->nullbitmap, dstIdx + i);
pDst->hasNull = true;
} else {
colDataClearNull_f(pDst->nullbitmap, dstIdx + i);
}
}
}
}
if (pSrc->pData != NULL) {
memcpy(pDst->pData + pDst->info.bytes * dstIdx, pSrc->pData + pSrc->info.bytes * srcIdx, pDst->info.bytes * numOfRows);
}
}
return 0;
}
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); }
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
@ -574,6 +689,60 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
return TSDB_CODE_SUCCESS;
}
int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows) {
if (pDest->info.rows + numOfRows > pDest->info.capacity) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
size_t numOfCols = taosArrayGetSize(pDest->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
colDataAssignNRows(pCol2, pDest->info.rows, pCol1, srcIdx, numOfRows);
}
pDest->info.rows += numOfRows;
return TSDB_CODE_SUCCESS;
}
void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows) {
if (numOfRows >= pBlock->info.rows) {
blockDataCleanup(pBlock);
return;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
pCol->varmeta.length = pCol->varmeta.offset[pBlock->info.rows - numOfRows];
memset(pCol->varmeta.offset + pBlock->info.rows - numOfRows, 0, sizeof(*pCol->varmeta.offset) * numOfRows);
} else {
int32_t i = pBlock->info.rows - numOfRows;
for (; i < pBlock->info.rows; ++i) {
if (BitPos(i)) {
colDataClearNull_f(pCol->nullbitmap, i);
} else {
break;
}
}
int32_t bytes = (pBlock->info.rows - i) / 8;
memset(&BMCharPos(pCol->nullbitmap, i), 0, bytes);
i += bytes * 8;
for (; i < pBlock->info.rows; ++i) {
colDataClearNull_f(pCol->nullbitmap, i);
}
}
}
pBlock->info.rows -= numOfRows;
}
size_t blockDataGetSize(const SSDataBlock* pBlock) {
size_t total = 0;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@ -653,25 +822,26 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
return NULL;
}
SSDataBlock* pDst = createDataBlock();
SSDataBlock* pDst = createOneDataBlock(pBlock, false);
if (pDst == NULL) {
return NULL;
}
pDst->info = pBlock->info;
pDst->info.rows = 0;
pDst->info.capacity = 0;
pDst->info.rowSize = 0;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0};
SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
colInfo.info = pSrcCol->info;
blockDataAppendColInfo(pDst, &colInfo);
}
blockDataEnsureCapacity(pDst, rowCount);
/* may have disorder varchar data, TODO
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
colDataAssignNRows(pDstCol, 0, pColData, startIndex, rowCount);
}
*/
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
@ -1206,18 +1376,43 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
pInfo->window.skey = 0;
}
void blockDataReset(SSDataBlock* pDataBlock) {
SDataBlockInfo* pInfo = &pDataBlock->info;
if (pInfo->capacity == 0) {
return;
}
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
p->hasNull = false;
p->reassigned = false;
if (IS_VAR_DATA_TYPE(p->info.type)) {
p->varmeta.length = 0;
}
}
pInfo->rows = 0;
pInfo->dataLoad = 0;
pInfo->window.ekey = 0;
pInfo->window.skey = 0;
pInfo->id.uid = 0;
pInfo->id.groupId = 0;
}
/*
* NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote
* the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to
* any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case.
*/
static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
bool clearPayload) {
if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) {
if (numOfRows <= 0 || pBlockInfo && numOfRows <= pBlockInfo->capacity) {
return TSDB_CODE_SUCCESS;
}
int32_t existedRows = pBlockInfo->rows;
int32_t existedRows = pBlockInfo ? pBlockInfo->rows : 0;
if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
@ -2505,6 +2700,8 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList
int32_t numOfRows = 0;
if (IS_VAR_DATA_TYPE(pDst->info.type)) {
pDst->varmeta.length = 0;
} else {
memset(pDst->nullbitmap, 0, bmLen);
}
}
return;

View File

@ -308,6 +308,9 @@ int32_t tsS3UploadDelaySec = 60;
bool tsExperimental = true;
int32_t tsMaxTsmaNum = 8;
int32_t tsMaxTsmaCalcDelay = 600;
#ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -563,6 +566,11 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
0)
return -1;
return 0;
}
@ -621,8 +629,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
@ -745,6 +751,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 12, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
@ -1122,6 +1129,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
return -1;
}
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
@ -1135,6 +1143,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
tsMultiResultFunctionStarReturnTags = cfgGetItem(pCfg, "multiResultFunctionStarReturnTags")->bval;
tsMaxTsmaCalcDelay = cfgGetItem(pCfg, "maxTsmaCalcDelay")->i32;
return 0;
}
@ -1166,7 +1175,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tstrncpy(tsEncryptAlgorithm, cfgGetItem(pCfg, "encryptAlgorithm")->str, 16);
tstrncpy(tsEncryptScope, cfgGetItem(pCfg, "encryptScope")->str, 100);
@ -1219,6 +1227,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
tmqRowSize = cfgGetItem(pCfg, "tmqRowSize")->i32;
tsMaxTsmaNum = cfgGetItem(pCfg, "maxTsmaNum")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
@ -1563,7 +1572,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"s3PageCacheSize", &tsS3PageCacheSize},
{"s3UploadDelaySec", &tsS3UploadDelaySec},
{"supportVnodes", &tsNumOfSupportVnodes},
{"experimental", &tsExperimental}};
{"experimental", &tsExperimental},
{"maxTsmaNum", &tsMaxTsmaNum}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false);
@ -1781,7 +1791,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental},
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags} };
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags},
{"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false);

View File

@ -67,6 +67,8 @@
static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq);
static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq);
static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoRsp *pRsp);
static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pRsp);
int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
if (pMsg == NULL) {
@ -887,6 +889,16 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq
}
if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1;
if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1;
if (tEncodeI64(&encoder, pReq->normSourceTbUid) < 0) return -1;
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1;
for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i);
if (tEncodeI32(&encoder, p->vgId) < 0) return -1;
if (tEncodeI64(&encoder, p->ver) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->recursiveTsma) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->baseTsmaName) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -937,6 +949,29 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR
}
if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->normSourceTbUid) < 0) return -1;
int32_t numOfVgVer;
if (tDecodeI32(&decoder, &numOfVgVer) < 0) return -1;
if (numOfVgVer > 0) {
pReq->pVgroupVerList = taosArrayInit(numOfVgVer, sizeof(SVgroupVer));
if (pReq->pVgroupVerList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < numOfVgVer; ++i) {
SVgroupVer v = {0};
if (tDecodeI32(&decoder, &v.vgId) < 0) return -1;
if (tDecodeI64(&decoder, &v.ver) < 0) return -1;
if (taosArrayPush(pReq->pVgroupVerList, &v) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
}
if (tDecodeI8(&decoder, &pReq->recursiveTsma) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->baseTsmaName) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -947,6 +982,7 @@ void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) {
taosMemoryFreeClear(pReq->tagsFilter);
taosMemoryFreeClear(pReq->sql);
taosMemoryFreeClear(pReq->ast);
taosArrayDestroy(pReq->pVgroupVerList);
}
int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) {
@ -3024,14 +3060,14 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) {
}
if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->keepTimeOffset) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
ENCODESQL();
if (tEncodeI8(&encoder, pReq->withArbitrator) < 0) return -1;
if (tEncodeI8(&encoder, pReq->encryptAlgorithm) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3ChunkSize) < 0) return -1;
if (tEncodeI32(&encoder, pReq->s3KeepLocal) < 0) return -1;
if (tEncodeI8(&encoder, pReq->s3Compact) < 0) return -1;
tEndEncode(&encoder);
@ -3100,22 +3136,19 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq)
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
DECODESQL();
pReq->withArbitrator = TSDB_DEFAULT_DB_WITH_ARBITRATOR;
pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->withArbitrator) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
}
tEndDecode(&decoder);
@ -3639,6 +3672,14 @@ int32_t tSerializeSDbHbRspImp(SEncoder *pEncoder, const SDbHbRsp *pRsp) {
if (tEncodeI8(pEncoder, 0) < 0) return -1;
}
if (pRsp->pTsmaRsp) {
if (tEncodeI8(pEncoder, 1) < 0) return -1;
if (tEncodeTableTSMAInfoRsp(pEncoder, pRsp->pTsmaRsp) < 0) return -1;
} else {
if (tEncodeI8(pEncoder, 0) < 0) return -1;
}
if (tEncodeI32(pEncoder, pRsp->dbTsmaVersion) < 0) return -1;
return 0;
}
@ -3719,6 +3760,17 @@ int32_t tDeserializeSDbHbRspImp(SDecoder *decoder, SDbHbRsp *pRsp) {
if (NULL == pRsp->cfgRsp) return -1;
if (tDeserializeSDbCfgRspImpl(decoder, pRsp->cfgRsp) < 0) return -1;
}
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &flag) < 0) return -1;
if (flag) {
pRsp->pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (!pRsp->pTsmaRsp) return -1;
if (tDecodeTableTSMAInfoRsp(decoder, pRsp->pTsmaRsp) < 0) return -1;
}
}
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI32(decoder, &pRsp->dbTsmaVersion) < 0) return -1;
}
return 0;
}
@ -3768,6 +3820,10 @@ void tFreeSDbHbRsp(SDbHbRsp *pDbRsp) {
tFreeSDbCfgRsp(pDbRsp->cfgRsp);
taosMemoryFree(pDbRsp->cfgRsp);
}
if (pDbRsp->pTsmaRsp) {
tFreeTableTSMAInfoRsp(pDbRsp->pTsmaRsp);
taosMemoryFree(pDbRsp->pTsmaRsp);
}
}
void tFreeSDbHbBatchRsp(SDbHbBatchRsp *pRsp) {
@ -5281,14 +5337,12 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI32(&decoder, &pReq->keepTimeOffset) < 0) return -1;
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
}
pReq->encryptAlgorithm = TSDB_DEFAULT_ENCRYPT_ALGO;
pReq->s3ChunkSize = TSDB_DEFAULT_S3_CHUNK_SIZE;
pReq->s3KeepLocal = TSDB_DEFAULT_S3_KEEP_LOCAL;
pReq->s3Compact = TSDB_DEFAULT_S3_COMPACT;
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI8(&decoder, &pReq->encryptAlgorithm) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3ChunkSize) < 0) return -1;
if (tDecodeI32(&decoder, &pReq->s3KeepLocal) < 0) return -1;
if (tDecodeI8(&decoder, &pReq->s3Compact) < 0) return -1;
@ -7774,6 +7828,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
}
if (tEncodeI64(&encoder, pReq->smaId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -7900,6 +7955,9 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
}
}
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI64(&decoder, &pReq->smaId)< 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -8294,8 +8352,8 @@ static int32_t tDecodeSVDropTbRsp(SDecoder *pCoder, SVDropTbRsp *pReq) {
}
int32_t tEncodeSVDropTbBatchReq(SEncoder *pCoder, const SVDropTbBatchReq *pReq) {
int32_t nReqs = taosArrayGetSize(pReq->pArray);
SVDropTbReq *pDropTbReq;
int32_t nReqs = taosArrayGetSize(pReq->pArray);
SVDropTbReq *pDropTbReq;
if (tStartEncode(pCoder) < 0) return -1;
@ -9947,3 +10005,443 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) {
taosArrayDestroy(pRsp->pViewRsp);
}
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->fetchingWithTsmaName) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, (uint8_t*)&pReq->fetchingWithTsmaName) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
static int32_t tEncodeTableTSMAInfo(SEncoder* pEncoder, const STableTSMAInfo* pTsmaInfo) {
if (tEncodeCStr(pEncoder, pTsmaInfo->name) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->tsmaId) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->tb) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->dbFName) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->suid) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->destTbUid) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->dbId) < 0) return -1;
if (tEncodeI32(pEncoder, pTsmaInfo->version) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->targetTb) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->targetDbFName) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->interval) < 0) return -1;
if (tEncodeI8(pEncoder, pTsmaInfo->unit) < 0) return -1;
int32_t size = pTsmaInfo->pFuncs ? pTsmaInfo->pFuncs->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAFuncInfo* pFuncInfo = taosArrayGet(pTsmaInfo->pFuncs, i);
if (tEncodeI32(pEncoder, pFuncInfo->funcId) < 0) return -1;
if (tEncodeI16(pEncoder, pFuncInfo->colId) < 0) return -1;
}
size = pTsmaInfo->pTags ? pTsmaInfo->pTags->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
const SSchema* pSchema = taosArrayGet(pTsmaInfo->pTags, i);
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
}
size = pTsmaInfo->pUsedCols ? pTsmaInfo->pUsedCols->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
const SSchema* pSchema = taosArrayGet(pTsmaInfo->pUsedCols, i);
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
}
if (tEncodeCStr(pEncoder, pTsmaInfo->ast) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->streamUid) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->reqTs) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->rspTs) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->delayDuration) < 0) return -1;
if (tEncodeI8(pEncoder, pTsmaInfo->fillHistoryFinished) < 0) return -1;
return 0;
}
static int32_t tDecodeTableTSMAInfo(SDecoder* pDecoder, STableTSMAInfo* pTsmaInfo) {
if (tDecodeCStrTo(pDecoder, pTsmaInfo->name) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->tsmaId) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->tb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->dbFName) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->suid) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->destTbUid) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->dbId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTsmaInfo->version) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->targetTb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->targetDbFName) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->interval) < 0) return -1;
if (tDecodeI8(pDecoder, &pTsmaInfo->unit) < 0) return -1;
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pTsmaInfo->pFuncs = taosArrayInit(size, sizeof(STableTSMAFuncInfo));
if (!pTsmaInfo->pFuncs) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAFuncInfo funcInfo = {0};
if (tDecodeI32(pDecoder, &funcInfo.funcId) < 0) return -1;
if (tDecodeI16(pDecoder, &funcInfo.colId) < 0) return -1;
if (!taosArrayPush(pTsmaInfo->pFuncs, &funcInfo)) return -1;
}
}
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pTsmaInfo->pTags = taosArrayInit(size, sizeof(SSchema));
if (!pTsmaInfo->pTags) return -1;
for (int32_t i = 0; i < size; ++i) {
SSchema schema = {0};
if(tDecodeSSchema(pDecoder, &schema) < 0) return -1;
taosArrayPush(pTsmaInfo->pTags, &schema);
}
}
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pTsmaInfo->pUsedCols = taosArrayInit(size, sizeof(SSchema));
if (!pTsmaInfo->pUsedCols) return -1;
for (int32_t i = 0; i < size; ++i) {
SSchema schema = {0};
if (tDecodeSSchema(pDecoder, &schema) < 0) return -1;
taosArrayPush(pTsmaInfo->pUsedCols, &schema);
}
}
if (tDecodeCStrAlloc(pDecoder, &pTsmaInfo->ast) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->streamUid) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->reqTs) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->rspTs) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->delayDuration) < 0) return -1;
if (tDecodeI8(pDecoder, (int8_t*)&pTsmaInfo->fillHistoryFinished) < 0) return -1;
return 0;
}
static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoRsp *pRsp) {
int32_t size = pRsp->pTsmas ? pRsp->pTsmas->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAInfo* pInfo = taosArrayGetP(pRsp->pTsmas, i);
if (tEncodeTableTSMAInfo(pEncoder, pInfo) < 0) return -1;
}
return 0;
}
static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pRsp) {
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size <= 0) return 0;
pRsp->pTsmas = taosArrayInit(size, POINTER_BYTES);
if (!pRsp->pTsmas) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAInfo *pTsma = taosMemoryCalloc(1, sizeof(STableTSMAInfo));
if (!pTsma) return -1;
taosArrayPush(pRsp->pTsmas, &pTsma);
if (tDecodeTableTSMAInfo(pDecoder, pTsma) < 0) return -1;
}
return 0;
}
int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAInfoRsp* pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeTableTSMAInfoRsp(&encoder, pRsp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRsp* pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeTableTSMAInfoRsp(&decoder, pRsp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeTableTSMAInfo(void* p) {
STableTSMAInfo *pTsmaInfo = p;
if (pTsmaInfo) {
taosArrayDestroy(pTsmaInfo->pFuncs);
taosArrayDestroy(pTsmaInfo->pTags);
taosArrayDestroy(pTsmaInfo->pUsedCols);
taosMemoryFree(pTsmaInfo->ast);
}
}
void tFreeAndClearTableTSMAInfo(void* p) {
STableTSMAInfo* pTsmaInfo = (STableTSMAInfo*)p;
if (pTsmaInfo) {
tFreeTableTSMAInfo(pTsmaInfo);
taosMemoryFree(pTsmaInfo);
}
}
int32_t tCloneTbTSMAInfo(STableTSMAInfo* pInfo, STableTSMAInfo** pRes) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pInfo) {
return TSDB_CODE_SUCCESS;
}
STableTSMAInfo* pRet = taosMemoryCalloc(1, sizeof(STableTSMAInfo));
if (!pRet) return TSDB_CODE_OUT_OF_MEMORY;
*pRet = *pInfo;
if (pInfo->pFuncs) {
pRet->pFuncs = taosArrayDup(pInfo->pFuncs, NULL);
if (!pRet->pFuncs) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->pTags && code == TSDB_CODE_SUCCESS) {
pRet->pTags = taosArrayDup(pInfo->pTags, NULL);
if (!pRet->pTags) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->pUsedCols && code == TSDB_CODE_SUCCESS) {
pRet->pUsedCols = taosArrayDup(pInfo->pUsedCols, NULL);
if (!pRet->pUsedCols) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->ast && code == TSDB_CODE_SUCCESS) {
pRet->ast = taosStrdup(pInfo->ast);
if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code) tFreeTableTSMAInfo(pRet);
*pRes = pRet;
return code;
}
void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp *pRsp) {
if (pRsp && pRsp->pTsmas) {
taosArrayDestroyP(pRsp->pTsmas, tFreeAndClearTableTSMAInfo);
}
}
static int32_t tEncodeStreamProgressReq(SEncoder *pEncoder, const SStreamProgressReq *pReq) {
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->fetchIdx) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProgressReq* pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeStreamProgressReq(&encoder, pReq) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
static int32_t tDecodeStreamProgressReq(SDecoder* pDecoder, SStreamProgressReq* pReq) {
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->fetchIdx) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgressReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, (char *)buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeStreamProgressReq(&decoder, pReq) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
static int32_t tEncodeStreamProgressRsp(SEncoder* pEncoder, const SStreamProgressRsp* pRsp) {
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->fillHisFinished) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->progressDelay) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->fetchIdx) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProgressRsp* pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeStreamProgressRsp(&encoder, pRsp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
static int32_t tDecodeStreamProgressRsp(SDecoder* pDecoder, SStreamProgressRsp* pRsp) {
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1;
if (tDecodeI8(pDecoder, (int8_t*)&pRsp->fillHisFinished) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->progressDelay) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->fetchIdx) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgressRsp* pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeStreamProgressRsp(&decoder, pRsp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder *pEncoder, const SMDropTbReqsOnSingleVg *pReq) {
const SVgroupInfo* pVgInfo = &pReq->vgInfo;
if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1;
int32_t size = pReq->pTbs ? pReq->pTbs->size: 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
const SVDropTbReq* pInfo = taosArrayGet(pReq->pTbs, i);
if (tEncodeSVDropTbReq(pEncoder, pInfo) < 0) return -1;
}
return 0;
}
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* pReq) {
if (tDecodeI32(pDecoder, &pReq->vgInfo.vgId) < 0) return -1;
if (tDecodeU32(pDecoder, &pReq->vgInfo.hashBegin) < 0) return -1;
if (tDecodeU32(pDecoder, &pReq->vgInfo.hashEnd) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pReq->vgInfo.epSet) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->vgInfo.numOfTable) < 0) return -1;
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
pReq->pTbs = taosArrayInit(size, sizeof(SVDropTbReq));
if (!pReq->pTbs) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
SVDropTbReq pTbReq = {0};
for (int32_t i = 0; i < size; ++i) {
if (tDecodeSVDropTbReq(pDecoder, &pTbReq) < 0) return -1;
taosArrayPush(pReq->pTbs, &pTbReq);
}
return 0;
}
void tFreeSMDropTbReqOnSingleVg(void *p) {
SMDropTbReqsOnSingleVg* pReq = p;
taosArrayDestroy(pReq->pTbs);
}
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq){
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
tStartEncode(&encoder);
int32_t size = pReq->pVgReqs ? pReq->pVgReqs->size : 0;
if (tEncodeI32(&encoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
SMDropTbReqsOnSingleVg* pVgReq = taosArrayGet(pReq->pVgReqs, i);
if (tEncodeSMDropTbReqOnSingleVg(&encoder, pVgReq) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
tStartDecode(&decoder);
int32_t size = 0;
if (tDecodeI32(&decoder, &size) < 0) return -1;
pReq->pVgReqs = taosArrayInit(size, sizeof(SMDropTbReqsOnSingleVg));
if (!pReq->pVgReqs) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < size; ++i) {
SMDropTbReqsOnSingleVg vgReq = {0};
tDecodeSMDropTbReqOnSingleVg(&decoder, &vgReq);
taosArrayPush(pReq->pVgReqs, &vgReq);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSMDropTbsReq(void* p) {
SMDropTbsReq* pReq = p;
taosArrayDestroyEx(pReq->pVgReqs, tFreeSMDropTbReqOnSingleVg);
}
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredTbsRsp* pRsp) {
if (tEncodeI32(pCoder, pRsp->vgId) < 0) return -1;
int32_t size = pRsp->pExpiredTbs ? pRsp->pExpiredTbs->size : 0;
if (tEncodeI32(pCoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
if (tEncodeSVDropTbReq(pCoder, taosArrayGet(pRsp->pExpiredTbs, i)) < 0) return -1;
}
return 0;
}
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp* pRsp) {
if (tDecodeI32(pCoder, &pRsp->vgId) < 0) return -1;
int32_t size = 0;
if (tDecodeI32(pCoder, &size) < 0) return -1;
if (size > 0) {
pRsp->pExpiredTbs = taosArrayInit(size, sizeof(SVDropTbReq));
if (!pRsp->pExpiredTbs) return TSDB_CODE_OUT_OF_MEMORY;
SVDropTbReq tb = {0};
for (int32_t i = 0; i < size; ++i) {
if (tDecodeSVDropTbReq(pCoder, &tb) < 0) return -1;
taosArrayPush(pRsp->pExpiredTbs, &tb);
}
}
return 0;
}
void tFreeFetchTtlExpiredTbsRsp(void* p) {
SVFetchTtlExpiredTbsRsp* pRsp = p;
taosArrayDestroy(pRsp->pExpiredTbs);
}

View File

@ -581,7 +581,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
return TSDB_CODE_SUCCESS;
}
static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
switch (unit) {
case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
@ -663,12 +663,12 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura
return getDuration(timestamp, *unit, duration, timePrecision);
}
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision) {
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) {
errno = 0;
/* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10);
if (*duration < 0 || errno != 0) {
if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1;
}

View File

@ -165,6 +165,14 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STB_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STB_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -224,6 +232,10 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

View File

@ -91,6 +91,7 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

View File

@ -918,6 +918,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -964,6 +965,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

View File

@ -29,6 +29,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb);
int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen);
int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq);
bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb);
void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList);
SSdbRaw *mndDbActionEncode(SDbObj *pDb);
const char *mndGetDbStr(const char *src);

View File

@ -402,6 +402,7 @@ typedef struct {
SRWLatch lock;
int64_t stateTs;
int64_t compactStartTime;
int32_t tsmaVersion;
} SDbObj;
typedef struct {
@ -461,12 +462,14 @@ typedef struct {
int32_t tagsFilterLen;
int32_t sqlLen;
int32_t astLen;
int32_t version;
char* expr;
char* tagsFilter;
char* sql;
char* ast;
SSchemaWrapper schemaRow; // for dstVgroup
SSchemaWrapper schemaTag; // for dstVgroup
char baseSmaName[TSDB_TABLE_FNAME_LEN];
} SSmaObj;
typedef struct {

View File

@ -29,6 +29,9 @@ void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma);
int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb);
int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist);
int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t numOfTsmas, void **ppRsp,
int32_t *pRspLen);
int32_t mndGetDbTsmas(SMnode *pMnode, const char *dbFName, uint64_t dbUid, STableTSMAInfoRsp *pRsp, bool *exist);
#ifdef __cplusplus
}

View File

@ -36,7 +36,7 @@
#include "tjson.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 31
#define DB_RESERVE_SIZE 27
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -147,6 +147,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.encryptAlgorithm, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER);
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -243,6 +244,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.encryptAlgorithm, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER);
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
@ -349,6 +351,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
pOld->compactStartTime = pNew->compactStartTime;
pOld->tsmaVersion = pNew->tsmaVersion;
taosWUnLockLatch(&pOld->lock);
return 0;
}
@ -687,6 +690,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_DB_FNAME_LEN);
dbObj.cfgVersion = 1;
dbObj.vgVersion = 1;
dbObj.tsmaVersion = 1;
memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN);
dbObj.cfg = (SDbCfg){
.numOfVgroups = pCreate->numOfVgroups,
@ -1574,7 +1578,7 @@ static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) {
return numOfTables;
}
static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
int32_t vindex = 0;
SSdb *pSdb = pMnode->pSdb;
@ -1735,6 +1739,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
pDbCacheInfo->cfgVersion = htonl(pDbCacheInfo->cfgVersion);
pDbCacheInfo->numOfTable = htonl(pDbCacheInfo->numOfTable);
pDbCacheInfo->stateTs = be64toh(pDbCacheInfo->stateTs);
pDbCacheInfo->tsmaVersion = htonl(pDbCacheInfo->tsmaVersion);
SDbHbRsp rsp = {0};
@ -1773,7 +1778,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
int32_t numOfTable = mndGetDBTableNum(pDb, pMnode);
if (pDbCacheInfo->vgVersion >= pDb->vgVersion && pDbCacheInfo->cfgVersion >= pDb->cfgVersion &&
numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs) {
numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs &&
pDbCacheInfo->tsmaVersion >= pDb->tsmaVersion) {
mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64
" numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d",
pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs,
@ -1792,6 +1798,16 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
mndDumpDbCfgInfo(rsp.cfgRsp, pDb);
}
if (pDbCacheInfo->tsmaVersion != pDb->tsmaVersion) {
rsp.pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (rsp.pTsmaRsp) rsp.pTsmaRsp->pTsmas = taosArrayInit(4, POINTER_BYTES);
if (rsp.pTsmaRsp && rsp.pTsmaRsp->pTsmas) {
rsp.dbTsmaVersion = pDb->tsmaVersion;
bool exist = false;
mndGetDbTsmas(pMnode, 0, pDb->uid, rsp.pTsmaRsp, &exist);
}
}
if (pDbCacheInfo->vgVersion < pDb->vgVersion || numOfTable != pDbCacheInfo->numOfTable ||
pDbCacheInfo->stateTs != pDb->stateTs) {
rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp));

View File

@ -25,6 +25,7 @@
#include "mndStb.h"
#include "mndUser.h"
#include "mndView.h"
#include "mndSma.h"
#include "tglobal.h"
#include "tversion.h"
@ -606,6 +607,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
break;
}
#endif
case HEARTBEAT_KEY_TSMA: {
void * rspMsg = NULL;
int32_t rspLen = 0;
mndValidateTSMAInfo(pMnode, kv->value, kv->valueLen / sizeof(STSMAVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_TSMA, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
}
break;
}
default:
mError("invalid kv key:%d", kv->key);
hbRsp.status = TSDB_CODE_APP_ERROR;

View File

@ -109,7 +109,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
mDebug("mndSetSinkTaskInfo to sma or table, taskId:%s", pTask->id.idStr);
if (pStream->smaId != 0) {
if (pStream->smaId != 0 && pStream->subTableWithoutMd5 != 1) {
pInfo->type = TASK_OUTPUT__SMA;
pInfo->smaSink.smaId = pStream->smaId;
} else {

View File

@ -133,6 +133,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_MACHINES;
} else if (strncasecmp(name, TSDB_INS_TABLE_ENCRYPTIONS, len) == 0) {
type = TSDB_MGMT_TABLE_ENCRYPTIONS;
} else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) {
type = TSDB_MGMT_TABLE_TSMAS;
} else {
mError("invalid show name:%s len:%d", name, len);
}

File diff suppressed because it is too large Load Diff

View File

@ -62,6 +62,10 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDb
static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq);
static int32_t mndProcessDropIndexReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessDropTbWithTsma(SRpcMsg* pReq);
static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pReq);
int32_t mndInitStb(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_STB,
@ -87,6 +91,11 @@ int32_t mndInitStb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
mndSetMsgHandle(pMnode, TDMT_MND_STB_DROP, mndProcessDropStbReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_MND_STB_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_TB_WITH_TSMA, mndProcessDropTbWithTsma);
mndSetMsgHandle(pMnode, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mndProcessFetchTtlExpiredTbs);
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TABLE_RSP, mndTransProcessRsp);
// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq);
// mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq);
@ -936,7 +945,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen, .info = pReq->info};
SRpcMsg rpcMsg = {.msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS, .pCont = pHead, .contLen = contLen, .info = pReq->info};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
@ -2532,11 +2541,6 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
if (pStream->smaId != 0) {
sdbRelease(pSdb, pStream);
continue;
}
if (pStream->targetStbUid == suid) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pStream);
@ -3227,6 +3231,11 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
continue;
}
if (isTsmaResSTb(pStb->name)) {
sdbRelease(pSdb, pStb);
continue;
}
cols = 0;
SName name = {0};
@ -3693,3 +3702,376 @@ static int32_t mndProcessDropIndexReq(SRpcMsg *pReq) {
_OVER:
return code;
}*/
static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessDropStbReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}
typedef struct SVDropTbVgReqs {
SVDropTbBatchReq req;
SVgroupInfo info;
} SVDropTbVgReqs;
typedef struct SMDropTbDbInfo {
SArray *dbVgInfos;
int32_t hashPrefix;
int32_t hashSuffix;
int32_t hashMethod;
} SMDropTbDbInfo;
typedef struct SMDropTbTsmaInfo {
char tsmaResTbDbFName[TSDB_DB_FNAME_LEN];
char tsmaResTbNamePrefix[TSDB_TABLE_NAME_LEN];
int32_t suid;
SMDropTbDbInfo dbInfo; // reference to DbInfo in pDbMap
} SMDropTbTsmaInfo;
typedef struct SMDropTbTsmaInfos {
SArray* pTsmaInfos; // SMDropTbTsmaInfo
} SMDropTbTsmaInfos;
typedef struct SMndDropTbsWithTsmaCtx {
SHashObj* pTsmaMap; // <suid, SMDropTbTsmaInfos>
SHashObj* pDbMap; // <dbuid, SMDropTbDbInfo>
SHashObj* pVgMap; // <vgId, SVDropTbVgReqs>
SArray* pResTbNames; // SArray<char*>
} SMndDropTbsWithTsmaCtx;
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode* pMnode, SMndDropTbsWithTsmaCtx* pCtx, SArray* pTbs, int32_t vgId);
static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx* p) {
if (!p) return;
if (p->pDbMap) {
void* pIter = taosHashIterate(p->pDbMap, NULL);
while (pIter) {
SMDropTbDbInfo* pInfo = pIter;
taosArrayDestroy(pInfo->dbVgInfos);
pIter = taosHashIterate(p->pDbMap, pIter);
}
taosHashCleanup(p->pDbMap);
}
if (p->pResTbNames) {
taosArrayDestroyP(p->pResTbNames, taosMemoryFree);
}
if (p->pTsmaMap) {
void* pIter = taosHashIterate(p->pTsmaMap, NULL);
while (pIter) {
SMDropTbTsmaInfos* pInfos = pIter;
taosArrayDestroy(pInfos->pTsmaInfos);
pIter = taosHashIterate(p->pTsmaMap, pIter);
}
taosHashCleanup(p->pTsmaMap);
}
if (p->pVgMap) {
void* pIter = taosHashIterate(p->pVgMap, NULL);
while (pIter) {
SVDropTbVgReqs *pReqs = pIter;
taosArrayDestroy(pReqs->req.pArray);
pIter = taosHashIterate(p->pVgMap, pIter);
}
taosHashCleanup(p->pVgMap);
}
taosMemoryFree(p);
}
static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx** ppCtx) {
int32_t code = 0;
SMndDropTbsWithTsmaCtx* pCtx = taosMemoryCalloc(1, sizeof(SMndDropTbsWithTsmaCtx));
if (!pCtx) return TSDB_CODE_OUT_OF_MEMORY;
pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (!pCtx->pTsmaMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (!pCtx->pDbMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (!pCtx->pVgMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
*ppCtx = pCtx;
_end:
if (code) mndDestroyDropTbsWithTsmaCtx(pCtx);
return code;
}
static void* mndBuildVDropTbsReq(SMnode* pMnode, const SVgroupInfo* pVgInfo, const SVDropTbBatchReq* pReq, int32_t *len) {
int32_t contLen = 0;
int32_t ret = 0;
SMsgHead *pHead = NULL;
SEncoder encoder = {0};
tEncodeSize(tEncodeSVDropTbBatchReq, pReq, contLen, ret);
if (ret < 0) return NULL;
contLen += sizeof(SMsgHead);
pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgInfo->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
tEncodeSVDropTbBatchReq(&encoder, pReq);
tEncoderClear(&encoder);
*len = contLen;
return pHead;
}
static int32_t mndSetDropTbsRedoActions(SMnode* pMnode, STrans* pTrans, const SVDropTbVgReqs* pVgReqs, void* pCont, int32_t contLen) {
STransAction action = {0};
action.epSet = pVgReqs->info.epSet;
action.pCont = pCont;
action.contLen = contLen;
action.msgType = TDMT_VND_DROP_TABLE;
action.acceptableCode = TSDB_CODE_TDB_TABLE_NOT_EXIST;
return mndTransAppendRedoAction(pTrans, &action);
}
static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg* pRsp, SMndDropTbsWithTsmaCtx* pCtx) {
SMnode *pMnode = pRsp->info.node;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pRsp, "drop-tbs");
mndTransSetChangeless(pTrans);
if (pTrans == NULL) goto _OVER;
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
void* pIter = taosHashIterate(pCtx->pVgMap, NULL);
while (pIter) {
const SVDropTbVgReqs* pVgReqs = pIter;
int32_t len = 0;
void* p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len);
if (!p || mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len) != 0) {
taosHashCancelIterate(pCtx->pVgMap, pIter);
goto _OVER;
}
pIter = taosHashIterate(pCtx->pVgMap, pIter);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
_OVER:
mndTransDrop(pTrans);
return terrno;
}
static int32_t mndProcessDropTbWithTsma(SRpcMsg* pReq) {
int32_t code = -1;
SMnode *pMnode = pReq->info.node;
SDbObj *pDb = NULL;
SStbObj *pStb = NULL;
SMDropTbsReq dropReq = {0};
bool locked = false;
if (tDeserializeSMDropTbsReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
SMndDropTbsWithTsmaCtx* pCtx = NULL;
terrno = mndInitDropTbsWithTsmaCtx(&pCtx);
if (terrno) goto _OVER;
for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) {
SMDropTbReqsOnSingleVg* pReq = taosArrayGet(dropReq.pVgReqs, i);
terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId);
if (terrno) goto _OVER;
}
if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0)
code = 0;
_OVER:
tFreeSMDropTbsReq(&dropReq);
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
return code;
}
static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid,
bool ignoreNotExists) {
SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists};
SVDropTbVgReqs * pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
SVDropTbVgReqs reqs = {0};
if (pReqs == NULL) {
reqs.info = *pVgInfo;
reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq));
taosArrayPush(reqs.req.pArray, &req);
taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs));
} else {
taosArrayPush(pReqs->req.pArray, &req);
}
return 0;
}
static int32_t mndGetDbVgInfoForTsma(SMnode* pMnode, const char* dbname, SMDropTbTsmaInfo* pInfo) {
int32_t code = 0;
SDbObj* pDb = mndAcquireDb(pMnode, dbname);
if (!pDb) {
code = TSDB_CODE_MND_DB_NOT_EXIST;
goto _end;
}
pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if ( !pInfo->dbInfo.dbVgInfos) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix;
pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix;
pInfo->dbInfo.hashMethod = pDb->cfg.hashMethod;
_end:
if (pDb) mndReleaseDb(pMnode, pDb);
if (code && pInfo->dbInfo.dbVgInfos) {
taosArrayDestroy(pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.dbVgInfos = NULL;
}
return code;
}
int32_t vgHashValCmp(const void* lp, const void* rp) {
uint32_t* key = (uint32_t*)lp;
SVgroupInfo* pVg = (SVgroupInfo*)rp;
if (*key < pVg->hashBegin) {
return -1;
} else if (*key > pVg->hashEnd) {
return 1;
}
return 0;
}
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode* pMnode, SMndDropTbsWithTsmaCtx* pCtx, SArray* pTbs, int32_t vgId) {
int32_t code = 0;
SVgObj* pVgObj = mndAcquireVgroup(pMnode, vgId);
if (!pVgObj) {
code = 0;
goto _end;
}
SVgroupInfo vgInfo = {.hashBegin = pVgObj->hashBegin, .hashEnd = pVgObj->hashEnd, .numOfTable = pVgObj->numOfTables, .vgId = pVgObj->vgId};
vgInfo.epSet = mndGetVgroupEpset(pMnode, pVgObj);
mndReleaseVgroup(pMnode, pVgObj);
// get all stb uids
for (int32_t i = 0; i < pTbs->size; ++i) {
const SVDropTbReq* pTb = taosArrayGet(pTbs, i);
if (taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid))) {
} else {
SMDropTbTsmaInfos infos = {0};
infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo));
if (!infos.pTsmaInfos) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos));
}
}
void *pIter = NULL;
SSmaObj *pSma = NULL;
char buf[TSDB_TABLE_FNAME_LEN] = {0};
// get used tsmas and it's dbs
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma);
if (!pIter) break;
SMDropTbTsmaInfos* pInfos = taosHashGet(pCtx->pTsmaMap, &pSma->stbUid, sizeof(pSma->stbUid));
if (pInfos) {
SMDropTbTsmaInfo info = {0};
int32_t len = sprintf(buf, "%s", pSma->name);
len = taosCreateMD5Hash(buf, len);
sprintf(info.tsmaResTbDbFName, "%s", pSma->db);
snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_NAME_LEN, "%s", buf);
SMDropTbDbInfo* pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN);
info.suid = pSma->dstTbUid;
if (!pDbInfo) {
code = mndGetDbVgInfoForTsma(pMnode, pSma->db, &info);
if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pSma);
goto _end;
}
taosHashPut(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN, &info.dbInfo, sizeof(SMDropTbDbInfo));
} else {
info.dbInfo = *pDbInfo;
}
taosArrayPush(pInfos->pTsmaInfos, &info);
}
sdbRelease(pMnode->pSdb, pSma);
}
// generate vg req map
for (int32_t i = 0; i < pTbs->size; ++i) {
SVDropTbReq* pTb = taosArrayGet(pTbs, i);
mndDropTbAdd(pMnode, pCtx->pVgMap, &vgInfo, pTb->name, pTb->suid, pTb->igNotExists);
SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid));
SArray *pVgInfos = NULL;
char buf[TSDB_TABLE_FNAME_LEN];
for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) {
SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j);
int32_t len = sprintf(buf, "%s.%s_%s", pInfo->tsmaResTbDbFName, pInfo->tsmaResTbNamePrefix, pTb->name);
uint32_t hashVal =
taosGetTbHashVal(buf, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix, pInfo->dbInfo.hashSuffix);
const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ);
void* p = taosStrdup(buf + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN);
taosArrayPush(pCtx->pResTbNames, &p);
mndDropTbAdd(pMnode, pCtx->pVgMap, pVgInfo, p, pInfo->suid, true);
}
}
_end:
return code;
}
static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) {
int32_t code = -1;
SDecoder decoder = {0};
SMnode *pMnode = pRsp->info.node;
SVFetchTtlExpiredTbsRsp rsp = {0};
SMndDropTbsWithTsmaCtx *pCtx = NULL;
if (pRsp->code != TSDB_CODE_SUCCESS) goto _end;
if (pRsp->contLen == 0) {
code = 0;
goto _end;
}
tDecoderInit(&decoder, pRsp->pCont, pRsp->contLen);
terrno = tDecodeVFetchTtlExpiredTbsRsp(&decoder, &rsp);
if (terrno) goto _end;
terrno = mndInitDropTbsWithTsmaCtx(&pCtx);
if (terrno) goto _end;
terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId);
if (terrno) goto _end;
if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0)
code = 0;
_end:
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
tDecoderClear(&decoder);
tFreeFetchTtlExpiredTbsRsp(&rsp);
return code;
}

View File

@ -41,6 +41,10 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream);
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq);
@ -102,6 +106,13 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
// for msgs inside mnode
// TODO change the name
mndSetMsgHandle(pMnode, TDMT_STREAM_CREATE, mndProcessCreateStreamReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_STREAM_CREATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_DROP, mndProcessDropStreamReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_STREAM_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
@ -335,7 +346,10 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj->createTime = taosGetTimestampMs();
pObj->updateTime = pObj->createTime;
pObj->version = 1;
pObj->smaId = 0;
if (pCreate->smaId > 0) {
pObj->subTableWithoutMd5 = 1;
}
pObj->smaId = pCreate->smaId;
pObj->indexForMultiAggBalance = -1;
pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));
@ -817,23 +831,57 @@ _OVER:
return terrno;
}
int64_t mndStreamGenChkpId(SMnode *pMnode) {
int64_t mndStreamGenChkptId(SMnode *pMnode, bool lock) {
SStreamObj *pStream = NULL;
void *pIter = NULL;
SSdb *pSdb = pMnode->pSdb;
int64_t maxChkpId = 0;
int64_t maxChkptId = 0;
while (1) {
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
maxChkpId = TMAX(maxChkpId, pStream->checkpointId);
maxChkptId = TMAX(maxChkptId, pStream->checkpointId);
mDebug("stream:%p, %s id:%" PRIx64 "checkpoint %" PRId64 "", pStream, pStream->name, pStream->uid,
pStream->checkpointId);
sdbRelease(pSdb, pStream);
}
mDebug("generated checkpoint %" PRId64 "", maxChkpId + 1);
return maxChkpId + 1;
{ // check the max checkpoint id from all vnodes.
int64_t maxCheckpointId = -1;
if (lock) {
taosThreadMutexLock(&execInfo.lock);
}
for (int32_t i = 0; i < taosArrayGetSize(execInfo.pTaskList); ++i) {
STaskId *p = taosArrayGet(execInfo.pTaskList, i);
STaskStatusEntry *pEntry = taosHashGet(execInfo.pTaskMap, p, sizeof(*p));
if (pEntry == NULL) {
continue;
}
if (pEntry->checkpointInfo.failed) {
continue;
}
if (maxCheckpointId < pEntry->checkpointInfo.latestId) {
maxCheckpointId = pEntry->checkpointInfo.latestId;
}
}
if (lock) {
taosThreadMutexUnlock(&execInfo.lock);
}
if (maxCheckpointId > maxChkptId) {
mDebug("max checkpointId in mnode:%" PRId64 ", smaller than max checkpointId in vnode:%" PRId64, maxChkptId,
maxCheckpointId);
maxChkptId = maxCheckpointId;
}
}
mDebug("generated checkpoint %" PRId64 "", maxChkptId + 1);
return maxChkptId + 1;
}
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
@ -844,7 +892,7 @@ static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq) {
}
SMStreamDoCheckpointMsg *pMsg = rpcMallocCont(sizeof(SMStreamDoCheckpointMsg));
pMsg->checkpointId = mndStreamGenChkpId(pMnode);
pMsg->checkpointId = mndStreamGenChkptId(pMnode, true);
int32_t size = sizeof(SMStreamDoCheckpointMsg);
SRpcMsg rpcMsg = {.msgType = TDMT_MND_STREAM_BEGIN_CHECKPOINT, .pCont = pMsg, .contLen = size};
@ -1179,6 +1227,24 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
}
}
if (pStream->smaId != 0) {
void *pIter = NULL;
SSmaObj *pSma = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
while (pIter) {
if (pSma && pSma->uid == pStream->smaId) {
sdbRelease(pMnode->pSdb, pSma);
sdbRelease(pMnode->pSdb, pStream);
sdbCancelFetch(pMnode->pSdb, pIter);
tFreeMDropStreamReq(&dropReq);
terrno = TSDB_CODE_TSMA_MUST_BE_DROPPED;
return -1;
}
if (pSma) sdbRelease(pMnode->pSdb, pSma);
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
}
}
if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq);
@ -2301,7 +2367,7 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
int32_t total = taosArrayGetSize(*pReqTaskList);
if (total == numOfTasks) { // all tasks has send the reqs
int64_t checkpointId = mndStreamGenChkpId(pMnode);
int64_t checkpointId = mndStreamGenChkptId(pMnode, false);
mInfo("stream:0x%" PRIx64 " all tasks req checkpoint, start checkpointId:%" PRId64, req.streamId, checkpointId);
if (pStream != NULL) { // TODO:handle error
@ -2338,3 +2404,25 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
return 0;
}
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessCreateStreamReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}
static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessDropStreamReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}

View File

@ -352,6 +352,11 @@ void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2
*/
int32_t sdbGetSize(SSdb *pSdb, ESdbType type);
/**
* @brief get valid number of rows, removed rows are ignored
*/
int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type);
/**
* @brief Get the max id of the table, keyType of table should be INT32
*

View File

@ -510,3 +510,15 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type) {
return pSdb->tableVer[type];
}
bool countValid(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
int32_t* pInt = p1;
(*pInt) += 1;
return true;
}
int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type) {
int32_t num = 0;
sdbTraverse(pSdb, type, countValid, &num, 0, 0);
return num;
}

View File

@ -126,7 +126,7 @@ int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2);
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// STSDBRowIter
int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);

View File

@ -120,6 +120,7 @@ void vnodeQueryClose(SVnode* pVnode);
int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg);
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
// vnodeCommit.c
int32_t vnodeBegin(SVnode* pVnode);

View File

@ -294,6 +294,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg);
int32_t tqStreamProgressRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
// sma
int32_t smaInit();

View File

@ -936,7 +936,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
atomic_store_32(&pTask->status.inScanHistorySentinel, 0);
if (retInfo.ret == TASK_SCANHISTORY_REXEC) {
streamReExecScanHistoryFuture(pTask, retInfo.idleTime);
streamExecScanHistoryInFuture(pTask, retInfo.idleTime);
} else {
SStreamTaskState* p = streamTaskGetStatus(pTask);
ETaskStatus s = p->state;
@ -1031,6 +1031,44 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
return 0;
}
int32_t tqStreamProgressRetrieveReq(STQ *pTq, SRpcMsg *pMsg) {
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamProgressReq req;
char* pRspBuf = taosMemoryCalloc(1, sizeof(SMsgHead) + sizeof(SStreamProgressRsp));
SStreamProgressRsp* pRsp = POINTER_SHIFT(pRspBuf, sizeof(SMsgHead));
if (!pRspBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER;
}
code = tDeserializeStreamProgressReq(msgBody, msgLen, &req);
if (code == TSDB_CODE_SUCCESS) {
code = tqGetStreamExecInfo(pTq->pVnode, req.streamId, &pRsp->progressDelay, &pRsp->fillHisFinished);
}
if (code == TSDB_CODE_SUCCESS) {
pRsp->fetchIdx = req.fetchIdx;
pRsp->subFetchIdx = req.subFetchIdx;
pRsp->vgId = req.vgId;
pRsp->streamId = req.streamId;
tSerializeStreamProgressRsp(pRsp, sizeof(SStreamProgressRsp) + sizeof(SMsgHead), pRsp);
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
rsp.pCont = pRspBuf;
pRspBuf = NULL;
rsp.contLen = sizeof(SMsgHead) + sizeof(SStreamProgressRsp);
tmsgSendRsp(&rsp);
}
_OVER:
if (pRspBuf) {
taosMemoryFree(pRspBuf);
}
return code;
}
int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta;

View File

@ -45,7 +45,7 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
// 1. the vnode has already been restored.
// 2. the vnode should be the leader.
// 3. the stream is not suspended yet.
if ((!tsDisableStream) && (numOfTasks > 0) && (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE)) {
if ((!tsDisableStream) && (numOfTasks > 0) /* && (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE)*/) {
tqScanWalAsync(pTq, true);
}

View File

@ -794,8 +794,8 @@ int32_t setDstTableDataUid(SVnode* pVnode, SStreamTask* pTask, SSDataBlock* pDat
return TDB_CODE_SUCCESS;
}
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema* pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id) {
int32_t tqSetDstTableDataPayload(uint64_t suid, const STSchema *pTSchema, int32_t blockIndex, SSDataBlock* pDataBlock,
SSubmitTbData* pTableData, const char* id) {
int32_t numOfRows = pDataBlock->info.rows;
tqDebug("s-task:%s sink data pipeline, build submit msg from %dth resBlock, including %d rows, dst suid:%" PRId64, id,

View File

@ -531,6 +531,11 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
*fhFinished = !HAS_RELATED_FILLHISTORY_TASK(pTask);
int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader);
if (ver == -1) {
ver = pTask->chkInfo.processedVer;
} else {
ver--;
}
SVersionRange verRange = {0};
walReaderValidVersionRange(pTask->exec.pWalReader, &verRange.minVer, &verRange.maxVer);
@ -545,13 +550,17 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
int64_t latest = 0;
code = walFetchHead(pReader, ver);
if (code != TSDB_CODE_SUCCESS) {
if (code == TSDB_CODE_SUCCESS) {
cur = pReader->pHead->head.ingestTs;
}
code = walFetchHead(pReader, verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
latest = pReader->pHead->head.ingestTs;
if (ver == verRange.maxVer) {
latest = cur;
} else {
code = walFetchHead(pReader, verRange.maxVer);
if (code == TSDB_CODE_SUCCESS) {
latest = pReader->pHead->head.ingestTs;
}
}
if (pDelay != NULL) { // delay in ms

View File

@ -124,7 +124,7 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
if (p1->pks[0].val == p2->pks[0].val) {
return 0;
} else {
return p1->pks[0].val > p2->pks[0].val? 1:-1;
return tValueCompare(&p1->pks[0], &p2->pks[0]);
}
}
}
@ -5037,12 +5037,11 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) {
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int32_t step = asc ? 1 : -1;
int64_t ts = 0;
if (asc) {
ts = (pReader->info.window.skey > INT64_MIN)? pReader->info.window.skey-1:pReader->info.window.skey;
ts = (pReader->info.window.skey > INT64_MIN) ? pReader->info.window.skey - 1 : pReader->info.window.skey;
} else {
ts = (pReader->info.window.ekey < INT64_MAX)? pReader->info.window.ekey + 1:pReader->info.window.ekey;
ts = (pReader->info.window.ekey < INT64_MAX) ? pReader->info.window.ekey + 1 : pReader->info.window.ekey;
}
resetAllDataBlockScanInfo(pStatus->pTableMap, ts, step);

View File

@ -136,20 +136,51 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
if (numOfPks > 0) {
pKey->pks[0].type = type;
if (IS_NUMERIC_TYPE(pKey->pks[0].type)) {
if (IS_NUMERIC_TYPE(type)) {
if (asc) {
switch(pKey->pks[0].type) {
case TSDB_DATA_TYPE_BIGINT:pKey->pks[0].val = INT64_MIN;break;
case TSDB_DATA_TYPE_INT:pKey->pks[0].val = INT32_MIN;break;
case TSDB_DATA_TYPE_SMALLINT:pKey->pks[0].val = INT16_MIN;break;
case TSDB_DATA_TYPE_TINYINT:pKey->pks[0].val = INT8_MIN;break;
switch(type) {
case TSDB_DATA_TYPE_BIGINT: {
pKey->pks[0].val = INT64_MIN;
break;
}
case TSDB_DATA_TYPE_INT:{
int32_t min = INT32_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_SMALLINT:{
int16_t min = INT16_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_TINYINT:{
int8_t min = INT8_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT: {
pKey->pks[0].val = 0;
break;
}
default:
ASSERT(0);
}
} else {
switch(pKey->pks[0].type) {
switch(type) {
case TSDB_DATA_TYPE_BIGINT:pKey->pks[0].val = INT64_MAX;break;
case TSDB_DATA_TYPE_INT:pKey->pks[0].val = INT32_MAX;break;
case TSDB_DATA_TYPE_SMALLINT:pKey->pks[0].val = INT16_MAX;break;
case TSDB_DATA_TYPE_TINYINT:pKey->pks[0].val = INT8_MAX;break;
case TSDB_DATA_TYPE_UBIGINT:pKey->pks[0].val = UINT64_MAX;break;
case TSDB_DATA_TYPE_UINT:pKey->pks[0].val = UINT32_MAX;break;
case TSDB_DATA_TYPE_USMALLINT:pKey->pks[0].val = UINT16_MAX;break;
case TSDB_DATA_TYPE_UTINYINT:pKey->pks[0].val = UINT8_MAX;break;
default:
ASSERT(0);
}
}
} else {

View File

@ -323,6 +323,9 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
case TDMT_VND_TABLE_CFG:
vnodeGetTableCfg(pVnode, &reqMsg, false);
break;
case TDMT_VND_GET_STREAM_PROGRESS:
vnodeGetStreamProgress(pVnode, &reqMsg, false);
break;
default:
qError("invalid req msgType %d", req->msgType);
reqMsg.code = TSDB_CODE_INVALID_MSG;
@ -711,3 +714,54 @@ void *vnodeGetIvtIdx(void *pVnode) {
int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) {
return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
}
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct) {
int32_t code = 0;
SStreamProgressReq req;
SStreamProgressRsp rsp = {0};
SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0};
char * buf = NULL;
int32_t rspLen = 0;
code = tDeserializeStreamProgressReq(pMsg->pCont, pMsg->contLen, &req);
if (code == TSDB_CODE_SUCCESS) {
rsp.fetchIdx = req.fetchIdx;
rsp.subFetchIdx = req.subFetchIdx;
rsp.vgId = req.vgId;
rsp.streamId = req.streamId;
rspLen = tSerializeStreamProgressRsp(0, 0, &rsp);
if (direct) {
buf = rpcMallocCont(rspLen);
} else {
buf = taosMemoryCalloc(1, rspLen);
}
if (!buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER;
}
}
if (code == TSDB_CODE_SUCCESS) {
code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished);
}
if (code == TSDB_CODE_SUCCESS) {
tSerializeStreamProgressRsp(buf, rspLen, &rsp);
rpcMsg.pCont = buf;
buf = NULL;
rpcMsg.contLen = rspLen;
rpcMsg.code = code;
rpcMsg.msgType = pMsg->msgType;
if (direct) {
tmsgSendRsp(&rpcMsg);
} else {
*pMsg = rpcMsg;
}
}
_OVER:
if (buf) {
taosMemoryFree(buf);
}
return code;
}

View File

@ -50,6 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l
static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp);
extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg);
@ -484,6 +485,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
case TDMT_VND_ALTER_TABLE: {
code = vnodePreProcessAlterTableMsg(pVnode, pMsg);
} break;
case TDMT_VND_FETCH_TTL_EXPIRED_TBS:
case TDMT_VND_DROP_TTL_TABLE: {
code = vnodePreProcessDropTtlMsg(pVnode, pMsg);
} break;
@ -563,6 +565,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
case TDMT_VND_DROP_TTL_TABLE:
if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_FETCH_TTL_EXPIRED_TBS:
if (vnodeProcessFetchTtlExpiredTbs(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_TRIM:
if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
@ -838,6 +843,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
case TDMT_VND_GET_STREAM_PROGRESS:
return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in stream queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR;
@ -924,6 +931,75 @@ end:
return ret;
}
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) {
int32_t code = -1;
SMetaReader mr = {0};
SVDropTtlTableReq ttlReq = {0};
SVFetchTtlExpiredTbsRsp rsp = {0};
SEncoder encoder = {0};
SArray* pNames = NULL;
pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
pRsp->contLen = 0;
if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _end;
}
ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids));
tb_uid_t suid;
char ctbName[TSDB_TABLE_NAME_LEN];
SVDropTbReq expiredTb = {.igNotExists = true};
metaReaderDoInit(&mr, pVnode->pMeta, 0);
rsp.vgId = TD_VID(pVnode);
rsp.pExpiredTbs = taosArrayInit(ttlReq.nUids, sizeof(SVDropTbReq));
if (!rsp.pExpiredTbs) goto _end;
pNames = taosArrayInit(ttlReq.nUids, TSDB_TABLE_NAME_LEN);
if (!pNames) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
char buf[TSDB_TABLE_NAME_LEN];
for (int32_t i = 0; i < ttlReq.nUids; ++i) {
tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i);
expiredTb.suid = *uid;
terrno = metaReaderGetTableEntryByUid(&mr, *uid);
if (terrno < 0) goto _end;
strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN);
void* p = taosArrayPush(pNames, buf);
expiredTb.name = p;
if (mr.me.type == TSDB_CHILD_TABLE) {
expiredTb.suid = mr.me.ctbEntry.suid;
}
taosArrayPush(rsp.pExpiredTbs, &expiredTb);
}
int32_t ret = 0;
tEncodeSize(tEncodeVFetchTtlExpiredTbsRsp, &rsp, pRsp->contLen, ret);
pRsp->pCont = rpcMallocCont(pRsp->contLen);
if (pRsp->pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _end;
}
tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen);
terrno = tEncodeVFetchTtlExpiredTbsRsp(&encoder, &rsp);
tEncoderClear(&encoder);
if (terrno == 0) code = 0;
_end:
metaReaderClear(&mr);
tFreeFetchTtlExpiredTbsRsp(&rsp);
taosArrayDestroy(ttlReq.pTbUids);
if (pNames) taosArrayDestroy(pNames);
pRsp->code = terrno;
return code;
}
static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SDecoder coder;

View File

@ -31,6 +31,7 @@ extern "C" {
#define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000
#define CTG_DEFAULT_CACHE_VIEW_NUMBER 256
#define CTG_DEFAULT_CACHE_TSMA_NUMBER 10
#define CTG_DEFAULT_RENT_SECOND 10
#define CTG_DEFAULT_RENT_SLOT_SIZE 10
#define CTG_DEFAULT_MAX_RETRY_TIMES 3
@ -69,6 +70,7 @@ typedef enum {
CTG_CI_UDF,
CTG_CI_SVR_VER,
CTG_CI_VIEW,
CTG_CI_TBL_TSMA,
CTG_CI_MAX_VALUE,
} CTG_CACHE_ITEM;
@ -85,6 +87,7 @@ enum {
CTG_RENT_DB = 1,
CTG_RENT_STABLE,
CTG_RENT_VIEW,
CTG_RENT_TSMA,
};
enum {
@ -101,6 +104,8 @@ enum {
CTG_OP_DROP_TB_INDEX,
CTG_OP_UPDATE_VIEW_META,
CTG_OP_DROP_VIEW_META,
CTG_OP_UPDATE_TB_TSMA,
CTG_OP_DROP_TB_TSMA,
CTG_OP_CLEAR_CACHE,
CTG_OP_MAX
};
@ -123,6 +128,8 @@ typedef enum {
CTG_TASK_GET_TB_HASH_BATCH,
CTG_TASK_GET_TB_TAG,
CTG_TASK_GET_VIEW,
CTG_TASK_GET_TB_TSMA,
CTG_TASK_GET_TSMA,
} CTG_TASK_TYPE;
typedef enum {
@ -255,14 +262,47 @@ typedef struct SCtgViewsCtx {
SArray* pFetchs;
} SCtgViewsCtx;
typedef enum {
FETCH_TSMA_SOURCE_TB_META,
FETCH_TB_TSMA,
FETCH_TSMA_STREAM_PROGRESS,
} CTG_TSMA_FETCH_TYPE;
typedef struct SCtgTSMAFetch {
CTG_TSMA_FETCH_TYPE fetchType;
int32_t dbIdx;
int32_t tbIdx;
int32_t fetchIdx;
int32_t resIdx;
// tb meta
int32_t flag;
int32_t vgId;
// stream progress
int32_t subFetchNum;
int32_t finishedSubFetchNum;
int32_t vgNum;
// tb tsma
SName tsmaSourceTbName;
} SCtgTSMAFetch;
typedef struct SCtgTbTSMACtx {
int32_t fetchNum;
SArray* pNames; // SArray<STablesReq>
SArray* pResList;
SArray* pFetches;
} SCtgTbTSMACtx;
typedef STableIndexRsp STableIndex;
typedef STableTSMAInfo STSMACache;
typedef struct SCtgTbCache {
SRWLatch metaLock;
SRWLatch indexLock;
STableMeta* pMeta;
STableIndex* pIndex;
SRWLatch metaLock;
SRWLatch indexLock;
STableMeta* pMeta;
STableIndex* pIndex;
} SCtgTbCache;
typedef struct SCtgVgCache {
@ -280,6 +320,10 @@ typedef struct SCtgViewCache {
SViewMeta* pMeta;
} SCtgViewCache;
typedef struct SCtgTSMACache {
SRWLatch tsmaLock;
SArray* pTsmas; // SArray<STSMACache*>
} SCtgTSMACache;
typedef struct SCtgDBCache {
SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads
@ -290,6 +334,8 @@ typedef struct SCtgDBCache {
SHashObj* viewCache; // key:viewname, value:SCtgViewCache
SHashObj* tbCache; // key:tbname, value:SCtgTbCache
SHashObj* stbCache; // key:suid, value:char*
SHashObj* tsmaCache; // key:tbname, value: SCtgTSMACache
int32_t tsmaVersion;
uint64_t dbCacheNum[CTG_CI_MAX_VALUE];
uint64_t dbCacheSize;
} SCtgDBCache;
@ -325,6 +371,7 @@ typedef struct SCatalog {
SCtgRentMgmt dbRent;
SCtgRentMgmt stbRent;
SCtgRentMgmt viewRent;
SCtgRentMgmt tsmaRent;
SCtgCacheStat cacheStat;
} SCatalog;
@ -370,6 +417,8 @@ typedef struct SCtgJob {
int32_t tbCfgNum;
int32_t svrVerNum;
int32_t viewNum;
int32_t tbTsmaNum;
int32_t tsmaNum; // currently, only 1 is possible
} SCtgJob;
typedef struct SCtgMsgCtx {
@ -548,6 +597,24 @@ typedef struct SCtgDropViewMetaMsg {
uint64_t viewId;
} SCtgDropViewMetaMsg;
typedef struct SCtgUpdateTbTSMAMsg {
SCatalog* pCtg;
STableTSMAInfo* pTsma;
int32_t dbTsmaVersion;
uint64_t dbId;
} SCtgUpdateTbTSMAMsg;
typedef struct SCtgDropTbTSMAMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
char tsmaName[TSDB_TABLE_NAME_LEN];
uint64_t tsmaId;
uint64_t dbId;
uint64_t tbId;
bool dropAllForTb;
} SCtgDropTbTSMAMsg;
typedef struct SCtgCacheOperation {
int32_t opId;
@ -726,7 +793,9 @@ typedef struct SCtgCacheItemInfo {
(CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || \
(CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_IS_BATCH_TASK(_taskType) ((CTG_TASK_GET_TB_META_BATCH == (_taskType)) || (CTG_TASK_GET_TB_HASH_BATCH == (_taskType)) || (CTG_TASK_GET_VIEW == (_taskType)))
#define CTG_IS_BATCH_TASK(_taskType) \
((CTG_TASK_GET_TB_META_BATCH == (_taskType)) || (CTG_TASK_GET_TB_HASH_BATCH == (_taskType)) || \
(CTG_TASK_GET_VIEW == (_taskType)) || (CTG_TASK_GET_TB_TSMA == (_taskType)))
#define CTG_GET_TASK_MSGCTX(_task, _id) \
(CTG_IS_BATCH_TASK((_task)->type) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx)
@ -917,7 +986,7 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache);
void ctgReleaseDBCache(SCatalog* pCtg, SCtgDBCache* dbCache);
void ctgRUnlockVgInfo(SCtgDBCache* dbCache);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32_t* exist);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, const char* dbFName, const char* tbName, int32_t* exist);
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType,
uint64_t* suid, char* stbName);
@ -943,10 +1012,12 @@ int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t si
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare);
void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgRemoveTSMARent(SCatalog* pCtg, SCtgDBCache* dbCache);
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache);
int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId,
SCtgViewCache *pCache);
SCtgViewCache *pCache);
int32_t ctgUpdateRentTSMAVersion(SCatalog* pCtg, char* dbFName, const STSMACache* pCache);
int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq);
int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq);
int32_t ctgStartUpdateThread();
@ -976,7 +1047,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch
SCtgTask* pTask);
int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, SGetUserAuthRsp* out,
SCtgTask* pTask);
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName,
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, const char* tbName,
STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out,
SCtgTaskReq* tReq);
@ -1018,9 +1089,11 @@ void ctgFreeDbCache(SCtgDBCache* dbCache);
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2);
int32_t ctgViewVersionSortCompare(const void* key1, const void* key2);
int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2);
int32_t ctgTSMAVersionSortCompare(const void* key1, const void* key2);
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2);
int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2);
int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2);
int32_t ctgTSMAVersionSearchCompare(const void* key1, const void* key2);
void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput);
int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target);
int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target);
@ -1073,6 +1146,26 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid
int32_t ctgGetTbTagCb(SCtgTask* pTask);
int32_t ctgGetUserCb(SCtgTask* pTask);
int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx,
SArray* pList);
int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaName);
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType);
int32_t ctgUpdateTbTSMAEnqueue(SCatalog* pCtg, STSMACache** pTsma, int32_t tsmaVersion, bool syncOp);
int32_t ctgDropTSMAForTbEnqueue(SCatalog* pCtg, SName* pName, bool syncOp);
int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp);
int32_t ctgOpDropTbTSMA(SCtgCacheOperation* operation);
int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation* operation);
uint64_t ctgGetTbTSMACacheSize(STSMACache* pTsmaInfo);
void ctgFreeTbTSMAInfo(void* p);
bool hasOutOfDateTSMACache(SArray* pTsmas);
bool isCtgTSMACacheOutOfDate(STSMACache* pTsmaCache);
int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTbName,
SVgroupInfo* vgroupInfo, SStreamProgressRsp* out, SCtgTaskReq* tReq,
void* bInput);
int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag,
CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName);
extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug;
extern SCtgAsyncFps gCtgAsyncFps[];

View File

@ -759,6 +759,10 @@ int32_t catalogInit(SCatalogCfg* cfg) {
gCtgMgmt.cfg.maxViewCacheNum = CTG_DEFAULT_CACHE_VIEW_NUMBER;
}
if (gCtgMgmt.cfg.maxTSMACacheNum == 0) {
gCtgMgmt.cfg.maxTSMACacheNum = CTG_DEFAULT_CACHE_TSMA_NUMBER;
}
if (gCtgMgmt.cfg.dbRentSec == 0) {
gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND;
}
@ -770,13 +774,19 @@ int32_t catalogInit(SCatalogCfg* cfg) {
if (gCtgMgmt.cfg.viewRentSec == 0) {
gCtgMgmt.cfg.viewRentSec = CTG_DEFAULT_RENT_SECOND;
}
if (gCtgMgmt.cfg.tsmaRentSec == 0) {
gCtgMgmt.cfg.tsmaRentSec = CTG_DEFAULT_RENT_SECOND;
}
} else {
gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER;
gCtgMgmt.cfg.maxViewCacheNum = CTG_DEFAULT_CACHE_VIEW_NUMBER;
gCtgMgmt.cfg.maxTSMACacheNum = CTG_DEFAULT_CACHE_TSMA_NUMBER;
gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.viewRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.tsmaRentSec = CTG_DEFAULT_RENT_SECOND;
}
gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT),
@ -862,6 +872,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)));
clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
false, HASH_ENTRY_LOCK);
@ -1562,6 +1573,16 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
int32_t catalogGetExpiredTsmas(SCatalog* pCtg, STSMAVersion** tsmas, uint32_t* num) {
CTG_API_ENTER();
if (!pCtg || !tsmas || !num) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->tsmaRent, (void**)tsmas, num, sizeof(STSMAVersion)));
}
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg) {
CTG_API_ENTER();
@ -1769,6 +1790,122 @@ int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName*
CTG_API_LEAVE(TSDB_CODE_OPS_NOT_SUPPORT);
}
int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma, int32_t tsmaVersion) {
CTG_API_ENTER();
if (!pCtg || !ppTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma) {
CTG_API_ENTER();
if (!pCtg || !pTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma) {
CTG_API_ENTER();
int32_t code = 0;
if (!pCtg || !pTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
if (!pCtg->dbCache) {
return TSDB_CODE_SUCCESS;
}
CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true));
_return:
CTG_API_LEAVE(code);
}
int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) {
STableTSMAInfoRsp tsmasRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
assert(tsmasRsp.pTsmas);
assert(tsmasRsp.pTsmas->size > 0);
*ppRes = tsmasRsp.pTsmas;
tsmasRsp.pTsmas = NULL;
for (int32_t i = 0; i < (*ppRes)->size; ++i) {
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false));
}
return TSDB_CODE_SUCCESS;
_return:
if (tsmasRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmasRsp);
}
CTG_RET(code);
}
int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetTbTsmas(pCtg, pConn, (SName*)pTableName, pRes));
_return:
CTG_API_LEAVE(code);
}
int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
STableTSMAInfoRsp tsmaRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1);
*pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0);
taosArrayDestroy(tsmaRsp.pTsmas);
tsmaRsp.pTsmas = NULL;
_return:
if (tsmaRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmaRsp);
}
CTG_RET(code);
}
int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
CTG_API_ENTER();
if (!pCtg || !pConn || !pTsmaName) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogClearCache(void) {
CTG_API_ENTER_NOLOCK();

View File

@ -476,6 +476,37 @@ int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) {
SCtgTask task = {0};
task.type = CTG_TASK_GET_TB_TSMA;
task.taskId = taskId;
task.pJob = pJob;
SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx));
if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
task.taskCtx = pTaskCtx;
pTaskCtx->pNames = param;
pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes));
taosArrayPush(pJob->pTasks, &task);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) {
SCtgTask task = {0};
task.type = CTG_TASK_GET_TSMA;
task.taskId = taskId;
task.pJob = pJob;
SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx));
if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
task.taskCtx = pTaskCtx;
pTaskCtx->pNames = param;
pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes));
taosArrayPush(pJob->pTasks, &task);
return 0;
}
int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) {
SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@ -573,6 +604,14 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con
ctgDropTbIndexEnqueue(pCtg, name, true);
}
for (int32_t i = 0; i < pJob->tbTsmaNum; ++i) {
STablesReq* pTbReq = taosArrayGet(pReq->pTableTSMAs, i);
for (int32_t j = 0; j < pTbReq->pTables->size; ++j) {
SName* name = taosArrayGet(pTbReq->pTables, j);
ctgDropTSMAForTbEnqueue(pCtg, name, true);
}
}
// REFRESH VIEW META
return ctgHandleForceUpdateView(pCtg, pReq);
}
@ -614,9 +653,11 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag);
int32_t viewNum = (int32_t)ctgGetTablesReqNum(pReq->pView);
int32_t tbTsmaNum = (int32_t)taosArrayGetSize(pReq->pTableTSMAs);
int32_t tsmaNum = (int32_t)taosArrayGetSize(pReq->pTSMAs);
int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum;
userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum + tbTsmaNum;
*job = taosMemoryCalloc(1, sizeof(SCtgJob));
if (NULL == *job) {
@ -649,6 +690,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
pJob->svrVerNum = svrVerNum;
pJob->tbTagNum = tbTagNum;
pJob->viewNum = viewNum;
pJob->tbTsmaNum = tbTsmaNum;
pJob->tsmaNum = tsmaNum;
#if CTG_BATCH_FETCH
pJob->pBatchs =
@ -741,6 +784,13 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL));
}
if (tbTsmaNum > 0) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL));
}
if (tsmaNum > 0) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TSMA, pReq->pTSMAs, NULL));
}
if (qnodeNum) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL));
}
@ -2526,7 +2576,8 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
SCtgViewsCtx* pCtx = (SCtgViewsCtx*)pTask->taskCtx;
SCtgJob* pJob = pTask->pJob;
bool tbMetaDone = false;
/*
ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone);
if (tbMetaDone) {
CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx));
@ -2535,6 +2586,7 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
*/
int32_t dbNum = taosArrayGetSize(pCtx->pNames);
int32_t fetchIdx = 0;
@ -2572,6 +2624,386 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) {
int32_t code = 0;
SCtgTask* pTask = pReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SCtgTbTSMACtx* pTaskCtx = pTask->taskCtx;
SCtgDBCache* pDbCache = NULL;
STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx);
ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache);
if (pDbCache) {
ctgReleaseVgInfoToCache(pCtg, pDbCache);
} else {
SBuildUseDBInput input = {0};
tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pReq));
}
_return:
return code;
}
int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgTbTSMACtx* pCtx = (SCtgTbTSMACtx*)pTask->taskCtx;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SArray* pRes = NULL;
SCtgJob* pJob = pTask->pJob;
int32_t dbNum = taosArrayGetSize(pCtx->pNames);
int32_t fetchIdx = 0, baseResIdx = 0;
for (int32_t idx = 0; idx < dbNum; ++idx) {
STablesReq* pReq = taosArrayGet(pCtx->pNames, idx);
CTG_ERR_RET(ctgGetTbTSMAFromCache(pCtg, pCtx, idx, &fetchIdx, baseResIdx, pReq->pTables));
baseResIdx += taosArrayGetSize(pReq->pTables);
}
pCtx->fetchNum = taosArrayGetSize(pCtx->pFetches);
if (pCtx->fetchNum <= 0) {
TSWAP(pTask->res, pCtx->pResList);
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum);
for (int32_t i = 0; i < pCtx->fetchNum; ++i) {
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, i);
STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx);
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i);
if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs;
SCtgTaskReq tReq;
tReq.pTask = pTask;
tReq.msgIdx = pFetch->fetchIdx;
switch (pFetch->fetchType) {
case FETCH_TSMA_SOURCE_TB_META: {
CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId));
} break;
case FETCH_TB_TSMA: {
CTG_ERR_RET(
ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, &tReq, TDMT_MND_GET_TABLE_TSMA));
} break;
default:
ASSERT(0);
break;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgTbTSMACtx* pCtx = (SCtgTbTSMACtx*)pTask->taskCtx;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SArray* pRes = NULL;
SCtgJob* pJob = pTask->pJob;
// currently, only support fetching one tsma
ASSERT(pCtx->pNames->size == 1);
STablesReq* pReq = taosArrayGet(pCtx->pNames, 0);
ASSERT(pReq->pTables->size == 1);
SName* pTsmaName = taosArrayGet(pReq->pTables, 0);
CTG_ERR_RET(ctgGetTSMAFromCache(pCtg, pCtx, pTsmaName));
if (pCtx->pResList->size == 0) {
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0);
if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs;
SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0};
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
CTG_ERR_RET(ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, NULL, &tReq, TDMT_MND_GET_TSMA));
} else {
SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0);
STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes;
ASSERT(pRsp->pTsmas->size == 1);
const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0);
TSWAP(pTask->res, pCtx->pResList);
// get tsma target stable meta if not existed in cache
int32_t exists = false;
CTG_ERR_RET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists));
if (!exists) {
SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0};
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0);
if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs;
CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, &tReq));
} else {
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
}
return TSDB_CODE_SUCCESS;
}
return 0;
}
int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0);
SName* pName = taosArrayGet(pTbReq->pTables, 0);
SRequestConnInfo* pConn = &pTask->pJob->conn;
CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
switch (reqType) {
case TDMT_MND_TABLE_META: {
STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out;
if (!CTG_IS_META_NULL(pOut->metaType)) {
CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP));
}
} break;
case TDMT_MND_GET_TSMA: {
STableTSMAInfoRsp* pOut = pMsgCtx->out;
pRes->code = 0;
if (pOut->pTsmas->size > 0) {
ASSERT(pOut->pTsmas->size == 1);
pRes->pRes = pOut;
pMsgCtx->out = NULL;
TSWAP(pTask->res, pCtx->pResList);
STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0);
int32_t exists = false;
CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists));
if (!exists) {
TSWAP(pMsgCtx->lastOut, pMsgCtx->out);
CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq));
}
}
} break;
default:
ASSERT(0);
}
_return:
if (code) {
if (TSDB_CODE_MND_SMA_NOT_EXIST == code) {
code = TSDB_CODE_SUCCESS;
} else {
ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname,
tstrerror(code));
}
}
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
int32_t subFetchIdx = 0;
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
SRequestConnInfo* pConn = &pTask->pJob->conn;
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
SVgroupInfo* pVgInfo = NULL;
pFetch->vgNum = taosHashGetSize(pVgHash);
for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i);
pVgInfo = taosHashIterate(pVgHash, NULL);
pTsmaInfo->reqTs = taosGetTimestampMs();
while (pVgInfo) {
// make StreamProgressReq, send it
SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx,
.streamId = pTsmaInfo->streamUid,
.subFetchIdx = subFetchIdx++,
.vgId = pVgInfo->vgId};
CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req));
pFetch->subFetchNum++;
pVgInfo = taosHashIterate(pVgHash, pVgInfo);
}
}
_return:
CTG_RET(code);
}
int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
bool taskDone = false;
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
SArray* pTsmas = NULL;
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx);
SHashObj* pVgHash = NULL;
SCtgDBCache* pDbCache = NULL;
STableTSMAInfo* pTsma = NULL;
SRequestConnInfo* pConn = &pTask->pJob->conn;
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (reqType != TDMT_VND_GET_STREAM_PROGRESS)
CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
switch (reqType) {
case TDMT_MND_GET_TABLE_TSMA: {
STableTSMAInfoRsp* pOut = pMsgCtx->out;
pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS;
pRes->pRes = pOut;
pMsgCtx->out = NULL;
if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) {
// fetch progress
ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache);
if (!pDbCache) {
// do not know which vnodes to fetch, fetch vnode list first
SBuildUseDBInput input = {0};
tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq));
} else {
// fetch progress from every vnode
CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut));
ctgReleaseVgInfoToCache(pCtg, pDbCache);
pDbCache = NULL;
}
} else {
// no tsmas
if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) {
TSWAP(pTask->res, pCtx->pResList);
taskDone = true;
}
}
} break;
case TDMT_VND_GET_STREAM_PROGRESS: {
SStreamProgressRsp rsp = {0};
CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
// update progress into res
STableTSMAInfoRsp* pTsmasRsp = pRes->pRes;
SArray* pTsmas = pTsmasRsp->pTsmas;
SStreamProgressRsp* pRsp = &rsp;
int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum;
STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx);
if (pTsmaInfo->rspTs == 0) pTsmaInfo->fillHistoryFinished = true;
pTsmaInfo->rspTs = taosGetTimestampMs();
pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration);
pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished;
qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name,
pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay);
if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) {
// subfetch all finished
for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) {
STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i);
CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma));
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false));
}
if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) {
TSWAP(pTask->res, pCtx->pResList);
taskDone = true;
}
}
} break;
case TDMT_MND_USE_DB: {
SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out;
switch (pFetch->fetchType) {
case FETCH_TSMA_SOURCE_TB_META: {
SVgroupInfo vgInfo = {0};
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo));
pFetch->vgId = vgInfo.vgId;
CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq));
} break;
case FETCH_TSMA_STREAM_PROGRESS: {
STableTSMAInfoRsp* pTsmas = pRes->pRes;
TSWAP(pOut->dbVgroup->vgHash, pVgHash);
CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas));
} break;
default:
ASSERT(0);
}
} break;
case TDMT_VND_TABLE_META: {
// handle source tb meta
ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META);
STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out;
pFetch->fetchType = FETCH_TB_TSMA;
pFetch->tsmaSourceTbName = *pTbName;
if (CTG_IS_META_NULL(pOut->metaType)) {
ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname);
ctgRemoveTbMetaFromCache(pCtg, pTbName, false);
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
}
if (META_TYPE_BOTH_TABLE == pOut->metaType) {
// rewrite tsma fetch table with it's super table name
sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName);
}
CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA));
} break;
default:
ASSERT(0);
}
_return:
if (pDbCache) {
ctgReleaseVgInfoToCache(pCtg, pDbCache);
}
if (pTsma) {
tFreeTableTSMAInfo(pTsma);
pTsma = NULL;
}
if (pVgHash) {
taosHashCleanup(pVgHash);
}
if (code) {
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx);
pRes->code = code;
if (TSDB_CODE_MND_SMA_NOT_EXIST == code) {
code = TSDB_CODE_SUCCESS;
} else {
ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname,
tstrerror(code));
}
bool allSubFetchFinished = false;
if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) {
allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum;
}
if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) {
TSWAP(pTask->res, pCtx->pResList);
taskDone = true;
}
}
if (pTask->res && taskDone) {
ctgHandleTaskEnd(pTask, code);
}
CTG_RET(code);
}
int32_t ctgDumpTbTSMARes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pTableTsmas = pTask->res;
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpTSMARes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pTsmas = pTask->res;
return TSDB_CODE_SUCCESS;
}
int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask) {
ctgResetTbMetaTask(pTask);
@ -2690,6 +3122,8 @@ SCtgAsyncFps gCtgAsyncFps[] = {
{ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL},
{ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL},
{ctgInitGetViewsTask, ctgLaunchGetViewsTask, ctgHandleGetViewsRsp, ctgDumpViewsRes, NULL, NULL},
{ctgInitGetTbTSMATask, ctgLaunchGetTbTSMATask, ctgHandleGetTbTSMARsp, ctgDumpTbTSMARes, NULL, NULL},
{ctgInitGetTSMATask, ctgLaunchGetTSMATask, ctgHandleGetTSMARsp, ctgDumpTSMARes, NULL, NULL},
};
int32_t ctgMakeAsyncRes(SCtgJob* pJob) {
@ -2821,7 +3255,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) {
CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask));
pTask = taosArrayGet(pJob->pTasks, i);
pTask->status = CTG_TASK_LAUNCHED;
atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED);
}
if (taskNum <= 0) {

View File

@ -32,6 +32,8 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v
{CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex},
{CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta},
{CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta},
{CTG_OP_UPDATE_TB_TSMA, "update tbTSMA", ctgOpUpdateTbTSMA},
{CTG_OP_DROP_TB_TSMA, "drop tbTSMA", ctgOpDropTbTSMA},
{CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}};
SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = {
@ -52,6 +54,7 @@ SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = {
{"TblTag ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TAG,
{"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_INDEX_INFO,
{"viewMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_VIEW,
{"TblTSMA ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TSMA
{"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_USER,
{"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_UDF,
{"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER,
@ -209,6 +212,17 @@ void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *
}
}
void ctgReleaseTSMAToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTSMACache* pCache) {
if (pCache && dbCache) {
CTG_UNLOCK(CTG_READ, &pCache->tsmaLock);
taosHashRelease(dbCache->tsmaCache, pCache);
}
if (dbCache) {
ctgReleaseDBCache(pCtg, dbCache);
}
}
int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) {
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
@ -245,7 +259,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) {
int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) {
SCtgDBCache *dbCache = NULL;
SCtgTbCache *pCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
@ -489,7 +503,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) {
int32_t ctgTbMetaExistInCache(SCatalog *pCtg, const char *dbFName, const char *tbName, int32_t *exist) {
SCtgDBCache *dbCache = NULL;
SCtgTbCache *tbCache = NULL;
ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache);
@ -1381,6 +1395,13 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
newDBCache.tsmaCache = taosHashInit(gCtgMgmt.cfg.maxTSMACacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
true, HASH_ENTRY_LOCK);
if (!newDBCache.tsmaCache) {
ctgError("taosHashInit %d tsmaCache failed", gCtgMgmt.cfg.maxTSMACacheNum);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache));
if (code) {
if (HASH_NODE_EXIST(code)) {
@ -1394,7 +1415,7 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
CTG_CACHE_NUM_INC(CTG_CI_DB, 1);
SDbCacheInfo dbCacheInfo = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0, .cfgVersion = -1};
SDbCacheInfo dbCacheInfo = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0, .cfgVersion = -1, .tsmaVersion = -1};
tstrncpy(dbCacheInfo.dbFName, dbFName, sizeof(dbCacheInfo.dbFName));
ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId);
@ -1424,6 +1445,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d
atomic_store_8(&dbCache->deleted, 1);
ctgRemoveStbRent(pCtg, dbCache);
ctgRemoveViewRent(pCtg, dbCache);
ctgRemoveTSMARent(pCtg, dbCache);
ctgFreeDbCache(dbCache);
CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock);
@ -1780,7 +1802,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
bool newAdded = false;
SDbCacheInfo dbCacheInfo = {
.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .cfgVersion = -1, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs};
.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .cfgVersion = -1, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs, .tsmaVersion = -1};
SCtgDBCache *dbCache = NULL;
CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache));
@ -1824,6 +1846,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
if (dbCache->cfgCache.cfgInfo) {
dbCacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion;
dbCacheInfo.tsmaVersion = dbCache->tsmaVersion;
}
vgCache->vgInfo = dbInfo;
@ -1891,6 +1914,7 @@ int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation) {
} else {
cacheInfo.vgVersion = -1;
}
cacheInfo.tsmaVersion = dbCache->tsmaVersion;
ctgWLockDbCfgInfo(dbCache);
@ -2074,7 +2098,8 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
}
tblType = pTbCache->pMeta->tableType;
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex));
atomic_sub_fetch_64(&dbCache->dbCacheSize,
ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex));
ctgFreeTbCacheImpl(pTbCache, true);
if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) {
@ -2126,7 +2151,8 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
}
tblType = pTbCache->pMeta->tableType;
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex));
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) +
ctgGetTbIndexCacheSize(pTbCache->pIndex));
ctgFreeTbCacheImpl(pTbCache, true);
if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) {
@ -2554,6 +2580,7 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
case CTG_OP_UPDATE_VG_EPSET:
case CTG_OP_DROP_TB_INDEX:
case CTG_OP_DROP_VIEW_META:
case CTG_OP_DROP_TB_TSMA:
case CTG_OP_CLEAR_CACHE: {
taosMemoryFreeClear(op->data);
break;
@ -2582,6 +2609,14 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
taosMemoryFreeClear(op->data);
break;
}
case CTG_OP_UPDATE_TB_TSMA: {
SCtgUpdateTbTSMAMsg *msg = op->data;
if (msg->pTsma) {
tFreeTableTSMAInfo(msg->pTsma);
taosMemoryFreeClear(msg->pTsma);
}
break;
}
default: {
qError("invalid cache op id:%d", op->opId);
break;
@ -2982,20 +3017,21 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq
CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta));
if (NULL == tblMeta) {
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
return TSDB_CODE_SUCCESS;
}
if (NULL != tblMeta) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
if (TSDB_SUPER_TABLE == tblMeta->tableType) {
CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq));
if (TSDB_SUPER_TABLE == tblMeta->tableType) {
CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq));
} else {
CTG_ERR_JRET(ctgDropTbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq));
}
} else {
CTG_ERR_JRET(ctgDropTbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq));
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
}
CTG_ERR_JRET(ctgDropTSMAForTbEnqueue(pCtg, pTableName, syncReq));
_return:
taosMemoryFreeClear(tblMeta);
@ -3136,5 +3172,462 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx,
SArray* pList) {
int32_t code = 0;
SCtgDBCache * dbCache = NULL;
SCtgTSMACache *pCache = NULL;
char dbFName[TSDB_DB_FNAME_LEN] = {0};
int32_t flag = CTG_FLAG_UNKNOWN_STB;
uint64_t lastSuid = 0;
STableMeta * pTableMeta = NULL;
SName * pName = taosArrayGet(pList, 0);
int32_t tbNum = taosArrayGetSize(pList);
SCtgTbCache * pTbCache = NULL;
if (IS_SYS_DBNAME(pName->dbname)) {
return TSDB_CODE_SUCCESS;
}
tNameGetFullDbName(pName, dbFName);
// get db cache
CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache));
if (!dbCache) {
ctgDebug("DB %s not in cache", dbFName);
for (int32_t i = 0; i < tbNum; ++i) {
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaData){0});
}
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < tbNum; ++i) {
// get tb cache
pName = taosArrayGet(pList, i);
pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname));
if (!pTbCache || !pTbCache->pMeta) {
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
uint64_t suid = pTbCache->pMeta->suid;
int8_t tbType = pTbCache->pMeta->tableType;
taosHashRelease(dbCache->tbCache, pTbCache);
SName tsmaSourceTbName = *pName;
// if child table, get stable name
if (tbType == TSDB_CHILD_TABLE) {
char* stbName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(uint64_t));
if (stbName) {
snprintf(tsmaSourceTbName.tname, TMIN(TSDB_TABLE_NAME_LEN, strlen(stbName) + 1), "%s", stbName);
taosHashRelease(dbCache->stbCache, stbName);
} else {
ctgDebug("stb in db: %s, uid: %" PRId64 " not in cache", dbFName, suid);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
}
// get tsma cache
pCache = taosHashAcquire(dbCache->tsmaCache, tsmaSourceTbName.tname, strlen(tsmaSourceTbName.tname));
if (!pCache || !pCache->pTsmas || pCache->pTsmas->size == 0) {
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
CTG_LOCK(CTG_READ, &pCache->tsmaLock);
if (hasOutOfDateTSMACache(pCache->pTsmas)) {
CTG_UNLOCK(CTG_READ, &pCache->tsmaLock);
taosHashRelease(dbCache->tsmaCache, pCache);
ctgDebug("tsma for tb: %s.%s not in cache", tsmaSourceTbName.tname, dbFName);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1);
continue;
}
CTG_CACHE_HIT_INC(CTG_CI_TBL_TSMA, 1);
STableTSMAInfoRsp *pRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (!pRsp) {
ctgReleaseTSMAToCache(pCtg, dbCache, pCache);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pRsp->pTsmas = taosArrayInit(pCache->pTsmas->size, POINTER_BYTES);
if (!pRsp->pTsmas) {
ctgReleaseTSMAToCache(pCtg, dbCache, pCache);
taosMemoryFreeClear(pRsp);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SMetaRes res = {0};
for (int32_t i = 0; i < pCache->pTsmas->size; ++i) {
STSMACache *pTsmaOut = NULL;
STSMACache *pTsmaCache = taosArrayGetP(pCache->pTsmas, i);
code = tCloneTbTSMAInfo(pTsmaCache, &pTsmaOut);
if (code) {
ctgReleaseTSMAToCache(pCtg, dbCache, pCache);
tFreeTableTSMAInfoRsp(pRsp);
taosMemoryFreeClear(pRsp);
CTG_ERR_RET(code);
}
taosArrayPush(pRsp->pTsmas, &pTsmaOut);
}
res.pRes = pRsp;
taosArrayPush(pCtx->pResList, &res);
CTG_UNLOCK(CTG_READ, &pCache->tsmaLock);
taosHashRelease(dbCache->tsmaCache, pCache);
}
ctgReleaseDBCache(pCtg, dbCache);
CTG_RET(code);
}
int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaName) {
char dbFName[TSDB_DB_FNAME_LEN] = {0};
SCtgDBCache *pDbCache = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SMetaRes res = {0};
bool found = false;
STSMACache * pTsmaOut = NULL;
tNameGetFullDbName(pTsmaName, dbFName);
CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &pDbCache));
if (!pDbCache) {
ctgDebug("DB %s not in cache", dbFName);
CTG_RET(code);
}
void *pIter = taosHashIterate(pDbCache->tsmaCache, NULL);
while (pIter && !found) {
SCtgTSMACache* pCtgCache = pIter;
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
int32_t size = pCtgCache ? (pCtgCache->pTsmas ? pCtgCache->pTsmas->size : 0) : 0;
for (int32_t i = 0; i < size; ++i) {
STSMACache* pCache = taosArrayGetP(pCtgCache->pTsmas, i);
if (memcmp(pCache->name, pTsmaName->tname, TSDB_TABLE_NAME_LEN) == 0) {
found = true;
CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1);
code = tCloneTbTSMAInfo(pCache, &pTsmaOut);
break;
}
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
pIter = taosHashIterate(pDbCache->tsmaCache, pIter);
}
taosHashCancelIterate(pDbCache->tsmaCache, pIter);
if (found && code == TSDB_CODE_SUCCESS) {
res.pRes = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (!res.pRes) {
tFreeAndClearTableTSMAInfo(pTsmaOut);
CTG_RET(TSDB_CODE_OUT_OF_MEMORY);
}
STableTSMAInfoRsp* pRsp = res.pRes;
pRsp->pTsmas = taosArrayInit(1, POINTER_BYTES);
if (!pRsp->pTsmas) {
tFreeAndClearTableTSMAInfo(pTsmaOut);
CTG_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pRsp->pTsmas, &pTsmaOut);
taosArrayPush(pCtx->pResList, &res);
}
ctgReleaseDBCache(pCtg, pDbCache);
CTG_RET(code);
}
int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaVersion, bool syncOp) {
int32_t code = 0;
SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
op->opId = CTG_OP_UPDATE_TB_TSMA;
op->syncOp = syncOp;
SCtgUpdateTbTSMAMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbTSMAMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbTSMAMsg));
taosMemoryFree(op);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
msg->pTsma = *pTsma;
msg->dbTsmaVersion = tsmaVersion;
msg->dbId = (*pTsma)->dbId;
op->data = msg;
CTG_ERR_JRET(ctgEnqueue(pCtg, op));
*pTsma = NULL;
return TSDB_CODE_SUCCESS;
_return:
CTG_RET(code);
}
int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp) {
int32_t code = 0;
SCtgCacheOperation* op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
if (!op) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
op->opId = CTG_OP_DROP_TB_TSMA;
op->syncOp = syncOp;
SCtgDropTbTSMAMsg* msg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg));
if (!msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg));
taosMemoryFree(op);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
msg->dbId = pTsma->dbId;
msg->tbId = pTsma->suid;
msg->tsmaId = pTsma->tsmaId;
tstrncpy(msg->dbFName, pTsma->dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(msg->tbName, pTsma->tb, TSDB_TABLE_NAME_LEN);
tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN);
op->data = msg;
CTG_ERR_JRET(ctgEnqueue(pCtg, op));
return TSDB_CODE_SUCCESS;
_return:
CTG_RET(code);
}
static SCtgCacheOperation* createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const STSMACache* pCache, bool syncOp) {
SCtgCacheOperation* pOp = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
if (!pOp) return NULL;
SCtgDropTbTSMAMsg* pMsg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg));
if (!pMsg) {
taosMemoryFree(pOp);
return NULL;
}
pOp->opId = CTG_OP_DROP_TB_TSMA;
pOp->syncOp = syncOp;
pMsg->pCtg = pCtg;
pMsg->dbId = pCache->dbId;
pMsg->tbId = pCache->suid;
pMsg->tsmaId = pCache->tsmaId;
pMsg->dropAllForTb = true;
tstrncpy(pMsg->tsmaName, pCache->name, TSDB_TABLE_NAME_LEN);
tstrncpy(pMsg->dbFName, pCache->dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(pMsg->tbName, pCache->tb, TSDB_TABLE_NAME_LEN);
pOp->data = pMsg;
return pOp;
}
int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) {
ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname);
int32_t code = 0;
SCtgDBCache* pDbCache = NULL;
SCtgCacheOperation* pOp = NULL;
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pName, dbFName);
CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache));
if (NULL == pDbCache || !pDbCache->tsmaCache) {
goto _return;
}
SCtgTSMACache *pCtgCache = taosHashGet(pDbCache->tsmaCache, pName->tname, strlen(pName->tname));
if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) {
goto _return;
}
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
STSMACache *pCache = taosArrayGetP(pCtgCache->pTsmas, 0);
pOp = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp);
if (!pOp) {
code = TSDB_CODE_OUT_OF_MEMORY;
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
goto _return;
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
CTG_ERR_JRET(ctgEnqueue(pCtg, pOp));
return TSDB_CODE_SUCCESS;
_return:
if (pOp) {
taosMemoryFree(pOp->data);
taosMemoryFree(pOp);
}
CTG_RET(code);
}
int32_t ctgWriteTbTSMAToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName,
STSMACache **ppTsmaCache) {
if (NULL == dbCache->tsmaCache) {
ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
STSMACache *pTsmaCache = *ppTsmaCache;
int32_t code = TSDB_CODE_SUCCESS;
SCtgTSMACache* pCache = taosHashGet(dbCache->tsmaCache, tbName, strlen(tbName));
if (!pCache) {
SCtgTSMACache cache = {0};
cache.pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES));
if (!cache.pTsmas) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
taosArrayPush(cache.pTsmas, &pTsmaCache);
if (taosHashPut(dbCache->tsmaCache, tbName, strlen(tbName), &cache, sizeof(cache))) {
ctgError("taosHashPut new tsmacache for tb: %s.%s failed", dbFName, tbName);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(STSMACache) + ctgGetTbTSMACacheSize(pTsmaCache));
CTG_DB_NUM_INC(CTG_CI_TBL_TSMA);
ctgDebug("tb %s tsma updated to cache, name: %s", tbName, pTsmaCache->name);
CTG_ERR_JRET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache));
*ppTsmaCache = NULL;
goto _return;
}
CTG_LOCK(CTG_WRITE, &pCache->tsmaLock);
if (pCache->pTsmas) {
uint64_t cacheSize = 0;
for (int32_t i = 0; i < pCache->pTsmas->size; ++i) {
STableTSMAInfo* pInfo = taosArrayGetP(pCache->pTsmas, i);
if (pInfo->tsmaId == pTsmaCache->tsmaId) {
ctgDebug("tsma: %s removed from cache, history from %d to %d, reqTs from %" PRId64 " to %" PRId64
"rspTs from %" PRId64 " to %" PRId64 " delay from %" PRId64 " to %" PRId64,
pInfo->name, pInfo->fillHistoryFinished, pTsmaCache->fillHistoryFinished, pInfo->reqTs,
pTsmaCache->reqTs, pInfo->rspTs, pTsmaCache->rspTs, pInfo->delayDuration, pTsmaCache->delayDuration);
cacheSize = ctgGetTbTSMACacheSize(pInfo);
taosArrayRemove(pCache->pTsmas, i);
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
tFreeTableTSMAInfo(pInfo);
taosMemoryFreeClear(pInfo);
break;
}
}
} else {
pCache->pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES));
if (!pCache->pTsmas) {
CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
// push the new cache
taosArrayPush(pCache->pTsmas, &pTsmaCache);
*ppTsmaCache = NULL;
atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbTSMACacheSize(pTsmaCache));
CTG_ERR_RET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache));
CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock);
ctgDebug("table %s tsma updated to cache, tsma: %s", tbName, pTsmaCache->name);
_return:
CTG_RET(code);
}
int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgDropTbTSMAMsg * msg = operation->data;
SCatalog *pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
if (pCtg->stopUpdate) {
goto _return;
}
CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
if (NULL == dbCache || !dbCache->tsmaCache || (msg->dbId != dbCache->dbId && msg->dbId != 0)) {
goto _return;
}
SCtgTSMACache* pCtgCache = taosHashGet(dbCache->tsmaCache, msg->tbName, strlen(msg->tbName));
if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) {
goto _return;
}
uint64_t cacheSize = 0;
STSMACache *pCache = NULL;
if (msg->dropAllForTb) {
CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock);
for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) {
pCache = taosArrayGetP(pCtgCache->pTsmas, i);
cacheSize += ctgGetTbTSMACacheSize(pCache);
ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare);
CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA);
}
taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo);
pCtgCache->pTsmas = NULL;
ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName);
taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN);
CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock);
} else {
CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock);
pCache = taosArrayGetP(pCtgCache->pTsmas, 0);
if (msg->tbId != 0 && pCache->suid != msg->tbId) {
// table id mismatch, skip drops
CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock);
goto _return;
}
for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) {
pCache = taosArrayGetP(pCtgCache->pTsmas, i);
if (pCache->tsmaId != msg->tsmaId) {
continue;
}
cacheSize = ctgGetTbTSMACacheSize(pCache);
ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare);
taosArrayRemove(pCtgCache->pTsmas, i);
tFreeAndClearTableTSMAInfo(pCache);
CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA);
break;
}
CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock);
}
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgUpdateTbTSMAMsg *msg = operation->data;
SCatalog * pCtg = msg->pCtg;
STableTSMAInfo * pTsmaInfo = msg->pTsma;
SCtgDBCache * dbCache = NULL;
if (pCtg->stopUpdate) {
goto _return;
}
CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pTsmaInfo->dbFName, pTsmaInfo->dbId, &dbCache));
CTG_ERR_JRET(ctgWriteTbTSMAToCache(pCtg, dbCache, pTsmaInfo->dbFName, pTsmaInfo->tb, &pTsmaInfo));
if (dbCache && msg->dbTsmaVersion > 0) {
dbCache->tsmaVersion = msg->dbTsmaVersion;
SDbCacheInfo cacheInfo = {0};
cacheInfo.dbId = dbCache->dbId;
if (dbCache->cfgCache.cfgInfo) {
cacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion;
tstrncpy(cacheInfo.dbFName, dbCache->cfgCache.cfgInfo->db, TSDB_DB_FNAME_LEN);
}
if (dbCache->vgCache.vgInfo) {
cacheInfo.vgVersion = dbCache->vgCache.vgInfo->vgVersion;
cacheInfo.numOfTable = dbCache->vgCache.vgInfo->numOfTable;
cacheInfo.stateTs = dbCache->vgCache.vgInfo->stateTs;
}
cacheInfo.tsmaVersion = dbCache->tsmaVersion;
CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo),
ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare));
}
_return:
if (pTsmaInfo) {
tFreeTableTSMAInfo(pTsmaInfo);
taosMemoryFreeClear(pTsmaInfo);
}
taosMemoryFreeClear(msg);
CTG_RET(code);
}

View File

@ -337,6 +337,35 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize,
qDebug("Got view-meta from mnode, viewFName:%s", target);
break;
}
case TDMT_MND_GET_TSMA:
case TDMT_MND_GET_TABLE_TSMA: {
if (TSDB_CODE_SUCCESS != rspCode) {
if (TSDB_CODE_MND_SMA_NOT_EXIST != rspCode) {
qError("error rsp for get table tsma, error:%s, tbFName:%s", tstrerror(rspCode), target);
}
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get table tsma rsp failed, error:%s, tbFName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got table tsma from mnode, tbFName:%s", target);
break;
}
case TDMT_VND_GET_STREAM_PROGRESS: {
if (TSDB_CODE_SUCCESS != rspCode) {
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get stream progress rsp failed, err: %s, tbFName: %s", tstrerror(code), target);
CTG_ERR_RET(code);
}
break;
}
default:
if (TSDB_CODE_SUCCESS != rspCode) {
qError("Got error rsp, error:%s", tstrerror(rspCode));
@ -528,10 +557,20 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
pName = ctgGetFetchName(ctx->pNames, fetch);
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
pName = ctx->pName;
}
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -578,10 +617,20 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
pName = ctgGetFetchName(ctx->pNames, fetch);
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
pName = ctx->pName;
}
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -1104,7 +1153,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName,
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, const char* tbName,
STableMetaOutput* out, SCtgTaskReq* tReq) {
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName};
@ -1463,4 +1512,120 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL;
int32_t msgLen = 0;
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(name, tbFName);
ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp);
if (code) {
ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, (char*)tbFName));
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen));
#else
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pTaskId, &pTask->taskId);
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
rpcFreeCont(rpcRsp.pCont);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTbName,
SVgroupInfo* vgroupInfo, SStreamProgressRsp* out, SCtgTaskReq* tReq,
void* bInput) {
char* msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS;
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTbName, tbFName);
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse];
ctgDebug("try to get stream progress from vnode, vgId:%d, ep num:%d, ep %s:%d, target:%s", vgroupInfo->vgId,
vgroupInfo->epSet.numOfEps, pEp->fqdn, pEp->port, tbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](bInput, &msg, 0, &msgLen, mallocFp);
if (code) {
ctgError("Build get stream progress failed, code:%s, tbFName:%s", tstrerror(code), tbFName);
CTG_ERR_RET(code);
}
if (pTask) {
SStreamProgressRsp* pOut = taosMemoryCalloc(1, sizeof(SStreamProgressRsp));
if (!pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, (char*)tbFName));
SRequestConnInfo vConn = {.pTrans = pConn->pTrans,
.requestId = pConn->requestId,
.requestObjRefId = pConn->requestObjRefId,
.mgmtEps = vgroupInfo->epSet};
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen));
#else
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTbName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pTaskId, &pTask->taskId);
CTG_RET(
ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
#endif
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
rpcFreeCont(rpcRsp.pCont);
return TSDB_CODE_SUCCESS;
}

View File

@ -259,6 +259,24 @@ void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache) {
}
}
void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) {
if (!dbCache->tsmaCache) return;
void* pIter = taosHashIterate(dbCache->tsmaCache, NULL);
while (pIter) {
SCtgTSMACache* pCtgCache = pIter;
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
int32_t size = (pCtgCache && pCtgCache->pTsmas) ? pCtgCache->pTsmas->size : 0;
for (int32_t i = 0; i < size; ++i) {
STSMACache* pCache = taosArrayGetP(pCtgCache->pTsmas, i);
if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare)) {
ctgDebug("tsma removed from rent, viewId: %" PRIx64 " name: %s.%s.%s", pCache->tsmaId, pCache->dbFName, pCache->tb, pCache->name);
}
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
pIter = taosHashIterate(dbCache->tsmaCache, pIter);
}
}
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache) {
@ -300,3 +318,17 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName,
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateRentTSMAVersion(SCatalog *pCtg, char *dbFName, const STSMACache *pCache) {
const STableTSMAInfo *pTsmaInfo = pCache;
STSMAVersion tsmaRent = {.dbId = pTsmaInfo->dbId, .tsmaId = pTsmaInfo->tsmaId, .version = pTsmaInfo->version};
tstrncpy(tsmaRent.name, pTsmaInfo->name, TSDB_TABLE_NAME_LEN);
tstrncpy(tsmaRent.dbFName, dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(tsmaRent.tbName, pTsmaInfo->tb, TSDB_TABLE_NAME_LEN);
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->tsmaRent, &tsmaRent, tsmaRent.tsmaId, sizeof(STSMAVersion),
ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare));
ctgDebug("db %s, 0x%" PRIx64 " tsma %s, 0x%" PRIx64 "version %d updated to tsmaRent", dbFName, tsmaRent.dbId,
pTsmaInfo->name, pTsmaInfo->tsmaId, pTsmaInfo->version);
return TSDB_CODE_SUCCESS;
}

View File

@ -190,6 +190,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pView);
pData->pView = NULL;
taosArrayDestroy(pData->pTableTsmas);
pData->pTableTsmas = NULL;
taosArrayDestroy(pData->pTsmas);
pData->pTsmas = NULL;
taosMemoryFreeClear(pData->pSvrVer);
}
@ -297,6 +303,31 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) {
dbCache->tbCache = NULL;
}
void ctgFreeTSMACacheImpl(SCtgTSMACache* pCache, bool lock) {
if (lock) {
CTG_LOCK(CTG_WRITE, &pCache->tsmaLock);
}
if (pCache->pTsmas) {
taosArrayDestroyP(pCache->pTsmas, tFreeAndClearTableTSMAInfo);
pCache->pTsmas = NULL;
}
if (lock) {
CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock);
}
}
void ctgFreeTSMACache(SCtgDBCache* dbCache) {
if (!dbCache) return;
SCtgTSMACache* pCache = taosHashIterate(dbCache->tsmaCache, NULL);
while (pCache) {
ctgFreeTSMACacheImpl(pCache, false);
pCache = taosHashIterate(dbCache->tsmaCache, pCache);
}
taosHashCleanup(dbCache->tsmaCache);
dbCache->tsmaCache = NULL;
}
void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); }
void ctgFreeCfgInfoCache(SCtgDBCache* dbCache) { freeDbCfgInfo(dbCache->cfgCache.cfgInfo); }
@ -310,6 +341,7 @@ void ctgFreeDbCache(SCtgDBCache* dbCache) {
ctgFreeStbMetaCache(dbCache);
ctgFreeTbCache(dbCache);
ctgFreeViewCache(dbCache);
ctgFreeTSMACache(dbCache);
}
void ctgFreeInstDbCache(SHashObj* pDbCache) {
@ -353,6 +385,7 @@ void ctgFreeHandleImpl(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeMetaRent(&pCtg->tsmaRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
@ -382,6 +415,7 @@ void ctgFreeHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeMetaRent(&pCtg->tsmaRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
@ -410,7 +444,8 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN
}
taosHashRemove(dbCache->tbCache, key, len);
cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex);
cacheSize =
len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex);
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
*pClearedSize += cacheSize;
(*pCleardNum)++;
@ -468,6 +503,7 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeMetaRent(&pCtg->tsmaRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
@ -475,6 +511,7 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo));
ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion));
ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion));
ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion));
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false,
HASH_ENTRY_LOCK);
@ -592,6 +629,20 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
}
break;
}
case TDMT_MND_GET_TSMA:
case TDMT_MND_GET_TABLE_TSMA: {
if (pCtx->out) {
tFreeTableTSMAInfoRsp(pCtx->out);
taosMemoryFreeClear(pCtx->out);
}
break;
}
case TDMT_VND_GET_STREAM_PROGRESS: {
if (pCtx->out) {
taosMemoryFreeClear(pCtx->out);
}
break;
}
default:
qError("invalid reqType %d", pCtx->reqType);
break;
@ -663,6 +714,20 @@ void ctgFreeViewMetaRes(void* res) {
}
}
void ctgFreeTbTSMARes(void* res) {
if (!res) {
return;
}
SMetaRes* pRes = res;
if (pRes->pRes) {
STableTSMAInfoRsp * pTsmaRsp = pRes->pRes;
tFreeTableTSMAInfoRsp(pTsmaRsp);
taosMemoryFree(pTsmaRsp);
pRes->pRes = NULL;
}
}
void ctgFreeJsonTagVal(void* val) {
if (NULL == val) {
return;
@ -760,6 +825,16 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
*pRes = NULL; // no need to free it
break;
}
case CTG_TASK_GET_TSMA:
case CTG_TASK_GET_TB_TSMA: {
SArray* pArr = (SArray*)*pRes;
int32_t num = taosArrayGetSize(pArr);
for (int32_t i = 0; i < num; ++i) {
ctgFreeTbTSMARes(taosArrayGet(pArr, i));
}
*pRes = NULL;
break;
}
default:
qError("invalid task type %d", type);
break;
@ -922,7 +997,16 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
taosMemoryFreeClear(pTask->taskCtx);
break;
}
}
case CTG_TASK_GET_TSMA:
case CTG_TASK_GET_TB_TSMA: {
SCtgTbTSMACtx* pTsmaCtx = pTask->taskCtx;
taosArrayDestroyEx(pTsmaCtx->pResList, ctgFreeTbTSMARes);
taosArrayDestroy(pTsmaCtx->pFetches);
taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx);
taosMemoryFreeClear(pTask->taskCtx);
break;
}
default:
qError("invalid task type %d", pTask->type);
break;
@ -1328,6 +1412,15 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) {
}
}
int32_t ctgTSMAVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t*)key1 < ((STSMAVersion*)key2)->tsmaId) {
return -1;
} else if (*(uint64_t*)key1 > ((STSMAVersion*)key2)->tsmaId) {
return 1;
} else {
return 0;
}
}
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) {
if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) {
@ -1359,6 +1452,15 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) {
}
}
int32_t ctgTSMAVersionSortCompare(const void* key1, const void* key2) {
if (((STSMAVersion*)key1)->tsmaId < ((STSMAVersion*)key2)->tsmaId) {
return -1;
} else if (((STSMAVersion*)key1)->tsmaId > ((STSMAVersion*)key2)->tsmaId) {
return 1;
} else {
return 0;
}
}
int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
if (NULL == dbInfo) {
@ -1657,6 +1759,10 @@ static void ctgFreeViewMeta(void* p) {
taosMemoryFree(pMeta);
}
void ctgFreeTbTSMAInfo(void* p) {
tFreeTableTSMAInfoRsp(((SMetaRes*)p)->pRes);
taosMemoryFree(((SMetaRes*)p)->pRes);
}
int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0;
@ -1990,6 +2096,8 @@ void ctgDestroySMetaData(SMetaData* pData) {
taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg);
taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList);
taosArrayDestroyEx(pData->pView, ctgFreeViewMeta);
taosArrayDestroyEx(pData->pTableTsmas, ctgFreeTbTSMAInfo);
taosArrayDestroyEx(pData->pTsmas, ctgFreeTbTSMAInfo);
taosMemoryFreeClear(pData->pSvrVer);
}
@ -2287,5 +2395,56 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) {
return TSDB_CODE_SUCCESS;
}
uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) {
if (!pTsmaInfo) return 0;
uint64_t size = sizeof(STableTSMAInfo);
if (pTsmaInfo->pFuncs) size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size;
if (pTsmaInfo->pTags) size += sizeof(SSchema) * pTsmaInfo->pTags->size;
if (pTsmaInfo->pUsedCols) size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size;
return size;
}
bool hasOutOfDateTSMACache(SArray* pTsmas) {
if (!pTsmas || pTsmas->size == 0) {
return false;
}
for (int32_t i = 0; i < pTsmas->size; ++i) {
STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i);
if (isCtgTSMACacheOutOfDate(pTsmaInfo)) return true;
}
return false;
}
bool isCtgTSMACacheOutOfDate(STSMACache* pTsmaCache) {
int64_t now = taosGetTimestampMs();
bool ret = !pTsmaCache->fillHistoryFinished ||
(tsMaxTsmaCalcDelay * 1000 - pTsmaCache->delayDuration) < (now - pTsmaCache->reqTs);
if (ret) {
qDebug("tsma %s.%s in cache has been out of date, history finished: %d, remain valid after: %" PRId64
" passed: %" PRId64,
pTsmaCache->dbFName, pTsmaCache->name, pTsmaCache->fillHistoryFinished,
tsMaxTsmaCalcDelay * 1000 - pTsmaCache->delayDuration, now - pTsmaCache->reqTs);
}
return ret;
}
int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag,
CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName) {
if (NULL == (*pFetchs)) {
*pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgTSMAFetch));
}
SCtgTSMAFetch fetch = {0};
fetch.dbIdx = dbIdx;
fetch.tbIdx = tbIdx;
fetch.fetchIdx = (*fetchIdx)++;
fetch.resIdx = resIdx;
fetch.flag = flag;
fetch.fetchType = fetchType;
if (sourceTbName) fetch.tsmaSourceTbName = *sourceTbName;
taosArrayPush(*pFetchs, &fetch);
return TSDB_CODE_SUCCESS;
}

View File

@ -53,9 +53,12 @@ extern "C" {
#define EXPLAIN_FILTER_FORMAT "Filter: "
#define EXPLAIN_MERGEBLOCKS_FORMAT "Merge ResBlocks: %s"
#define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: "
#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
#define EXPLAIN_PRIM_CONDITIONS_FORMAT "Join Prim Cond: "
#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Full Cond: "
#define EXPLAIN_COL_ON_CONDITIONS_FORMAT "Join Col Cond: "
#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
#define EXPLAIN_OUTPUT_FORMAT "Output: "
#define EXPLAIN_JOIN_PARAM_FORMAT "Join Param: "
#define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c"
#define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64
#define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f"
@ -71,8 +74,11 @@ extern "C" {
#define EXPLAIN_DYN_QRY_CTRL_FORMAT "Dynamic Query Control for %s"
#define EXPLAIN_COUNT_FORMAT "Count"
#define EXPLAIN_COUNT_INFO_FORMAT "Window Count Info"
#define EXPLAIN_JOIN_EQ_LEFT_FORMAT "Left Equal Cond: "
#define EXPLAIN_JOIN_EQ_RIGHT_FORMAT "Right Equal Cond: "
#define EXPLAIN_COUNT_NUM_FORMAT "Window Count=%" PRId64
#define EXPLAIN_COUNT_SLIDING_FORMAT "Window Sliding=%" PRId64
#define EXPLAIN_TABLE_TIMERANGE_FORMAT "%s Table Time Range: [%" PRId64 ", %" PRId64 "]"
#define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms"
#define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms"
@ -93,7 +99,6 @@ extern "C" {
#define EXPLAIN_SCAN_MODE_FORMAT "mode=%s"
#define EXPLAIN_SCAN_DATA_LOAD_FORMAT "data_load=%s"
#define EXPLAIN_GROUPS_FORMAT "groups=%d"
#define EXPLAIN_WIDTH_FORMAT "width=%d"
#define EXPLAIN_INTERVAL_VALUE_FORMAT "interval=%" PRId64 "%c"
#define EXPLAIN_FUNCTIONS_FORMAT "functions=%d"
#define EXPLAIN_EXECINFO_FORMAT "cost=%.3f..%.3f rows=%" PRIu64
@ -112,6 +117,12 @@ extern "C" {
#define EXPLAIN_SRC_SCAN_FORMAT "src_scan=%d,%d"
#define EXPLAIN_PLAN_BLOCKING "blocking=%d"
#define EXPLAIN_MERGE_MODE_FORMAT "mode=%s"
#define EXPLAIN_ASOF_OP_FORMAT "asof_op='%s'"
#define EXPLAIN_WIN_OFFSET_FORMAT "window_offset=(%s, %s)"
#define EXPLAIN_JLIMIT_FORMAT "jlimit=%" PRId64
#define EXPLAIN_SEQ_WIN_GRP_FORMAT "seq_win_grp=%d"
#define EXPLAIN_GRP_JOIN_FORMAT "group_join=%d"
#define EXPLAIN_JOIN_ALGO "algo=%s"
#define COMMAND_RESET_LOG "resetLog"
#define COMMAND_SCHEDULE_POLICY "schedulePolicy"

View File

@ -769,6 +769,7 @@ static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreate
}
SViewMeta* pMeta = pStmt->pViewMeta;
ASSERT(pMeta);
snprintf(varDataVal(buf2), SHOW_CREATE_VIEW_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE VIEW `%s`.`%s` AS %s", pStmt->dbName, pStmt->viewName, pMeta->querySql);
int32_t len = strlen(varDataVal(buf2));
varDataLen(buf2) = (len > 65535) ? 65535 : len;

View File

@ -22,6 +22,14 @@
#include "systable.h"
#include "functionMgt.h"
char *gJoinTypeStr[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = {
/* NONE OUTER SEMI ANTI ASOF WINDOW */
/*INNER*/ {"Inner Join", NULL, NULL, NULL, NULL, NULL},
/*LEFT*/ {"Left Join", "Left Join", "Left Semi Join", "Left Anti Join", "Left ASOF Join", "Left Window Join"},
/*RIGHT*/ {"Right Join", "Right Join", "Right Semi Join", "Right Anti Join", "Right ASOF Join", "Right Window Join"},
/*FULL*/ {"Full Join", "Full Join", NULL, NULL, NULL, NULL},
};
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel);
@ -36,6 +44,33 @@ char *qExplainGetDynQryCtrlType(EDynQueryType type) {
return "unknown task";
}
char* qExplainGetAsofOpStr(int32_t opType) {
switch (opType) {
case OP_TYPE_GREATER_THAN:
return ">";
case OP_TYPE_GREATER_EQUAL:
return ">=";
case OP_TYPE_LOWER_THAN:
return "<";
case OP_TYPE_LOWER_EQUAL:
return "<=";
case OP_TYPE_EQUAL:
return "=";
default:
return "UNKNOWN";
}
}
char* qExplainGetTimerangeTargetStr(int32_t target) {
static char* targetName[] = {"", "Left", "Right", "Left/Right"};
if (target <= 0 || target > 3) {
return "Unknown";
}
return targetName[target];
}
void qExplainFreeResNode(SExplainResNode *resNode) {
if (NULL == resNode) {
return;
@ -375,6 +410,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
if (pTagScanNode->scan.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pTagScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
@ -593,7 +636,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode *pJoinNode = (SSortMergeJoinPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType));
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, gJoinTypeStr[pJoinNode->joinType][pJoinNode->subType]);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@ -605,6 +648,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pJoinNode->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_JOIN_ALGO, "Merge");
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -620,6 +664,36 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (IS_ASOF_JOIN(pJoinNode->subType) || IS_WINDOW_JOIN(pJoinNode->subType)) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_JOIN_PARAM_FORMAT);
if (IS_ASOF_JOIN(pJoinNode->subType)) {
EXPLAIN_ROW_APPEND(EXPLAIN_ASOF_OP_FORMAT, qExplainGetAsofOpStr(pJoinNode->asofOpType));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (NULL != pJoinNode->pWindowOffset) {
SWindowOffsetNode* pWinOffset = (SWindowOffsetNode*)pJoinNode->pWindowOffset;
SValueNode* pStart = (SValueNode*)pWinOffset->pStartOffset;
SValueNode* pEnd = (SValueNode*)pWinOffset->pEndOffset;
EXPLAIN_ROW_APPEND(EXPLAIN_WIN_OFFSET_FORMAT, pStart->literal, pEnd->literal);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (NULL != pJoinNode->pJLimit) {
SLimitNode* pJLimit = (SLimitNode*)pJoinNode->pJLimit;
EXPLAIN_ROW_APPEND(EXPLAIN_JLIMIT_FORMAT, pJLimit->limit);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (IS_WINDOW_JOIN(pJoinNode->subType)) {
EXPLAIN_ROW_APPEND(EXPLAIN_SEQ_WIN_GRP_FORMAT, pJoinNode->seqWinGroup);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
@ -628,16 +702,48 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
if (pJoinNode->pOtherOnCond) {
EXPLAIN_ROW_APPEND(" AND ");
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pOtherOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
if (NULL != pJoinNode->pPrimKeyCond) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_JOIN_EQ_LEFT_FORMAT);
SNode* pCol = NULL;
FOREACH(pCol, pJoinNode->pEqLeft) {
EXPLAIN_ROW_APPEND("`%s`.`%s` ", ((SColumnNode*)pCol)->tableAlias, ((SColumnNode*)pCol)->colName);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pEqRight && pJoinNode->pEqRight->length > 0) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_JOIN_EQ_RIGHT_FORMAT);
SNode* pCol = NULL;
FOREACH(pCol, pJoinNode->pEqRight) {
EXPLAIN_ROW_APPEND("`%s`.`%s` ", ((SColumnNode*)pCol)->tableAlias, ((SColumnNode*)pCol)->colName);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pFullOnCond) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pColOnCond) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_COL_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
break;
}
@ -1575,7 +1681,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:{
SHashJoinPhysiNode *pJoinNode = (SHashJoinPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType));
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, gJoinTypeStr[pJoinNode->joinType][pJoinNode->subType]);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@ -1587,6 +1693,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pJoinNode->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_JOIN_ALGO, "Hash");
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -1602,19 +1709,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->node.pConditions || pJoinNode->pFilterConditions) {
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
if (pJoinNode->node.pConditions) {
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
if (pJoinNode->pFilterConditions) {
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFilterConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
@ -1642,8 +1741,21 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true;
}
if (pJoinNode->pFullOnCond) {
if (conditionsGot) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->timeRangeTarget) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}

View File

@ -836,6 +836,7 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* de
extern void doDestroyExchangeOperatorInfo(void* param);
int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, SColumnInfoData** pResCol);
int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache);
@ -974,6 +975,9 @@ void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArr
int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition, int32_t order);
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status);
#ifdef __cplusplus
}

View File

@ -65,6 +65,7 @@ typedef struct SGcDownstreamCtx {
SHashObj* pSessions;
SHashObj* pWaitSessions;
SGcFileCacheCtx fileCtx;
bool fetchDone;
} SGcDownstreamCtx;
typedef struct SGcVgroupCtx {

View File

@ -20,6 +20,13 @@ extern "C" {
#endif
#define HASH_JOIN_DEFAULT_PAGE_SIZE 10485760
#define HJOIN_DEFAULT_BLK_ROWS_NUM 4096
#define HJOIN_BLK_SIZE_LIMIT 10485760
#define HJOIN_ROW_BITMAP_SIZE (2 * 1048576)
#define HJOIN_BLK_THRESHOLD_RATIO 0.9
typedef int32_t (*hJoinImplFp)(SOperatorInfo*);
#pragma pack(push, 1)
typedef struct SBufRowInfo {
@ -29,27 +36,36 @@ typedef struct SBufRowInfo {
} SBufRowInfo;
#pragma pack(pop)
typedef enum EHJoinPhase {
E_JOIN_PHASE_PRE = 1,
E_JOIN_PHASE_CUR,
E_JOIN_PHASE_POST
} EHJoinPhase;
typedef struct SHJoinCtx {
bool rowRemains;
bool midRemains;
int64_t limit;
SBufRowInfo* pBuildRow;
SSDataBlock* pProbeData;
int32_t probeIdx;
EHJoinPhase probePhase;
int32_t probePreIdx;
int32_t probeStartIdx;
int32_t probeEndIdx;
int32_t probePostIdx;
bool readMatch;
} SHJoinCtx;
typedef struct SRowLocation {
SSDataBlock* pDataBlock;
int32_t pos;
} SRowLocation;
typedef struct SHJoinColInfo {
int32_t srcSlot;
int32_t dstSlot;
bool keyCol;
bool vardata;
int32_t* offset;
int32_t bytes;
char* data;
char* bitMap;
int32_t srcSlot;
int32_t dstSlot;
bool keyCol;
bool vardata;
int32_t* offset;
int32_t bytes;
char* data;
char* bitMap;
SColumnInfoData* colData;
} SHJoinColInfo;
typedef struct SBufPageInfo {
@ -63,11 +79,31 @@ typedef struct SGroupData {
SBufRowInfo* rows;
} SGroupData;
typedef struct SHJoinTableInfo {
typedef struct SHJoinColMap {
int32_t srcSlot;
int32_t dstSlot;
bool vardata;
int32_t bytes;
} SHJoinColMap;
// for now timetruncate only
typedef struct SHJoinPrimExprCtx {
int64_t truncateUnit;
int64_t timezoneUnit;
int32_t targetSlotId;
} SHJoinPrimExprCtx;
typedef struct SHJoinTableCtx {
int32_t downStreamIdx;
SOperatorInfo* downStream;
int32_t blkId;
SQueryStat inputStat;
bool hasTimeRange;
SHJoinColMap* primCol;
SNode* primExpr;
SHJoinPrimExprCtx primCtx;
int32_t keyNum;
SHJoinColInfo* keyCols;
@ -81,7 +117,7 @@ typedef struct SHJoinTableInfo {
int32_t valBufSize;
SArray* valVarCols;
bool valColExist;
} SHJoinTableInfo;
} SHJoinTableCtx;
typedef struct SHJoinExecInfo {
int64_t buildBlkNum;
@ -94,21 +130,57 @@ typedef struct SHJoinExecInfo {
typedef struct SHJoinOperatorInfo {
int32_t joinType;
SHJoinTableInfo tbs[2];
SHJoinTableInfo* pBuild;
SHJoinTableInfo* pProbe;
SSDataBlock* pRes;
EJoinType joinType;
EJoinSubType subType;
SHJoinTableCtx tbs[2];
SHJoinTableCtx* pBuild;
SHJoinTableCtx* pProbe;
SFilterInfo* pPreFilter;
SFilterInfo* pFinFilter;
SSDataBlock* finBlk;
SSDataBlock* midBlk;
STimeWindow tblTimeRange;
int32_t pResColNum;
int8_t* pResColMap;
SArray* pRowBufs;
SNode* pCond;
SSHashObj* pKeyHash;
bool keyHashBuilt;
SHJoinCtx ctx;
SHJoinExecInfo execInfo;
int32_t blkThreshold;
hJoinImplFp joinFp;
} SHJoinOperatorInfo;
#define HJ_ERR_RET(c) \
do { \
int32_t _code = (c); \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define HJ_ERR_JRET(c) \
do { \
code = (c); \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
int32_t hInnerJoinDo(struct SOperatorInfo* pOperator);
int32_t hLeftJoinDo(struct SOperatorInfo* pOperator);
void hJoinSetDone(struct SOperatorInfo* pOperator);
void hJoinAppendResToBlock(struct SOperatorInfo* pOperator, SSDataBlock* pRes, bool* allFetched);
bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *pBufLen);
int32_t hJoinCopyMergeMidBlk(SHJoinCtx* pCtx, SSDataBlock** ppMid, SSDataBlock** ppFin);
int32_t hJoinHandleMidRemains(SHJoinOperatorInfo* pJoin, SHJoinCtx* pCtx);
bool hJoinBlkReachThreshold(SHJoinOperatorInfo* pInfo, int64_t blkRows);
int32_t hJoinCopyNMatchRowsToBlock(SHJoinOperatorInfo* pJoin, SSDataBlock* pRes, int32_t startIdx, int32_t rows);
#ifdef __cplusplus
}
#endif

View File

@ -0,0 +1,482 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MERGEJOIN_H
#define TDENGINE_MERGEJOIN_H
#ifdef __cplusplus
extern "C" {
#endif
#if 0
#define MJOIN_DEFAULT_BLK_ROWS_NUM 3 //4096
#define MJOIN_HJOIN_CART_THRESHOLD 10
#define MJOIN_BLK_SIZE_LIMIT 0 //10485760
#define MJOIN_ROW_BITMAP_SIZE (2 * 1048576)
#else
#define MJOIN_DEFAULT_BLK_ROWS_NUM 4096
#define MJOIN_HJOIN_CART_THRESHOLD 16
#define MJOIN_BLK_SIZE_LIMIT 10485760
#define MJOIN_ROW_BITMAP_SIZE (2 * 1048576)
#endif
#define MJOIN_SEMI_ANTI_BLK_ROWS_NUM 64
#define MJOIN_BLK_THRESHOLD_RATIO 0.9
struct SMJoinOperatorInfo;
typedef enum EJoinTableType {
E_JOIN_TB_BUILD = 1,
E_JOIN_TB_PROBE
} EJoinTableType;
#define MJOIN_TBTYPE(_type) (E_JOIN_TB_BUILD == (_type) ? "BUILD" : "PROBE")
#define IS_FULL_OUTER_JOIN(_jtype, _stype) ((_jtype) == JOIN_TYPE_FULL && (_stype) == JOIN_STYPE_OUTER)
typedef struct SMJoinRowPos {
SSDataBlock* pBlk;
int32_t pos;
} SMJoinRowPos;
typedef struct SMJoinColMap {
int32_t srcSlot;
int32_t dstSlot;
bool vardata;
int32_t bytes;
} SMJoinColMap;
typedef struct SMJoinColInfo {
int32_t srcSlot;
int32_t dstSlot;
bool jsonData;
bool vardata;
int32_t* offset;
int32_t bytes;
char* data;
char* bitMap;
SColumnInfoData* colData;
} SMJoinColInfo;
typedef struct SMJoinHashGrpRows {
int32_t rowBitmapOffset;
int32_t rowMatchNum;
bool allRowsMatch;
bool allRowsNMatch;
SArray* pRows;
} SMJoinHashGrpRows;
typedef struct SMJoinNMatchCtx {
void* pGrp;
int32_t iter;
int32_t bitIdx;
int32_t rowIdx;
int32_t grpIdx;
} SMJoinNMatchCtx;
// for now timetruncate only
typedef struct SMJoinPrimExprCtx {
int64_t truncateUnit;
int64_t timezoneUnit;
int32_t targetSlotId;
} SMJoinPrimExprCtx;
typedef struct SMJoinTableCtx {
EJoinTableType type;
int32_t downStreamIdx;
SOperatorInfo* downStream;
bool dsInitDone;
bool dsFetchDone;
SNode* primExpr;
SMJoinPrimExprCtx primCtx;
int32_t blkId;
SQueryStat inputStat;
uint64_t lastInGid;
SSDataBlock* remainInBlk;
SMJoinColMap* primCol;
char* primData;
int32_t finNum;
SMJoinColMap* finCols;
int32_t keyNum;
int32_t keyNullSize;
SMJoinColInfo* keyCols;
char* keyBuf;
char* keyData;
bool newBlk;
SSDataBlock* blk;
int32_t blkRowIdx;
// merge join
int64_t grpTotalRows;
int32_t grpIdx;
bool noKeepEqGrpRows;
bool multiEqGrpRows;
int64_t eqRowLimit;
int64_t eqRowNum;
SArray* eqGrps;
SArray* createdBlks;
// hash join
int32_t grpArrayIdx;
SArray* pGrpArrays;
bool multiRowsGrp;
int32_t grpRowIdx;
SArray* pHashCurGrp;
SMJoinHashGrpRows* pHashGrpRows;
SSHashObj* pGrpHash;
int64_t rowBitmapSize;
int64_t rowBitmapOffset;
char* pRowBitmap;
SMJoinNMatchCtx nMatchCtx;
} SMJoinTableCtx;
typedef struct SMJoinMatchInfo {
int32_t rowBitmapOffset;
int32_t rowMatchNum;
bool allRowsNMatch;
bool allRowsMatch;
} SMJoinMatchInfo;
typedef struct SMJoinGrpRows {
SSDataBlock* blk;
int32_t beginIdx;
int32_t endIdx;
int32_t readIdx;
int32_t rowBitmapOffset;
int32_t rowMatchNum;
bool allRowsNMatch;
bool allRowsMatch;
bool readMatch;
bool clonedBlk;
} SMJoinGrpRows;
#define MJOIN_COMMON_CTX \
struct SMJoinOperatorInfo* pJoin; \
bool ascTs; \
bool grpRemains; \
SSDataBlock* finBlk; \
bool lastEqGrp; \
bool lastProbeGrp; \
bool seqWinGrp; \
bool groupJoin; \
int32_t blkThreshold; \
int64_t limit; \
int64_t jLimit
typedef struct SMJoinCommonCtx {
MJOIN_COMMON_CTX;
} SMJoinCommonCtx;
typedef int32_t (*joinCartFp)(void*);
typedef struct SMJoinMergeCtx {
// KEEP IT FIRST
struct SMJoinOperatorInfo* pJoin;
bool ascTs;
bool grpRemains;
SSDataBlock* finBlk;
bool lastEqGrp;
bool lastProbeGrp;
bool seqWinGrp;
bool groupJoin;
int32_t blkThreshold;
int64_t limit;
int64_t jLimit;
// KEEP IT FIRST
bool hashCan;
bool midRemains;
bool nmatchRemains;
SSDataBlock* midBlk;
int64_t lastEqTs;
SMJoinGrpRows probeNEqGrp;
SMJoinGrpRows buildNEqGrp;
bool hashJoin;
joinCartFp hashCartFp;
joinCartFp mergeCartFp;
} SMJoinMergeCtx;
typedef enum {
E_CACHE_NONE = 0,
E_CACHE_OUTBLK,
E_CACHE_INBLK
} SMJoinCacheMode;
typedef struct SMJoinWinCache {
int32_t pageLimit;
int32_t outRowIdx;
int32_t colNum;
int32_t rowNum;
int8_t grpIdx;
SArray* grps;
SArray* grpsQueue;
SSDataBlock* outBlk;
} SMJoinWinCache;
typedef int32_t (*joinMoveWin)(void*);
typedef struct SMJoinWindowCtx {
// KEEP IT FIRST
struct SMJoinOperatorInfo* pJoin;
bool ascTs;
bool grpRemains;
SSDataBlock* finBlk;
bool lastEqGrp;
bool lastProbeGrp;
bool seqWinGrp;
bool groupJoin;
int32_t blkThreshold;
int64_t limit;
int64_t jLimit;
// KEEP IT FIRST
int32_t asofOpType;
int64_t winBeginOffset;
int64_t winEndOffset;
bool lowerRowsAcq;
bool eqRowsAcq;
bool greaterRowsAcq;
bool forwardRowsAcq;
int64_t winBeginTs;
int64_t winEndTs;
joinMoveWin moveWinBeginFp;
joinMoveWin moveWinEndFp;
bool eqPostDone;
int64_t lastTs;
SMJoinGrpRows probeGrp;
SMJoinGrpRows buildGrp;
SMJoinWinCache cache;
} SMJoinWindowCtx;
typedef struct SMJoinFlowFlags {
bool mergeJoin;
bool windowJoin;
bool preFilter;
bool retrieveAfterBuildDone;
} SMJoinFlowFlags;
typedef struct SMJoinCtx {
SMJoinFlowFlags* pFlags;
bool mergeCtxInUse;
union {
SMJoinMergeCtx mergeCtx;
SMJoinWindowCtx windowCtx;
};
} SMJoinCtx;
typedef struct SMJoinExecInfo {
int64_t buildBlkNum;
int64_t buildBlkRows;
int64_t probeBlkNum;
int64_t probeBlkRows;
int64_t resRows;
int64_t expectRows;
} SMJoinExecInfo;
typedef SSDataBlock* (*joinImplFp)(SOperatorInfo*);
typedef SSDataBlock* (*joinRetrieveFp)(struct SMJoinOperatorInfo*, SMJoinTableCtx*);
typedef void (*joinResetFp)(struct SMJoinOperatorInfo*);
typedef struct SMJoinOperatorInfo {
SOperatorInfo* pOperator;
int32_t joinType;
int32_t subType;
int32_t inputTsOrder;
int32_t errCode;
int64_t outGrpId;
uint64_t outBlkId;
SMJoinTableCtx tbs[2];
SMJoinTableCtx* build;
SMJoinTableCtx* probe;
SFilterInfo* pFPreFilter;
SFilterInfo* pPreFilter;
SFilterInfo* pFinFilter;
joinImplFp joinFp;
joinRetrieveFp retrieveFp;
joinResetFp grpResetFp;
SMJoinCtx ctx;
SMJoinExecInfo execInfo;
} SMJoinOperatorInfo;
#define MJOIN_DS_REQ_INIT(_pOp) ((_pOp)->pOperatorGetParam && ((SSortMergeJoinOperatorParam*)(_pOp)->pOperatorGetParam->value)->initDownstream)
#define MJOIN_DS_NEED_INIT(_pOp, _tbctx) (MJOIN_DS_REQ_INIT(_pOp) && (!(_tbctx)->dsInitDone))
#define MJOIN_TB_LOW_BLK(_tbctx) ((_tbctx)->blkNum <= 0 || ((_tbctx)->blkNum == 1 && (_tbctx)->pHeadBlk->cloned))
#define REACH_HJOIN_THRESHOLD(_prb, _bld) ((_bld)->grpTotalRows >= MJOIN_HJOIN_CART_THRESHOLD)
#define SET_SAME_TS_GRP_HJOIN(_pair, _octx) ((_pair)->hashJoin = (_octx)->hashCan && REACH_HJOIN_THRESHOLD(_pair))
#define PROBE_TS_NMATCH(_asc, _pts, _bts) (((_asc) && (_pts) < (_bts)) || (!(_asc) && (_pts) > (_bts)))
#define PROBE_TS_NREACH(_asc, _pts, _bts) (((_asc) && (_pts) > (_bts)) || (!(_asc) && (_pts) < (_bts)))
#define MJOIN_BUILD_BLK_OOR(_asc, _pts, _pidx, _bts, _bnum) (((_asc) && (*((int64_t*)(_pts) + (_pidx)) > *((int64_t*)(_bts) + (_bnum) - 1))) || ((!(_asc)) && (*((int64_t*)(_pts) + (_pidx)) < *((int64_t*)(_bts) + (_bnum) - 1))))
#define GRP_REMAIN_ROWS(_grp) ((_grp)->endIdx - (_grp)->readIdx + 1)
#define GRP_DONE(_grp) ((_grp)->readIdx > (_grp)->endIdx)
#define MJOIN_PROBE_TB_ROWS_DONE(_tb) ((_tb)->blkRowIdx >= (_tb)->blk->info.rows)
#define FJOIN_PROBE_TB_ROWS_DONE(_tb) ((NULL == (_tb)->blk) || ((_tb)->blkRowIdx >= (_tb)->blk->info.rows))
#define MJOIN_BUILD_TB_ROWS_DONE(_tb) ((NULL == (_tb)->blk) || ((_tb)->blkRowIdx >= (_tb)->blk->info.rows))
#define MJOIN_TB_GRP_ROWS_DONE(_tb, _grpJoin) ((_tb)->dsFetchDone || ((_grpJoin) && NULL == (_tb)->blk && NULL != (_tb)->remainInBlk))
#define BLK_IS_FULL(_blk) ((_blk)->info.rows == (_blk)->info.capacity)
#define MJOIN_ROW_BITMAP_SET(_b, _base, _idx) (!colDataIsNull_f((_b + _base), _idx))
#define MJOIN_SET_ROW_BITMAP(_b, _base, _idx) colDataClearNull_f((_b + _base), _idx)
#define ASOF_EQ_ROW_INCLUDED(_op) (OP_TYPE_GREATER_EQUAL == (_op) || OP_TYPE_LOWER_EQUAL == (_op) || OP_TYPE_EQUAL == (_op))
#define ASOF_LOWER_ROW_INCLUDED(_op) (OP_TYPE_GREATER_EQUAL == (_op) || OP_TYPE_GREATER_THAN == (_op))
#define ASOF_GREATER_ROW_INCLUDED(_op) (OP_TYPE_LOWER_EQUAL == (_op) || OP_TYPE_LOWER_THAN == (_op))
#define WIN_ONLY_EQ_ROW_INCLUDED(_soff, _eoff) (0 == ((SValueNode*)(_soff))->datum.i && 0 == ((SValueNode*)(_eoff))->datum.i)
#define MJOIN_PUSH_BLK_TO_CACHE(_cache, _blk) \
do { \
ASSERT(taosArrayGetSize((_cache)->grps) <= 1); \
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayReserve((_cache)->grps, 1); \
(_cache)->rowNum += (_blk)->info.rows; \
pGrp->blk = (_blk); \
pGrp->beginIdx = 0; \
pGrp->endIdx = (_blk)->info.rows - 1; \
} while (0)
#define MJOIN_RESTORE_TB_BLK(_cache, _tb) \
do { \
SMJoinGrpRows* pGrp = taosArrayGet((_cache)->grps, 0); \
if (NULL != pGrp) { \
(_tb)->blk = pGrp->blk; \
(_tb)->blkRowIdx = pGrp->beginIdx; \
} else { \
(_tb)->blk = NULL; \
(_tb)->blkRowIdx = 0; \
} \
} while (0)
#define MJOIN_SAVE_TB_BLK(_cache, _tb) \
do { \
SMJoinGrpRows* pGrp = taosArrayGet((_cache)->grps, 0); \
if (NULL != pGrp) { \
ASSERT(pGrp->blk == (_tb)->blk); \
pGrp->beginIdx = (_tb)->blkRowIdx; \
pGrp->readIdx = pGrp->beginIdx; \
} \
} while (0)
#define MJOIN_POP_TB_BLK(_cache) \
do { \
SMJoinGrpRows* pGrp = taosArrayGet((_cache)->grps, 0); \
if (NULL != pGrp) { \
if (pGrp->blk == (_cache)->outBlk) { \
blockDataCleanup(pGrp->blk); \
} \
taosArrayPopFrontBatch((_cache)->grps, 1); \
} \
} while (0)
#define MJOIN_GET_TB_COL_TS(_col, _ts, _tb) \
do { \
if (NULL != (_tb)->blk && (_tb)->blkRowIdx < (_tb)->blk->info.rows) { \
(_col) = taosArrayGet((_tb)->blk->pDataBlock, (_tb)->primCtx.targetSlotId); \
(_ts) = *((int64_t*)(_col)->pData + (_tb)->blkRowIdx); \
} else { \
(_ts) = INT64_MAX; \
} \
} while (0)
#define MJOIN_GET_TB_CUR_TS(_col, _ts, _tb) \
do { \
(_ts) = *((int64_t*)(_col)->pData + (_tb)->blkRowIdx); \
} while (0)
#define MJ_ERR_RET(c) \
do { \
int32_t _code = (c); \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define MJ_ERR_JRET(c) \
do { \
code = (c); \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin);
void mJoinDestroyWindowCtx(SMJoinOperatorInfo* pJoin);
int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJoinNode);
int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJoinNode);
void mWinJoinResetWindowCache(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache);
SSDataBlock* mInnerJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mLeftJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mFullJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mSemiJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mAntiJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator);
void mJoinResetGroupTableCtx(SMJoinTableCtx* pCtx);
void mJoinResetTableCtx(SMJoinTableCtx* pCtx);
void mLeftJoinGroupReset(SMJoinOperatorInfo* pJoin);
void mWinJoinGroupReset(SMJoinOperatorInfo* pJoin);
bool mJoinRetrieveBlk(SMJoinOperatorInfo* pJoin, int32_t* pIdx, SSDataBlock** ppBlk, SMJoinTableCtx* pTb);
void mJoinSetDone(SOperatorInfo* pOperator);
bool mJoinIsDone(SOperatorInfo* pOperator);
bool mJoinCopyKeyColsDataToBuf(SMJoinTableCtx* pTable, int32_t rowIdx, size_t *pBufLen);
int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart);
int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp);
int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable);
int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp);
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build);
int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond);
int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx);
int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp);
int32_t mJoinNonEqCart(SMJoinCommonCtx* pCtx, SMJoinGrpRows* pGrp, bool probeGrp, bool singleProbeRow);
int32_t mJoinCopyMergeMidBlk(SMJoinMergeCtx* pCtx, SSDataBlock** ppMid, SSDataBlock** ppFin);
int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SMJoinTableCtx* build, int32_t startGrpIdx, int32_t startRowIdx);
int32_t mJoinFilterAndMarkHashRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SMJoinTableCtx* build, int32_t startRowIdx);
int32_t mJoinGetRowBitmapOffset(SMJoinTableCtx* pTable, int32_t rowNum, int32_t *rowBitmapOffset);
int32_t mJoinProcessLowerGrp(SMJoinMergeCtx* pCtx, SMJoinTableCtx* pTb, SColumnInfoData* pCol, int64_t* probeTs, int64_t* buildTs);
int32_t mJoinProcessGreaterGrp(SMJoinMergeCtx* pCtx, SMJoinTableCtx* pTb, SColumnInfoData* pCol, int64_t* probeTs, int64_t* buildTs);
int32_t mJoinFilterAndKeepSingleRow(SSDataBlock* pBlock, SFilterInfo* pFilterInfo);
int32_t mJoinFilterAndNoKeepRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo);
int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_MERGEJOIN_H

View File

@ -47,6 +47,7 @@ typedef struct SAggOperatorInfo {
bool groupKeyOptimized;
bool hasValidBlock;
SSDataBlock* pNewGroupBlock;
bool hasCountFunc;
} SAggOperatorInfo;
static void destroyAggOperatorInfo(void* param);
@ -111,6 +112,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
pInfo->groupId = UINT64_MAX;
pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder;
pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder;
pInfo->hasCountFunc = pAggNode->hasCountLikeFunc;
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
!pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo);
@ -317,18 +319,8 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
}
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
bool hasCountFunc = false;
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) ||
(strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) {
hasCountFunc = true;
break;
}
}
if (!hasCountFunc) {
if (!pAggInfo->hasCountFunc) {
return TSDB_CODE_SUCCESS;
}

View File

@ -48,7 +48,7 @@ static void destroyStbJoinTableList(SStbJoinTableList* pListHead) {
}
static void destroyStbJoinDynCtrlInfo(SStbJoinDynCtrlInfo* pStbJoin) {
qError("dynQueryCtrl exec info, prevBlk:%" PRId64 ", prevRows:%" PRId64 ", postBlk:%" PRId64 ", postRows:%" PRId64 ", leftCacheNum:%" PRId64 ", rightCacheNum:%" PRId64,
qDebug("dynQueryCtrl exec info, prevBlk:%" PRId64 ", prevRows:%" PRId64 ", postBlk:%" PRId64 ", postRows:%" PRId64 ", leftCacheNum:%" PRId64 ", rightCacheNum:%" PRId64,
pStbJoin->execInfo.prevBlkNum, pStbJoin->execInfo.prevBlkRows, pStbJoin->execInfo.postBlkNum,
pStbJoin->execInfo.postBlkRows, pStbJoin->execInfo.leftCacheNum, pStbJoin->execInfo.rightCacheNum);
@ -432,7 +432,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
int64_t* rightUid = pPrev->pListHead->pRightUid + rowIdx;
int32_t code = TSDB_CODE_SUCCESS;
qError("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid);
updatePostJoinCurrTableInfo(&pInfo->stbJoin);
@ -457,7 +457,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
}
}
bool initParam = pSrcParam0 ? true : false;
bool initParam = pSrcParam0 ? true : false;
if (TSDB_CODE_SUCCESS == code) {
code = buildGroupCacheOperatorParam(&pGcParam0, 0, *leftVg, *leftUid, pPost->leftNeedCache, pSrcParam0);
pSrcParam0 = NULL;
@ -524,7 +524,7 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost
int32_t vgId = leftTable ? pPost->leftVgId : pPost->rightVgId;
int64_t uid = leftTable ? pPost->leftCurrUid : pPost->rightCurrUid;
qError("notify table %" PRIu64 " in vgId %d downstreamId %d cache end", uid, vgId, downstreamId);
qDebug("notify table %" PRIu64 " in vgId %d downstreamId %d cache end", uid, vgId, downstreamId);
int32_t code = buildGroupCacheNotifyOperatorParam(&pGcParam, downstreamId, vgId, uid);
if (TSDB_CODE_SUCCESS != code) {
@ -758,7 +758,7 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
}
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
qError("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache));
qDebug("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache));
// debug only
iter = 0;

View File

@ -77,7 +77,6 @@ static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock*
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status);
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,

View File

@ -348,7 +348,7 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr
taosWLockLatch(&pCache->dirtyLock);
pCache->blkCacheSize += pBufInfo->basic.bufSize;
qError("group cache total dirty block num:%d size:%" PRId64 , taosHashGetSize(pCache->pDirtyBlk), pCache->blkCacheSize);
qDebug("group cache total dirty block num:%d size:%" PRId64 , taosHashGetSize(pCache->pDirtyBlk), pCache->blkCacheSize);
if (NULL == pCache->pDirtyHead) {
pCache->pDirtyHead = pBufInfo;
@ -664,7 +664,7 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
}
if (pBlock) {
qError("%s blk retrieved from group %" PRIu64, GET_TASKID(pOperator->pTaskInfo), pBlock->info.id.groupId);
qDebug("%s blk retrieved from group %" PRIu64, GET_TASKID(pOperator->pTaskInfo), pBlock->info.id.groupId);
pGCache->execInfo.pDownstreamBlkNum[downstreamIdx]++;
if (NULL == pGCache->pDownstreams[downstreamIdx].pBaseBlock) {
@ -803,7 +803,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
}
initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache);
qError("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
if (pParam->pChildren) {
SGcNewGroupInfo newGroup;
@ -821,6 +821,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
taosArrayDestroy(pParam->pChildren);
pParam->pChildren = NULL;
pCtx->fetchDone = false;
}
return TSDB_CODE_SUCCESS;
@ -832,7 +833,7 @@ static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcB
*pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1;
taosWUnLockLatch(&pGroup->blkList.lock);
qError("block added to group cache, total block num:%" PRId64, *pIdx + 1);
qDebug("block added to group cache, total block num:%" PRId64, *pIdx + 1);
return TSDB_CODE_SUCCESS;
}
@ -870,7 +871,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
}
if (pGroup->needCache) {
qError("add block to group cache");
qDebug("add block to group cache");
SGcBlkBufInfo newBlkBuf;
code = addBlkToBufCache(pOperator, pBlock, pCtx, pGroup, &newBlkBuf);
@ -883,7 +884,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
return code;
}
} else {
qError("no need to add block to group cache");
qDebug("no need to add block to group cache");
pGroup->pBlock = pBlock;
}
@ -910,6 +911,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) {
handleGroupFetchDone(pGroup);
}
pCtx->fetchDone = true;
} else {
int32_t uidNum = 0;
SGcVgroupCtx* pVgCtx = NULL;
@ -979,33 +981,41 @@ static int32_t getBlkFromSessionCacheImpl(struct SOperatorInfo* pOperator, int64
SGroupCacheOperatorInfo* pGCache = pOperator->info;
*got = true;
if (pSession->pGroupData->needCache) {
SGcBlkList* pBlkList = &pSession->pGroupData->blkList;
taosRLockLatch(&pBlkList->lock);
int64_t blkNum = taosArrayGetSize(pBlkList->pList);
if (pSession->lastBlkId < 0) {
if (blkNum > 0) {
SGcBlkBufBasic* pBasic = taosArrayGet(pBlkList->pList, 0);
if (NULL != pSession->pGroupData) {
if (pSession->pGroupData->needCache) {
SGcBlkList* pBlkList = &pSession->pGroupData->blkList;
taosRLockLatch(&pBlkList->lock);
int64_t blkNum = taosArrayGetSize(pBlkList->pList);
if (pSession->lastBlkId < 0) {
if (blkNum > 0) {
SGcBlkBufBasic* pBasic = taosArrayGet(pBlkList->pList, 0);
taosRUnLockLatch(&pBlkList->lock);
code = retrieveBlkFromBufCache(pGCache, pSession->pGroupData, sessionId, pBasic, ppRes);
pSession->lastBlkId = 0;
return code;
}
} else if ((pSession->lastBlkId + 1) < blkNum) {
SGcBlkBufBasic* pBasic = taosArrayGet(pBlkList->pList, pSession->lastBlkId + 1);
taosRUnLockLatch(&pBlkList->lock);
code = retrieveBlkFromBufCache(pGCache, pSession->pGroupData, sessionId, pBasic, ppRes);
pSession->lastBlkId = 0;
pSession->lastBlkId++;
return code;
}
} else if ((pSession->lastBlkId + 1) < blkNum) {
SGcBlkBufBasic* pBasic = taosArrayGet(pBlkList->pList, pSession->lastBlkId + 1);
taosRUnLockLatch(&pBlkList->lock);
code = retrieveBlkFromBufCache(pGCache, pSession->pGroupData, sessionId, pBasic, ppRes);
pSession->lastBlkId++;
} else if (pSession->pGroupData->pBlock) {
*ppRes = pSession->pGroupData->pBlock;
pSession->pGroupData->pBlock = NULL;
return TSDB_CODE_SUCCESS;
}
if (atomic_load_8((int8_t*)&pSession->pGroupData->fetchDone)) {
*ppRes = NULL;
qDebug("sessionId: %" PRIu64 " fetch done", sessionId);
return code;
}
taosRUnLockLatch(&pBlkList->lock);
} else if (pSession->pGroupData->pBlock) {
*ppRes = pSession->pGroupData->pBlock;
pSession->pGroupData->pBlock = NULL;
}
if (atomic_load_8((int8_t*)&pSession->pGroupData->fetchDone)) {
} else {
*ppRes = NULL;
qDebug("sessionId: %" PRIu64 " fetch done since downstream fetch done", sessionId);
return code;
}
@ -1136,13 +1146,17 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP
SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash;
SGroupCacheData* pGroup = taosHashGet(pGrpHash, &pGcParam->tbUid, sizeof(pGcParam->tbUid));
if (NULL == pGroup) {
if (NULL == pGroup && (NULL != pParam->pChildren || !pCtx->fetchDone)) {
code = addNewGroupData(pOperator, pParam, &pGroup, pGCache->batchFetch ? GROUP_CACHE_DEFAULT_VGID : pGcParam->vgId, pGcParam->tbUid);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
if (NULL == pGroup) {
return TSDB_CODE_SUCCESS;
}
initGroupCacheSessionCtx(&ctx, pGcParam, pGroup);
code = taosHashPut(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId), &ctx, sizeof(ctx));
@ -1166,6 +1180,10 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL == pSession) {
qDebug("session %" PRId64 " in downstream %d total got 0 rows since downtream fetch done", pGcParam->sessionId, pCtx->id);
return TSDB_CODE_SUCCESS;
}
} else if (pSession->pGroupData->needCache) {
SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (ppBlock) {
@ -1176,11 +1194,11 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes);
if (NULL == *ppRes) {
qError("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
} else {
pSession->resRows += (*ppRes)->info.rows;
qError("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
}
return code;

View File

@ -0,0 +1,349 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "operator.h"
#include "os.h"
#include "querynodes.h"
#include "querytask.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "thash.h"
#include "tmsg.h"
#include "ttypes.h"
#include "hashjoin.h"
int32_t hInnerJoinDo(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinTableCtx* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
SSDataBlock* pRes = pJoin->finBlk;
size_t bufLen = 0;
int32_t code = 0;
bool allFetched = false;
if (pJoin->ctx.pBuildRow) {
hJoinAppendResToBlock(pOperator, pRes, &allFetched);
if (pRes->info.rows >= pRes->info.capacity) {
if (allFetched) {
++pCtx->probeStartIdx;
}
return code;
} else {
++pCtx->probeStartIdx;
}
}
for (; pCtx->probeStartIdx <= pCtx->probeEndIdx; ++pCtx->probeStartIdx) {
if (hJoinCopyKeyColsDataToBuf(pProbe, pCtx->probeStartIdx, &bufLen)) {
continue;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (pGroup) {
pCtx->pBuildRow = pGroup->rows;
hJoinAppendResToBlock(pOperator, pRes, &allFetched);
if (pRes->info.rows >= pRes->info.capacity) {
if (allFetched) {
++pCtx->probeStartIdx;
}
return code;
}
}
}
pCtx->rowRemains = false;
return code;
}
int32_t hLeftJoinHandleSeqRowRemains(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
bool allFetched = false;
SHJoinCtx* pCtx = &pJoin->ctx;
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
if (pCtx->midRemains) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
}
if (allFetched && !pCtx->readMatch) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
}
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
++pCtx->probeStartIdx;
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
SHJoinTableCtx* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
size_t bufLen = 0;
bool allFetched = false;
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
return TSDB_CODE_SUCCESS;
}
for (; pCtx->probeStartIdx <= pCtx->probeEndIdx; ++pCtx->probeStartIdx) {
if (hJoinCopyKeyColsDataToBuf(pProbe, pCtx->probeStartIdx, &bufLen)) {
continue;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (NULL == pGroup) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
++pCtx->probeStartIdx;
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
continue;
}
pCtx->readMatch = false;
pCtx->pBuildRow = pGroup->rows;
allFetched = false;
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
if (pCtx->midRemains) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
}
if (allFetched && !pCtx->readMatch) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
}
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
}
pCtx->probePhase = E_JOIN_PHASE_POST;
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinHandleRowRemains(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
bool allFetched = false;
SHJoinCtx* pCtx = &pJoin->ctx;
hJoinAppendResToBlock(pOperator, pJoin->finBlk, &allFetched);
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
} else {
++pCtx->probeStartIdx;
}
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinHandleProbeRows(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
SHJoinTableCtx* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
size_t bufLen = 0;
bool allFetched = false;
for (; pCtx->probeStartIdx <= pCtx->probeEndIdx; ++pCtx->probeStartIdx) {
if (hJoinCopyKeyColsDataToBuf(pProbe, pCtx->probeStartIdx, &bufLen)) {
continue;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (NULL == pGroup) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
++pCtx->probeStartIdx;
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
continue;
}
pCtx->pBuildRow = pGroup->rows;
hJoinAppendResToBlock(pOperator, pJoin->finBlk, &allFetched);
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
pCtx->probePhase = E_JOIN_PHASE_POST;
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinDo(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinCtx* pCtx = &pJoin->ctx;
while (pCtx->rowRemains) {
switch (pCtx->probePhase) {
case E_JOIN_PHASE_PRE: {
int32_t rows = pCtx->probeStartIdx - pCtx->probePreIdx;
int32_t rowsLeft = pJoin->finBlk->info.capacity - pJoin->finBlk->info.rows;
if (rows <= rowsLeft) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, 0, rows));
pCtx->probePhase = E_JOIN_PHASE_CUR;
} else {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, 0, rowsLeft));
pJoin->ctx.probePreIdx += rowsLeft;
return TSDB_CODE_SUCCESS;
}
break;
}
case E_JOIN_PHASE_CUR: {
bool loopCont = false;
if (NULL == pJoin->ctx.pBuildRow) {
HJ_ERR_RET(pJoin->pPreFilter ? hLeftJoinHandleSeqProbeRows(pOperator, pJoin, &loopCont) : hLeftJoinHandleProbeRows(pOperator, pJoin, &loopCont));
} else {
HJ_ERR_RET(pJoin->pPreFilter ? hLeftJoinHandleSeqRowRemains(pOperator, pJoin, &loopCont) : hLeftJoinHandleRowRemains(pOperator, pJoin, &loopCont));
}
if (!loopCont) {
return TSDB_CODE_SUCCESS;
}
break;
}
case E_JOIN_PHASE_POST: {
if (pCtx->probeEndIdx < (pCtx->pProbeData->info.rows - 1) && pCtx->probePostIdx <= (pCtx->pProbeData->info.rows - 1)) {
int32_t rowsLeft = pJoin->finBlk->info.capacity - pJoin->finBlk->info.rows;
int32_t rows = pCtx->pProbeData->info.rows - pCtx->probePostIdx;
if (rows <= rowsLeft) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pJoin->ctx.probePostIdx, rows));
pCtx->rowRemains = false;
} else {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pJoin->ctx.probePostIdx, rowsLeft));
pCtx->probePostIdx += rowsLeft;
return TSDB_CODE_SUCCESS;
}
} else {
pJoin->ctx.rowRemains = false;
}
break;
}
default:
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
}
return TSDB_CODE_SUCCESS;
}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -282,6 +282,7 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) {
idx = NON_SORT_NEXT_SRC(pNonSortMerge, pNonSortMerge->lastSourceIdx);
continue;
}
pNonSortMerge->lastSourceIdx = idx - 1;
break;
}

View File

@ -231,7 +231,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS
// set current group id
pLimitInfo->currentGroupId = groupId;
bool limitReached = applyLimitOffset(pLimitInfo, pBlock, pOperator->pTaskInfo);
if (pBlock->info.rows == 0) {
if (pBlock->info.rows == 0 && 0 != pLimitInfo->limit.limit) {
return PROJECT_RETRIEVE_CONTINUE;
} else {
if (limitReached && (pLimitInfo->slimit.limit >= 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
@ -352,7 +352,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
// continue merge data, ignore the group id
blockDataMerge(pFinalRes, pRes);
if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold) {
if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold && (pOperator->status != OP_EXEC_DONE)) {
continue;
}
}

View File

@ -928,7 +928,7 @@ static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) {
return TSDB_CODE_INVALID_PARA;
}
qError("vgId:%d add total %d dynamic tables to scan, tableSeq:%d, exist num:%" PRId64 ", operator status:%d",
qDebug("vgId:%d add total %d dynamic tables to scan, tableSeq:%d, exist num:%" PRId64 ", operator status:%d",
pTaskInfo->id.vgId, num, pParam->tableSeq, (int64_t)taosArrayGetSize(pListInfo->pTableList), pOperator->status);
if (pParam->tableSeq) {
@ -962,7 +962,7 @@ static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) {
}
tableIdx++;
qError("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
qDebug("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
}
return code;
@ -4056,7 +4056,7 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) {
taosMemoryFree(pSubTblsInfo);
pInfo->pSubTablesMergeInfo = NULL;
taosMemoryTrim(0);
//taosMemoryTrim(0);
}
return TSDB_CODE_SUCCESS;
}
@ -4074,6 +4074,8 @@ SSDataBlock* doTableMergeScanParaSubTables(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code);
}
int64_t st = taosGetTimestampUs();
size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
@ -4102,7 +4104,7 @@ SSDataBlock* doTableMergeScanParaSubTables(SOperatorInfo* pOperator) {
pBlock->info.id.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->bGroupProcessed = true;
return pBlock;
break;
} else {
// Data of this group are all dumped, let's try the next group
stopSubTablesTableMergeScan(pInfo);
@ -4118,6 +4120,8 @@ SSDataBlock* doTableMergeScanParaSubTables(SOperatorInfo* pOperator) {
}
}
pOperator->cost.totalCost += (taosGetTimestampUs() - st) / 1000.0;;
return pBlock;
}
@ -4480,6 +4484,8 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code);
}
int64_t st = taosGetTimestampUs();
size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
@ -4509,7 +4515,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
pBlock->info.id.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->bGroupProcessed = true;
return pBlock;
break;
} else {
if (pInfo->bNewFilesetEvent) {
stopDurationForGroupTableMergeScan(pOperator);
@ -4533,6 +4539,8 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
}
}
pOperator->cost.totalCost += (taosGetTimestampUs() - st) / 1000.0;;
return pBlock;
}

View File

@ -1408,6 +1408,11 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
if (isTsmaResSTb(mr.me.name)) {
pAPI->metaReaderFn.clearReader(&mr);
continue;
}
// number of columns
pColInfoData = taosArrayGet(p->pDataBlock, 3);
colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false);

View File

@ -1,35 +1,35 @@
MESSAGE(STATUS "build parser unit test")
IF(NOT TD_DARWIN)
# GoogleTest requires at least C++11
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
# IF(NOT TD_DARWIN)
# # GoogleTest requires at least C++11
# SET(CMAKE_CXX_STANDARD 11)
# AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
#
# ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
# TARGET_LINK_LIBRARIES(
# executorTest
# PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
# )
#
# TARGET_INCLUDE_DIRECTORIES(
# executorTest
# PUBLIC "${TD_SOURCE_DIR}/include/libs/executor/"
# PRIVATE "${TD_SOURCE_DIR}/source/libs/executor/inc"
# )
# ENDIF ()
ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
TARGET_LINK_LIBRARIES(
executorTest
PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
)
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
TARGET_INCLUDE_DIRECTORIES(
executorTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/executor/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/executor/inc"
)
ENDIF ()
ADD_EXECUTABLE(joinTests joinTests.cpp)
TARGET_LINK_LIBRARIES(
joinTests
PRIVATE os util common executor gtest_main qcom function planner scalar nodes vnode
)
# SET(CMAKE_CXX_STANDARD 11)
# AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
# ADD_EXECUTABLE(tSimpleHashTest tSimpleHashTests.cpp)
# TARGET_LINK_LIBRARIES(
# tSimpleHashTest
# PRIVATE os util common executor gtest_main
# )
# TARGET_INCLUDE_DIRECTORIES(
# tSimpleHashTest
# PUBLIC "${TD_SOURCE_DIR}/include/common"
# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
# )
TARGET_INCLUDE_DIRECTORIES(
joinTests
PUBLIC "${TD_SOURCE_DIR}/include/common"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)

File diff suppressed because it is too large Load Diff

View File

@ -47,6 +47,7 @@ typedef struct SBuiltinFuncDefinition {
const char* pPartialFunc;
const char* pMiddleFunc;
const char* pMergeFunc;
const char* pStateFunc;
FCreateMergeFuncParameters createMergeParaFuc;
FEstimateReturnRows estimateReturnRowsFunc;
} SBuiltinFuncDefinition;

View File

@ -53,7 +53,10 @@ extern "C" {
#define FUNC_MGT_GEOMETRY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(24)
#define FUNC_MGT_FORBID_SYSTABLE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(25)
#define FUNC_MGT_SKIP_SCAN_CHECK_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(26)
#define FUNC_MGT_PRIMARY_KEY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(27)
#define FUNC_MGT_IGNORE_NULL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(27)
#define FUNC_MGT_PRIMARY_KEY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(28)
#define FUNC_MGT_TSMA_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(29)
#define FUNC_MGT_COUNT_LIKE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

View File

@ -46,7 +46,7 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
#define TIME_UNIT_TOO_SMALL 2
static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
if (!pVal->isDuration) {
if (!IS_DURATION_VAL(pVal->flag)) {
return TIME_UNIT_INVALID;
}
@ -225,7 +225,6 @@ static int32_t addTimezoneParam(SNodeList* pList) {
}
pVal->literal = strndup(buf, len);
pVal->isDuration = false;
pVal->translate = true;
pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
@ -245,7 +244,6 @@ static int32_t addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
}
pVal->literal = NULL;
pVal->isDuration = false;
pVal->translate = true;
pVal->notReserved = true;
pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT;
@ -462,6 +460,24 @@ static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return TSDB_CODE_SUCCESS;
}
static int32_t translateAvgState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType))
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateAvgStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateStddevPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@ -491,6 +507,35 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t
return TSDB_CODE_SUCCESS;
}
static int32_t translateStddevState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getStddevInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateStddevStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (TSDB_DATA_TYPE_BINARY != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getStddevInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT,
@ -547,6 +592,9 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
for (int32_t i = 1; i < numOfParams; ++i) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, i);
if (QUERY_NODE_VALUE != nodeType(pValue)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pValue->notReserved = true;
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, i))->type;
@ -834,6 +882,31 @@ static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t
return translateSpreadImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateSpreadState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_NUMERIC_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateSpreadStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (paraType != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (1 != numOfParams && 2 != numOfParams) {
@ -1317,6 +1390,24 @@ static int32_t translateHLLMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return translateHLLImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateHLLState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateHLLPartial(pFunc, pErrBuf, len);
}
static int32_t translateHLLStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
if (getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType =
(SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static bool validateStateOper(const SValueNode* pVal) {
if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
return false;
@ -1804,6 +1895,28 @@ static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int3
return translateFirstLastImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateFirstLastState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
int32_t paraBytes = getSDataTypeFromNode(pPara)->bytes;
int32_t pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0;
pFunc->node.resType =
(SDataType){.bytes = getFirstLastInfoSize(paraBytes, pkBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateFirstLastStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
int32_t paraBytes = getSDataTypeFromNode(pPara)->bytes;
uint8_t paraType = getSDataTypeFromNode(pPara)->type;
if (paraType != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = paraBytes, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isUnique) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@ -2394,12 +2507,26 @@ static int32_t translateTableCountPseudoColumn(SFunctionNode* pFunc, char* pErrB
return TSDB_CODE_SUCCESS;
}
static int32_t translateMd5(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t para1Type = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (para1Type != TSDB_DATA_TYPE_VARCHAR) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = MD5_OUTPUT_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
// clang-format off
const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "count",
.type = FUNCTION_TYPE_COUNT,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_COUNT_LIKE_FUNC,
.translateFunc = translateCount,
.dataRequiredFunc = countDataRequired,
.getEnvFunc = getCountFuncEnv,
@ -2412,12 +2539,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = combineFunction,
.pPartialFunc = "count",
.pStateFunc = "count",
.pMergeFunc = "sum"
},
{
.name = "sum",
.type = FUNCTION_TYPE_SUM,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSum,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSumFuncEnv,
@ -2430,12 +2558,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = sumCombine,
.pPartialFunc = "sum",
.pStateFunc = "sum",
.pMergeFunc = "sum"
},
{
.name = "min",
.type = FUNCTION_TYPE_MIN,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInOutNum,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getMinmaxFuncEnv,
@ -2445,12 +2574,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = minmaxFunctionFinalize,
.combineFunc = minCombine,
.pPartialFunc = "min",
.pStateFunc = "min",
.pMergeFunc = "min"
},
{
.name = "max",
.type = FUNCTION_TYPE_MAX,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInOutNum,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getMinmaxFuncEnv,
@ -2460,12 +2590,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = minmaxFunctionFinalize,
.combineFunc = maxCombine,
.pPartialFunc = "max",
.pStateFunc = "max",
.pMergeFunc = "max"
},
{
.name = "stddev",
.type = FUNCTION_TYPE_STDDEV,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInNumOutDou,
.getEnvFunc = getStddevFuncEnv,
.initFunc = stddevFunctionSetup,
@ -2477,6 +2608,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = stddevCombine,
.pPartialFunc = "_stddev_partial",
.pStateFunc = "_stddev_state",
.pMergeFunc = "_stddev_merge"
},
{
@ -2506,6 +2638,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = stddevInvertFunction,
#endif
.combineFunc = stddevCombine,
.pPartialFunc = "_stddev_state_merge",
.pMergeFunc = "_stddev_merge",
},
{
.name = "leastsquares",
@ -2525,7 +2659,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "avg",
.type = FUNCTION_TYPE_AVG,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInNumOutDou,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getAvgFuncEnv,
@ -2539,12 +2673,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.combineFunc = avgCombine,
.pPartialFunc = "_avg_partial",
.pMiddleFunc = "_avg_middle",
.pMergeFunc = "_avg_merge"
.pMergeFunc = "_avg_merge",
.pStateFunc = "_avg_state",
},
{
.name = "_avg_partial",
.type = FUNCTION_TYPE_AVG_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateAvgPartial,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getAvgFuncEnv,
@ -2559,7 +2694,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_avg_merge",
.type = FUNCTION_TYPE_AVG_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateAvgMerge,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
@ -2569,6 +2704,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = avgInvertFunction,
#endif
.combineFunc = avgCombine,
.pPartialFunc = "_avg_state_merge",
.pMergeFunc = "_avg_merge",
},
{
.name = "percentile",
@ -2636,7 +2773,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "top",
.type = FUNCTION_TYPE_TOP,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC |
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC,
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateTopBot,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = topBotFunctionSetup,
@ -2652,7 +2789,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "bottom",
.type = FUNCTION_TYPE_BOTTOM,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC |
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC,
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateTopBot,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = topBotFunctionSetup,
@ -2667,7 +2804,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "spread",
.type = FUNCTION_TYPE_SPREAD,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSpread,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
@ -2680,6 +2817,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = spreadCombine,
.pPartialFunc = "_spread_partial",
.pStateFunc = "_spread_state",
.pMergeFunc = "_spread_merge"
},
{
@ -2711,6 +2849,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = NULL,
#endif
.combineFunc = spreadCombine,
.pPartialFunc = "_spread_state_merge",
.pMergeFunc = "_spread_merge",
},
{
.name = "elapsed",
@ -2850,7 +2990,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_cache_last",
.type = FUNCTION_TYPE_CACHE_LAST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateFirstLast,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -2884,7 +3024,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "first",
.type = FUNCTION_TYPE_FIRST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateFirstLast,
.dynDataRequiredFunc = firstDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2893,6 +3033,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = firstLastScalarFunction,
.finalizeFunc = firstLastFinalize,
.pPartialFunc = "_first_partial",
.pStateFunc = "_first_state",
.pMergeFunc = "_first_merge",
.combineFunc = firstCombine,
},
@ -2900,7 +3041,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_first_partial",
.type = FUNCTION_TYPE_FIRST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastPartial,
.dynDataRequiredFunc = firstDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2913,19 +3054,21 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_first_merge",
.type = FUNCTION_TYPE_FIRST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = firstFunctionMerge,
.finalizeFunc = firstLastFinalize,
.combineFunc = firstCombine,
.pPartialFunc = "_first_state_merge",
.pMergeFunc = "_first_merge",
},
{
.name = "last",
.type = FUNCTION_TYPE_LAST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateFirstLast,
.dynDataRequiredFunc = lastDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2934,6 +3077,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = firstLastScalarFunction,
.finalizeFunc = firstLastFinalize,
.pPartialFunc = "_last_partial",
.pStateFunc = "_last_state",
.pMergeFunc = "_last_merge",
.combineFunc = lastCombine,
},
@ -2941,7 +3085,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_last_partial",
.type = FUNCTION_TYPE_LAST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastPartial,
.dynDataRequiredFunc = lastDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2954,13 +3098,15 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_last_merge",
.type = FUNCTION_TYPE_LAST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastFunctionMerge,
.finalizeFunc = firstLastFinalize,
.combineFunc = lastCombine,
.pPartialFunc = "_last_state_merge",
.pMergeFunc = "_last_merge",
},
{
.name = "twa",
@ -3023,7 +3169,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "hyperloglog",
.type = FUNCTION_TYPE_HYPERLOGLOG,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_COUNT_LIKE_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateHLL,
.getEnvFunc = getHLLFuncEnv,
.initFunc = functionSetup,
@ -3035,6 +3181,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = hllCombine,
.pPartialFunc = "_hyperloglog_partial",
.pStateFunc = "_hyperloglog_state",
.pMergeFunc = "_hyperloglog_merge"
},
{
@ -3064,6 +3211,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = NULL,
#endif
.combineFunc = hllCombine,
.pPartialFunc = "_hyperloglog_state_merge",
.pMergeFunc = "_hyperloglog_merge",
},
{
.name = "diff",
@ -3823,7 +3972,153 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = NULL,
.sprocessFunc = qPseudoTagFunction,
.finalizeFunc = NULL
}
},
{
.name = "_stddev_state",
.type = FUNCTION_TYPE_STDDEV_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateStddevState,
.getEnvFunc = getStddevFuncEnv,
.initFunc = stddevFunctionSetup,
.processFunc = stddevFunction,
.finalizeFunc = stddevPartialFinalize,
.pPartialFunc = "_stddev_partial",
.pMergeFunc = "_stddev_state_merge",
},
{
.name = "_stddev_state_merge",
.type = FUNCTION_TYPE_STDDEV_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateStddevStateMerge,
.getEnvFunc = getStddevFuncEnv,
.initFunc = stddevFunctionSetup,
.processFunc = stddevFunctionMerge,
.finalizeFunc = stddevPartialFinalize,
},
{
.name = "_avg_state",
.type = FUNCTION_TYPE_AVG_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateAvgState,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
.processFunc = avgFunction,
.finalizeFunc = avgPartialFinalize,
.pPartialFunc = "_avg_partial",
.pMergeFunc = "_avg_state_merge"
},
{
.name = "_avg_state_merge",
.type = FUNCTION_TYPE_AVG_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateAvgStateMerge,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
.processFunc = avgFunctionMerge,
.finalizeFunc = avgPartialFinalize,
},
{
.name = "_spread_state",
.type = FUNCTION_TYPE_SPREAD_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSpreadState,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunction,
.finalizeFunc = spreadPartialFinalize,
.pPartialFunc = "_spread_partial",
.pMergeFunc = "_spread_state_merge"
},
{
.name = "_spread_state_merge",
.type = FUNCTION_TYPE_SPREAD_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSpreadStateMerge,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunctionMerge,
.finalizeFunc = spreadPartialFinalize,
},
{
.name = "_first_state",
.type = FUNCTION_TYPE_FIRST_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastState,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = firstFunction,
.finalizeFunc = firstLastPartialFinalize,
.pPartialFunc = "_first_partial",
.pMergeFunc = "_first_state_merge"
},
{
.name = "_first_state_merge",
.type = FUNCTION_TYPE_FIRST_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastStateMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = firstFunctionMerge,
.finalizeFunc = firstLastPartialFinalize,
},
{
.name = "_last_state",
.type = FUNCTION_TYPE_LAST_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastState,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastFunction,
.finalizeFunc = firstLastPartialFinalize,
.pPartialFunc = "_last_partial",
.pMergeFunc = "_last_state_merge"
},
{
.name = "_last_state_merge",
.type = FUNCTION_TYPE_LAST_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastStateMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastFunctionMerge,
.finalizeFunc = firstLastPartialFinalize,
},
{
.name = "_hyperloglog_state",
.type = FUNCTION_TYPE_HYPERLOGLOG_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_COUNT_LIKE_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateHLLState,
.getEnvFunc = getHLLFuncEnv,
.initFunc = functionSetup,
.processFunc = hllFunction,
.finalizeFunc = hllPartialFinalize,
.pPartialFunc = "_hyperloglog_partial",
.pMergeFunc = "_hyperloglog_state_merge",
},
{
.name = "_hyperloglog_state_merge",
.type = FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_COUNT_LIKE_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateHLLStateMerge,
.getEnvFunc = getHLLFuncEnv,
.initFunc = functionSetup,
.processFunc = hllFunctionMerge,
.finalizeFunc = hllPartialFinalize,
},
{
.name = "md5",
.type = FUNCTION_TYPE_MD5,
.classification = FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateMd5,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = md5Function,
.finalizeFunc = NULL
},
};
// clang-format on

View File

@ -867,6 +867,12 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
}
_sum_over:
if (numOfElem == 0) {
if (tsCountAlwaysReturnValue && pCtx->pExpr->pExpr->_function.pFunctNode->hasOriginalFunc &&
fmIsCountLikeFunc(pCtx->pExpr->pExpr->_function.pFunctNode->originalFuncId)) {
numOfElem = 1;
}
}
// data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS;
@ -1448,6 +1454,12 @@ static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) {
int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -1455,6 +1467,7 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {
SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data);
stddevTransferInfo(pInputInfo, pInfo);
@ -2846,6 +2859,12 @@ static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput
static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -3273,6 +3292,11 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
if (pDiffInfo->includeNull) {
colDataSetNull_f_s(pOutput, pos);
// handle selectivity
if (pCtx->subsidiaries.num > 0) {
appendSelectivityCols(pCtx, row.block, row.rowIndex, pos);
}
numOfElems += 1;
}
continue;
@ -3847,6 +3871,12 @@ static void spreadTransferInfo(SSpreadInfo* pInput, SSpreadInfo* pOutput) {
int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -3855,6 +3885,7 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
if(colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data);
if (pInputInfo->hasResult) {
@ -4583,6 +4614,11 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_SUCCESS;
}
@ -4592,6 +4628,7 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
if (colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data);
hllTransferInfo(pInputInfo, pInfo);

View File

@ -705,6 +705,12 @@ static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) {
int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -714,6 +720,7 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
if(colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data);
avgTransferInfo(pInputInfo, pInfo);

View File

@ -274,6 +274,8 @@ bool fmIsBlockDistFunc(int32_t funcId) {
return FUNCTION_TYPE_BLOCK_DIST == funcMgtBuiltins[funcId].type;
}
bool fmIsIgnoreNullFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IGNORE_NULL_FUNC); }
void fmFuncMgtDestroy() {
void* m = gFunMgtService.pFuncNameHashTable;
if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) {
@ -411,6 +413,8 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pPartialFunc)->hasOriginalFunc = true;
(*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0};
int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
taosCreateMD5Hash(name, len);
@ -473,6 +477,8 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
}
}
if (TSDB_CODE_SUCCESS == code) {
pFunc->hasOriginalFunc = true;
pFunc->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
// overwrite function restype set by translate function
if (fmIsSameInOutType(pSrcFunc->funcId)) {
pFunc->node.resType = pSrcFunc->node.resType;
@ -496,7 +502,7 @@ int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc
}
int32_t code = createPartialFunction(pFunc, pPartialFunc);
if (TSDB_CODE_SUCCESS == code) {
if (TSDB_CODE_SUCCESS == code && pMidFunc) {
code = createMidFunction(pFunc, *pPartialFunc, pMidFunc);
}
if (TSDB_CODE_SUCCESS == code) {
@ -505,7 +511,7 @@ int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)*pPartialFunc);
nodesDestroyNode((SNode*)*pMidFunc);
if (pMidFunc) nodesDestroyNode((SNode*)*pMidFunc);
nodesDestroyNode((SNode*)*pMergeFunc);
}
@ -518,3 +524,122 @@ char* fmGetFuncName(int32_t funcId) {
}
return taosStrdup(funcMgtBuiltins[funcId].name);
}
/// @param [out] pStateFunc, not changed if error occured or no need to create state func
/// @retval 0 for succ, otherwise err occured
static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pStateFunc) {
if (funcMgtBuiltins[pFunc->funcId].pStateFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams);
if (!*pStateFunc) {
nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR;
}
strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias);
}
return TSDB_CODE_SUCCESS;
}
bool fmIsTSMASupportedFunc(func_id_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_TSMA_FUNC) &&
!isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_STREAM_FUNC);
}
int32_t fmCreateStateFuncs(SNodeList* pFuncs) {
int32_t code;
SNode* pNode;
char buf[128] = {0};
FOREACH(pNode, pFuncs) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
code = fmGetFuncInfo(pFunc, buf, 128);
if (code) break;
if (fmIsTSMASupportedFunc(pFunc->funcId)) {
SFunctionNode* pNewFunc = NULL;
code = fmCreateStateFunc(pFunc, &pNewFunc);
if (code) {
// error
break;
} else if (!pNewFunc) {
// no need state func
continue;
} else {
REPLACE_NODE(pNewFunc);
nodesDestroyNode(pNode);
}
}
}
return code;
}
static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pStateMergeFunc) {
if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateMergeFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams);
if (!*pStateMergeFunc) {
nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR;
}
strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias);
}
return TSDB_CODE_SUCCESS;
}
int32_t fmCreateStateMergeFuncs(SNodeList* pFuncs) {
int32_t code;
SNode* pNode;
char buf[128] = {0};
FOREACH(pNode, pFuncs) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (fmIsTSMASupportedFunc(pFunc->funcId)) {
SFunctionNode* pNewFunc = NULL;
code = fmCreateStateMergeFunc(pFunc, &pNewFunc);
if (code) {
// error
break;
} else if (!pNewFunc) {
// no state merge func
continue;
} else {
REPLACE_NODE(pNewFunc);
nodesDestroyNode(pNode);
}
}
}
return code;
}
int32_t fmGetFuncId(const char* name) {
if (NULL != gFunMgtService.pFuncNameHashTable) {
void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, name, strlen(name));
if (NULL != pVal) {
return *(int32_t*)pVal;
}
return -1;
}
for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) {
if (0 == strcmp(funcMgtBuiltins[i].name, name)) {
return i;
}
}
return -1;
}
bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) {
const SBuiltinFuncDefinition* pFunc = &funcMgtBuiltins[funcId];
const SBuiltinFuncDefinition* pStateFunc = &funcMgtBuiltins[stateFuncId];
if (!pFunc->pStateFunc) {
return false;
}
if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true;
int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc);
const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId];
return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0;
}
bool fmIsCountLikeFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_COUNT_LIKE_FUNC);
}

View File

@ -114,6 +114,7 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
COPY_SCALAR_FIELD(projIdx);
COPY_SCALAR_FIELD(colType);
COPY_SCALAR_FIELD(hasIndex);
COPY_SCALAR_FIELD(isPrimTs);
COPY_CHAR_ARRAY_FIELD(dbName);
COPY_CHAR_ARRAY_FIELD(tableName);
COPY_CHAR_ARRAY_FIELD(tableAlias);
@ -138,7 +139,7 @@ static int32_t columnDefNodeCopy(const SColumnDefNode* pSrc, SColumnDefNode* pDs
static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_CHAR_POINT_FIELD(literal);
COPY_SCALAR_FIELD(isDuration);
COPY_SCALAR_FIELD(flag);
COPY_SCALAR_FIELD(translate);
COPY_SCALAR_FIELD(notReserved);
COPY_SCALAR_FIELD(isNull);
@ -223,6 +224,8 @@ static int32_t functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst)
COPY_SCALAR_FIELD(udfBufSize);
COPY_SCALAR_FIELD(hasPk);
COPY_SCALAR_FIELD(pkBytes);
COPY_SCALAR_FIELD(hasOriginalFunc);
COPY_SCALAR_FIELD(originalFuncId);
return TSDB_CODE_SUCCESS;
}
@ -297,6 +300,7 @@ static int32_t tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDs
static int32_t joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) {
COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
COPY_SCALAR_FIELD(joinType);
COPY_SCALAR_FIELD(subType);
COPY_SCALAR_FIELD(hasSubQuery);
COPY_SCALAR_FIELD(isLowLevelJoin);
CLONE_NODE_FIELD(pLeft);
@ -413,6 +417,14 @@ static int32_t hintNodeCopy(const SHintNode* pSrc, SHintNode* pDst) {
return copyHintValue(pSrc, pDst);
}
static int32_t windowOffsetCopy(const SWindowOffsetNode* pSrc, SWindowOffsetNode* pDst) {
COPY_SCALAR_FIELD(type);
CLONE_NODE_FIELD(pStartOffset);
CLONE_NODE_FIELD(pEndOffset);
return TSDB_CODE_SUCCESS;
}
static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions);
@ -473,20 +485,37 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
CLONE_OBJECT_FIELD(pFuncTypes, functParamClone);
COPY_SCALAR_FIELD(paraTablesSort);
COPY_SCALAR_FIELD(smallDataTsSort);
COPY_SCALAR_FIELD(needSplit);
return TSDB_CODE_SUCCESS;
}
static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(joinType);
COPY_SCALAR_FIELD(subType);
COPY_SCALAR_FIELD(joinAlgo);
CLONE_NODE_FIELD(pWindowOffset);
CLONE_NODE_FIELD(pJLimit);
CLONE_NODE_FIELD(addPrimEqCond);
CLONE_NODE_FIELD(pPrimKeyEqCond);
CLONE_NODE_FIELD(pColEqCond);
CLONE_NODE_FIELD(pColOnCond);
CLONE_NODE_FIELD(pTagEqCond);
CLONE_NODE_FIELD(pTagOnCond);
CLONE_NODE_FIELD(pOtherOnCond);
CLONE_NODE_FIELD(pFullOnCond);
CLONE_NODE_LIST_FIELD(pLeftEqNodes);
CLONE_NODE_LIST_FIELD(pRightEqNodes);
COPY_SCALAR_FIELD(allEqTags);
COPY_SCALAR_FIELD(isSingleTableJoin);
COPY_SCALAR_FIELD(hasSubQuery);
COPY_SCALAR_FIELD(isLowLevelJoin);
COPY_SCALAR_FIELD(seqWinGroup);
COPY_SCALAR_FIELD(grpJoin);
COPY_SCALAR_FIELD(hashJoinHint);
CLONE_NODE_FIELD(pLeftOnCond);
CLONE_NODE_FIELD(pRightOnCond);
COPY_SCALAR_FIELD(timeRangeTarget);
COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
return TSDB_CODE_SUCCESS;
}
@ -546,6 +575,8 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst
CLONE_NODE_LIST_FIELD(pInputs);
COPY_SCALAR_FIELD(numOfChannels);
COPY_SCALAR_FIELD(srcGroupId);
COPY_SCALAR_FIELD(srcEndGroupId);
COPY_SCALAR_FIELD(numOfSubplans);
COPY_SCALAR_FIELD(colsMerge);
COPY_SCALAR_FIELD(needSort);
COPY_SCALAR_FIELD(groupSort);
@ -811,6 +842,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
COPY_SCALAR_FIELD(precision);
COPY_SCALAR_FIELD(isEmptyResult);
COPY_SCALAR_FIELD(timeLineResMode);
COPY_SCALAR_FIELD(timeLineCurMode);
COPY_SCALAR_FIELD(hasAggFuncs);
COPY_SCALAR_FIELD(hasRepeatScanFuncs);
CLONE_NODE_LIST_FIELD(pHint);
@ -924,6 +956,9 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_HINT:
code = hintNodeCopy((const SHintNode*)pNode, (SHintNode*)pDst);
break;
case QUERY_NODE_WINDOW_OFFSET:
code = windowOffsetCopy((const SWindowOffsetNode*)pNode, (SWindowOffsetNode*)pDst);
break;
case QUERY_NODE_SET_OPERATOR:
code = setOperatorCopy((const SSetOperator*)pNode, (SSetOperator*)pDst);
break;

View File

@ -91,6 +91,8 @@ const char* nodesNodeName(ENodeType type) {
return "CaseWhen";
case QUERY_NODE_EVENT_WINDOW:
return "EventWindow";
case QUERY_NODE_WINDOW_OFFSET:
return "WindowOffset";
case QUERY_NODE_COUNT_WINDOW:
return "CountWindow";
case QUERY_NODE_SET_OPERATOR:
@ -1623,7 +1625,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqCondition, nodeToJson, pNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOtherOnCond);
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pFullOnCond);
}
return code;
}
@ -1648,7 +1650,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqCondition, &pNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOtherOnCond);
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pFullOnCond);
}
return code;
@ -2168,15 +2170,34 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
}
static const char* jkJoinPhysiPlanJoinType = "JoinType";
static const char* jkJoinPhysiPlanSubType = "SubType";
static const char* jkJoinPhysiPlanWinOffset = "WindowOffset";
static const char* jkJoinPhysiPlanJoinLimit = "JoinLimit";
static const char* jkJoinPhysiPlanAsofOp = "AsofOp";
static const char* jkJoinPhysiPlanLeftPrimExpr = "LeftPrimExpr";
static const char* jkJoinPhysiPlanRightPrimExpr = "RightPrimExpr";
static const char* jkJoinPhysiPlanLeftPrimSlotId = "LeftPrimSlotId";
static const char* jkJoinPhysiPlanRightPrimSlotId = "RightPrimSlotId";
static const char* jkJoinPhysiPlanLeftEqCols = "LeftEqCols";
static const char* jkJoinPhysiPlanRightEqCols = "RightEqCols";
static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder";
static const char* jkJoinPhysiPlanOnLeftCols = "OnLeftColumns";
static const char* jkJoinPhysiPlanOnRightCols = "OnRightColumns";
static const char* jkJoinPhysiPlanPrimKeyCondition = "PrimKeyCondition";
static const char* jkJoinPhysiPlanOnConditions = "OnConditions";
static const char* jkJoinPhysiPlanTargets = "Targets";
static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions";
static const char* jkJoinPhysiPlanInputRowNum = "InputRowNum";
static const char* jkJoinPhysiPlanInputRowSize = "InputRowSize";
static const char* jkJoinPhysiPlanColOnConditions = "ColumnOnConditions";
static const char* jkJoinPhysiPlanLeftInputRowNum = "LeftInputRowNum";
static const char* jkJoinPhysiPlanRightInputRowNum = "RightInputRowNum";
static const char* jkJoinPhysiPlanLeftInputRowSize = "LeftInputRowSize";
static const char* jkJoinPhysiPlanRightInputRowSize = "RightInputRowSize";
static const char* jkJoinPhysiPlanSeqWinGroup = "SeqWinGroup";
static const char* jkJoinPhysiPlanGroupJoin = "GroupJoin";
static const char* jkJoinPhysiPlanLeftOnCond = "LeftOnCond";
static const char* jkJoinPhysiPlanRightOnCond = "RightOnCond";
static const char* jkJoinPhysiPlanTimeRangeSKey = "TimeRangeSKey";
static const char* jkJoinPhysiPlanTimeRangeEKey = "TimeRangeEKey";
static const char* jkJoinPhysiPlanTimeRangeTarget = "TimeRangeTarget";
static int32_t physiMergeJoinNodeToJson(const void* pObj, SJson* pJson) {
const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj;
@ -2186,17 +2207,63 @@ static int32_t physiMergeJoinNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanPrimKeyCondition, nodeToJson, pNode->pPrimKeyCond);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanSubType, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOtherOnCond);
code = tjsonAddObject(pJson, jkJoinPhysiPlanWinOffset, nodeToJson, pNode->pWindowOffset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanJoinLimit, nodeToJson, pNode->pJLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanAsofOp, pNode->asofOpType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanLeftPrimExpr, nodeToJson, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanRightPrimExpr, nodeToJson, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanLeftEqCols, pNode->pEqLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanRightEqCols, pNode->pEqRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqCond);
code = tjsonAddObject(pJson, jkJoinPhysiPlanColOnConditions, nodeToJson, pNode->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkJoinPhysiPlanSeqWinGroup, pNode->seqWinGroup);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkJoinPhysiPlanGroupJoin, pNode->grpJoin);
}
return code;
}
@ -2208,17 +2275,63 @@ static int32_t jsonToPhysiMergeJoinNode(const SJson* pJson, void* pObj) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOtherOnCond);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanSubType, pNode->subType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanPrimKeyCondition, &pNode->pPrimKeyCond);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanWinOffset, &pNode->pWindowOffset);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanJoinLimit, &pNode->pJLimit);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanAsofOp, pNode->asofOpType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanLeftPrimExpr, &pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanRightPrimExpr, &pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanLeftEqCols, &pNode->pEqLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanRightEqCols, &pNode->pEqRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqCond);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColOnConditions, &pNode->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkJoinPhysiPlanSeqWinGroup, &pNode->seqWinGroup);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkJoinPhysiPlanGroupJoin, &pNode->grpJoin);
}
return code;
}
@ -2229,6 +2342,9 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanSubType, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanOnLeftCols, pNode->pOnLeft);
}
@ -2236,23 +2352,51 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
code = nodeListToJson(pJson, jkJoinPhysiPlanOnRightCols, pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFilterConditions);
code = tjsonAddObject(pJson, jkJoinPhysiPlanLeftPrimExpr, nodeToJson, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanRightPrimExpr, nodeToJson, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[0].inputRowNum);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[0].inputRowSize);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[1].inputRowNum);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[1].inputRowSize);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanLeftOnCond, nodeToJson, pNode->pLeftOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanRightOnCond, nodeToJson, pNode->pRightOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanTimeRangeTarget, pNode->timeRangeTarget);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanTimeRangeSKey, pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanTimeRangeEKey, pNode->timeRange.ekey);
}
return code;
}
@ -2263,6 +2407,9 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanSubType, pNode->subType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanOnLeftCols, &pNode->pOnLeft);
}
@ -2270,23 +2417,51 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanOnRightCols, &pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFilterConditions);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanLeftPrimExpr, &pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanRightPrimExpr, &pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[0].inputRowNum, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[0].inputRowSize, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[1].inputRowNum, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[1].inputRowSize, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanLeftOnCond, &pNode->pLeftOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanRightOnCond, &pNode->pRightOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanTimeRangeTarget, pNode->timeRangeTarget, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkJoinPhysiPlanTimeRangeSKey, &pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkJoinPhysiPlanTimeRangeEKey, &pNode->timeRange.ekey);
}
return code;
}
@ -2295,6 +2470,7 @@ static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock";
static const char* jkAggPhysiPlanGroupKeyOptimized = "GroupKeyOptimized";
static const char* jkAggPhysiPlanHasCountLikeFunc = "HasCountFunc";
static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj;
@ -2315,6 +2491,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanGroupKeyOptimized, pNode->groupKeyOptimized);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanHasCountLikeFunc, pNode->hasCountLikeFunc);
}
return code;
}
@ -2338,6 +2517,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanGroupKeyOptimized, &pNode->groupKeyOptimized);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanHasCountLikeFunc, &pNode->hasCountLikeFunc);
}
return code;
}
@ -3730,7 +3912,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
static const char* jkValueLiteralSize = "LiteralSize";
static const char* jkValueLiteral = "Literal";
static const char* jkValueDuration = "Duration";
static const char* jkValueFlag = "Flag";
static const char* jkValueTranslate = "Translate";
static const char* jkValueNotReserved = "NotReserved";
static const char* jkValueIsNull = "IsNull";
@ -3814,7 +3996,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkValueDuration, pNode->isDuration);
code = tjsonAddBoolToObject(pJson, jkValueFlag, pNode->flag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkValueTranslate, pNode->translate);
@ -3968,7 +4150,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) {
code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->isDuration);
code = tjsonGetIntValue(pJson, jkValueFlag, &pNode->flag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkValueTranslate, &pNode->translate);
@ -4065,6 +4247,8 @@ static const char* jkFunctionParameter = "Parameters";
static const char* jkFunctionUdfBufSize = "UdfBufSize";
static const char* jkFunctionHasPk = "HasPk";
static const char* jkFunctionPkBytes = "PkBytes";
static const char* jkFunctionIsMergeFunc = "IsMergeFunc";
static const char* jkFunctionMergeFuncOf = "MergeFuncOf";
static int32_t functionNodeToJson(const void* pObj, SJson* pJson) {
const SFunctionNode* pNode = (const SFunctionNode*)pObj;
@ -4091,6 +4275,13 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkFunctionPkBytes, pNode->pkBytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkFunctionIsMergeFunc, pNode->hasOriginalFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkFunctionMergeFuncOf, pNode->originalFuncId);
}
return code;
}
@ -4119,6 +4310,13 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkFunctionPkBytes, &pNode->pkBytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkFunctionIsMergeFunc, &pNode->hasOriginalFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkFunctionMergeFuncOf, &pNode->originalFuncId);
}
return code;
}
@ -4313,6 +4511,7 @@ static int32_t jsonToTempTableNode(const SJson* pJson, void* pObj) {
}
static const char* jkJoinTableJoinType = "JoinType";
static const char* jkJoinTableSubType = "SubType";
static const char* jkJoinTableLeft = "Left";
static const char* jkJoinTableRight = "Right";
static const char* jkJoinTableOnCond = "OnCond";
@ -4324,6 +4523,9 @@ static int32_t joinTableNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinTableJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinTableSubType, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinTableLeft, nodeToJson, pNode->pLeft);
}
@ -4344,6 +4546,9 @@ static int32_t jsonToJoinTableNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinTableJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinTableSubType, pNode->subType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinTableLeft, &pNode->pLeft);
}
@ -4832,6 +5037,29 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkWindowOffsetStartOffset = "StartOffset";
static const char* jkWindowOffsetEndOffset = "EndOffset";
static int32_t windowOffsetNodeToJson(const void* pObj, SJson* pJson) {
const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkWindowOffsetStartOffset, nodeToJson, pNode->pStartOffset);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkWindowOffsetEndOffset, nodeToJson, pNode->pEndOffset);
}
return code;
}
static int32_t jsonToWindowOffsetNode(const SJson* pJson, void* pObj) {
SWindowOffsetNode* pNode = (SWindowOffsetNode*)pObj;
int32_t code = jsonToNodeObject(pJson, jkWindowOffsetStartOffset, &pNode->pStartOffset);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkWindowOffsetEndOffset, &pNode->pEndOffset);
}
return code;
}
static const char* jkDatabaseOptionsBuffer = "Buffer";
static const char* jkDatabaseOptionsCacheModel = "CacheModel";
static const char* jkDatabaseOptionsCompressionLevel = "CompressionLevel";
@ -7112,6 +7340,105 @@ static int32_t jsonToInsertStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkTSMAOptionFuncs = "Funcs";
static const char* jkTSMAOptionCols = "Cols";
static const char* jkTSMAOptionInterval = "Interval";
static const char* jkTSMAOptionTsPrecision = "Precision";
static int32_t tsmaOptionToJson(const void* pObj, SJson* pJson) {
const STSMAOptions* pNode = (const STSMAOptions*)pObj;
int32_t code = nodeListToJson(pJson, jkTSMAOptionFuncs, pNode->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTSMAOptionInterval, nodeToJson, pNode->pInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTSMAOptionTsPrecision, pNode->tsPrecision);
}
return code;
}
static int32_t jsonToTSMAOption(const SJson* pJson, void* pObj) {
STSMAOptions* pNode = (STSMAOptions*)pObj;
int32_t code = jsonToNodeList(pJson, jkTSMAOptionFuncs, &pNode->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkTSMAOptionInterval, &pNode->pInterval);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetUTinyIntValue(pJson, jkTSMAOptionTsPrecision, &pNode->tsPrecision);
}
return code;
}
static const char* jkCreateTSMAStmtIgnoreExists = "IgnoreExists";
static const char* jkCreateTSMAStmtTsmaName = "TSMAName";
static const char* jkCreateTSMAStmtDbName = "DbName";
static const char* jkCreateTSMAStmtTableName = "TableName";
static const char* jkCreateTSMAStmtpOptions = "Options";
static int32_t createTSMAStmtToJson(const void* pObj, SJson* pJson) {
const SCreateTSMAStmt* pNode = (const SCreateTSMAStmt*)pObj;
int32_t code = tjsonAddBoolToObject(pJson, jkCreateTSMAStmtIgnoreExists, pNode->ignoreExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTSMAStmtTsmaName, pNode->tsmaName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTSMAStmtTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkCreateTSMAStmtpOptions, nodeToJson, pNode->pOptions);
}
return code;
}
static int32_t jsonToCreateTSMAStmt(const SJson* pJson, void* pObj) {
SCreateTSMAStmt* pNode = (SCreateTSMAStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkCreateTSMAStmtIgnoreExists, &pNode->ignoreExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTSMAStmtTsmaName, pNode->tsmaName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTSMAStmtTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkCreateTSMAStmtpOptions, (SNode**)&pNode->pOptions);
}
return code;
}
static const char* jkDropTSMAStmtIgnoreNotExists = "IgnoreNotExists";
static const char* jkDropTSMAStmtDbName = "DbName";
static const char* jkDropTSMAStmtTsmaName = "TSMAName";
static int32_t dropTSMAStmtToJson(const void* pObj, SJson* pJson) {
const SDropTSMAStmt* pNode = (const SDropTSMAStmt*)pObj;
int32_t code = tjsonAddBoolToObject(pJson, jkDropTSMAStmtIgnoreNotExists, pNode->ignoreNotExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkDropTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkDropTSMAStmtTsmaName, pNode->tsmaName);
}
return code;
}
static int32_t jsonToDropTSMAStmt(const SJson* pJson, void* pObj) {
SDropTSMAStmt* pNode = (SDropTSMAStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkDropTSMAStmtIgnoreNotExists, &pNode->ignoreNotExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkDropTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkDropTSMAStmtTsmaName, pNode->tsmaName);
}
return code;
}
static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
switch (nodeType(pObj)) {
case QUERY_NODE_COLUMN:
@ -7176,6 +7503,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return caseWhenNodeToJson(pObj, pJson);
case QUERY_NODE_EVENT_WINDOW:
return eventWindowNodeToJson(pObj, pJson);
case QUERY_NODE_WINDOW_OFFSET:
return windowOffsetNodeToJson(pObj, pJson);
case QUERY_NODE_COUNT_WINDOW:
return countWindowNodeToJson(pObj, pJson);
case QUERY_NODE_SET_OPERATOR:
@ -7519,6 +7848,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToCaseWhenNode(pJson, pObj);
case QUERY_NODE_EVENT_WINDOW:
return jsonToEventWindowNode(pJson, pObj);
case QUERY_NODE_WINDOW_OFFSET:
return jsonToWindowOffsetNode(pJson, pObj);
case QUERY_NODE_COUNT_WINDOW:
return jsonToCountWindowNode(pJson, pObj);
case QUERY_NODE_SET_OPERATOR:

View File

@ -808,7 +808,7 @@ static int32_t msgToColumnNode(STlvDecoder* pDecoder, void* pObj) {
enum {
VALUE_CODE_EXPR_BASE = 1,
VALUE_CODE_LITERAL,
VALUE_CODE_IS_DURATION,
VALUE_CODE_FLAG,
VALUE_CODE_TRANSLATE,
VALUE_CODE_NOT_RESERVED,
VALUE_CODE_IS_NULL,
@ -869,7 +869,7 @@ static int32_t valueNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = tlvEncodeCStr(pEncoder, VALUE_CODE_LITERAL, pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, VALUE_CODE_IS_DURATION, pNode->isDuration);
code = tlvEncodeI32(pEncoder, VALUE_CODE_FLAG, pNode->flag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, VALUE_CODE_TRANSLATE, pNode->translate);
@ -992,8 +992,8 @@ static int32_t msgToValueNode(STlvDecoder* pDecoder, void* pObj) {
case VALUE_CODE_LITERAL:
code = tlvDecodeCStrP(pTlv, &pNode->literal);
break;
case VALUE_CODE_IS_DURATION:
code = tlvDecodeBool(pTlv, &pNode->isDuration);
case VALUE_CODE_FLAG:
code = tlvDecodeI32(pTlv, &pNode->flag);
break;
case VALUE_CODE_TRANSLATE:
code = tlvDecodeBool(pTlv, &pNode->translate);
@ -1109,7 +1109,9 @@ enum {
FUNCTION_CODE_PARAMETERS,
FUNCTION_CODE_UDF_BUF_SIZE,
FUNCTION_NODE_HAS_PK,
FUNCTION_NODE_PK_BYTES
FUNCTION_NODE_PK_BYTES,
FUNCTION_CODE_IS_MERGE_FUNC,
FUNCTION_CODE_MERGE_FUNC_OF,
};
static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -1137,6 +1139,13 @@ static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, FUNCTION_NODE_PK_BYTES, pNode->pkBytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, FUNCTION_CODE_IS_MERGE_FUNC, pNode->hasOriginalFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, FUNCTION_CODE_MERGE_FUNC_OF, pNode->originalFuncId);
}
return code;
}
@ -1171,6 +1180,12 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) {
case FUNCTION_NODE_PK_BYTES:
code = tlvDecodeI32(pTlv, &pNode->pkBytes);
break;
case FUNCTION_CODE_IS_MERGE_FUNC:
code = tlvDecodeBool(pTlv, &pNode->hasOriginalFunc);
break;
case FUNCTION_CODE_MERGE_FUNC_OF:
code = tlvDecodeI32(pTlv, &pNode->originalFuncId);
break;
default:
break;
}
@ -1879,6 +1894,41 @@ static int32_t msgToCaseWhenNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum { WINDOW_OFFSET_CODE_START_OFFSET = 1, WINDOW_OFFSET_CODE_END_OFFSET };
static int32_t windowOffsetNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, WINDOW_OFFSET_CODE_START_OFFSET, nodeToMsg, pNode->pStartOffset);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, WINDOW_OFFSET_CODE_END_OFFSET, nodeToMsg, pNode->pEndOffset);
}
return code;
}
static int32_t msgToWindowOffsetNode(STlvDecoder* pDecoder, void* pObj) {
SWindowOffsetNode* pNode = (SWindowOffsetNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case WINDOW_OFFSET_CODE_START_OFFSET:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartOffset);
break;
case WINDOW_OFFSET_CODE_END_OFFSET:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndOffset);
break;
default:
break;
}
}
return code;
}
enum {
PHY_NODE_CODE_OUTPUT_DESC = 1,
PHY_NODE_CODE_CONDITIONS,
@ -2465,11 +2515,25 @@ static int32_t msgToPhysiProjectNode(STlvDecoder* pDecoder, void* pObj) {
enum {
PHY_SORT_MERGE_JOIN_CODE_BASE_NODE = 1,
PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE,
PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION,
PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_SUB_TYPE,
PHY_SORT_MERGE_JOIN_CODE_WINDOW_OFFSET,
PHY_SORT_MERGE_JOIN_CODE_JOIN_LIMIT,
PHY_SORT_MERGE_JOIN_CODE_ASOF_OP,
PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_EXPR,
PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_EXPR,
PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_SLOT_ID,
PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_SLOT_ID,
PHY_SORT_MERGE_JOIN_CODE_LEFT_EQ_COLS,
PHY_SORT_MERGE_JOIN_CODE_RIGHT_EQ_COLS,
PHY_SORT_MERGE_JOIN_CODE_FULL_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_TARGETS,
PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER,
PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS
PHY_SORT_MERGE_JOIN_CODE_COL_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM0,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE0,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM1,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE1,
PHY_SORT_MERGE_JOIN_CODE_SEQ_WIN_GROUP,
PHY_SORT_MERGE_JOIN_CODE_GROUP_JOIN
};
static int32_t physiMergeJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2480,17 +2544,63 @@ static int32_t physiMergeJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder)
code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION, nodeToMsg, pNode->pPrimKeyCond);
code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_SUB_TYPE, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pOtherOnCond);
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_WINDOW_OFFSET, nodeToMsg, pNode->pWindowOffset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_LIMIT, nodeToMsg, pNode->pJLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ASOF_OP, pNode->asofOpType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_EXPR, nodeToMsg, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_EXPR, nodeToMsg, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_SLOT_ID, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_SLOT_ID, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_LEFT_EQ_COLS, nodeListToMsg, pNode->pEqLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_RIGHT_EQ_COLS, nodeListToMsg, pNode->pEqRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_COL_ON_CONDITIONS, nodeToMsg, pNode->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_FULL_ON_CONDITIONS, nodeToMsg, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqCond);
code = tlvEncodeI64(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM0, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM1, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE0, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE1, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_SORT_MERGE_JOIN_CODE_SEQ_WIN_GROUP, pNode->seqWinGroup);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_SORT_MERGE_JOIN_CODE_GROUP_JOIN, pNode->grpJoin);
}
return code;
}
@ -2507,17 +2617,62 @@ static int32_t msgToPhysiMergeJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType));
break;
case PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pPrimKeyCond);
case PHY_SORT_MERGE_JOIN_CODE_SUB_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->subType, sizeof(pNode->subType));
break;
case PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOtherOnCond);
case PHY_SORT_MERGE_JOIN_CODE_WINDOW_OFFSET:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pWindowOffset);
break;
case PHY_SORT_MERGE_JOIN_CODE_JOIN_LIMIT:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pJLimit);
break;
case PHY_SORT_MERGE_JOIN_CODE_ASOF_OP:
code = tlvDecodeI32(pTlv, &pNode->asofOpType);
break;
case PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->leftPrimExpr);
break;
case PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->rightPrimExpr);
break;
case PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_SLOT_ID:
code = tlvDecodeI32(pTlv, &pNode->leftPrimSlotId);
break;
case PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_SLOT_ID:
code = tlvDecodeI32(pTlv, &pNode->rightPrimSlotId);
break;
case PHY_SORT_MERGE_JOIN_CODE_LEFT_EQ_COLS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pEqLeft);
break;
case PHY_SORT_MERGE_JOIN_CODE_RIGHT_EQ_COLS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pEqRight);
break;
case PHY_SORT_MERGE_JOIN_CODE_COL_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColOnCond);
break;
case PHY_SORT_MERGE_JOIN_CODE_FULL_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFullOnCond);
break;
case PHY_SORT_MERGE_JOIN_CODE_TARGETS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
break;
case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqCond);
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM0:
code = tlvDecodeI64(pTlv, &pNode->inputStat[0].inputRowNum);
break;
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM1:
code = tlvDecodeI64(pTlv, &pNode->inputStat[1].inputRowNum);
break;
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE0:
code = tlvDecodeI32(pTlv, &pNode->inputStat[0].inputRowSize);
break;
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE1:
code = tlvDecodeI32(pTlv, &pNode->inputStat[1].inputRowSize);
break;
case PHY_SORT_MERGE_JOIN_CODE_SEQ_WIN_GROUP:
code = tlvDecodeBool(pTlv, &pNode->seqWinGroup);
break;
case PHY_SORT_MERGE_JOIN_CODE_GROUP_JOIN:
code = tlvDecodeBool(pTlv, &pNode->grpJoin);
break;
default:
break;
@ -2530,14 +2685,25 @@ static int32_t msgToPhysiMergeJoinNode(STlvDecoder* pDecoder, void* pObj) {
enum {
PHY_HASH_JOIN_CODE_BASE_NODE = 1,
PHY_HASH_JOIN_CODE_JOIN_TYPE,
PHY_HASH_JOIN_CODE_JOIN_STYPE,
PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN,
PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN,
PHY_HASH_JOIN_CODE_LEFT_PRIM_EXPR,
PHY_HASH_JOIN_CODE_RIGHT_PRIM_EXPR,
PHY_HASH_JOIN_CODE_LEFT_PRIM_SLOTID,
PHY_HASH_JOIN_CODE_RIGHT_PRIM_SLOTID,
PHY_HASH_JOIN_CODE_TIME_RANGE_TARGET,
PHY_HASH_JOIN_CODE_ON_CONDITIONS,
PHY_HASH_JOIN_CODE_TARGETS,
PHY_HASH_JOIN_CODE_INPUT_ROW_NUM0,
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE0,
PHY_HASH_JOIN_CODE_INPUT_ROW_NUM1,
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1,
PHY_HASH_JOIN_CODE_LEFT_ON_COND,
PHY_HASH_JOIN_CODE_RIGHT_ON_COND,
PHY_HASH_JOIN_CODE_TIME_RANGE_SKEY,
PHY_HASH_JOIN_CODE_TIME_RANGE_EKEY,
};
static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2547,6 +2713,9 @@ static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_HASH_JOIN_CODE_JOIN_TYPE, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_HASH_JOIN_CODE_JOIN_STYPE, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN, nodeListToMsg, pNode->pOnLeft);
}
@ -2554,7 +2723,22 @@ static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN, nodeListToMsg, pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pFilterConditions);
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_LEFT_PRIM_EXPR, nodeToMsg, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_RIGHT_PRIM_EXPR, nodeToMsg, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_LEFT_PRIM_SLOTID, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_RIGHT_PRIM_SLOTID, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_TIME_RANGE_TARGET, pNode->timeRangeTarget);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
@ -2571,6 +2755,19 @@ static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_LEFT_ON_COND, nodeToMsg, pNode->pLeftOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_RIGHT_ON_COND, nodeToMsg, pNode->pRightOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_HASH_JOIN_CODE_TIME_RANGE_SKEY, pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_HASH_JOIN_CODE_TIME_RANGE_EKEY, pNode->timeRange.ekey);
}
return code;
}
@ -2588,14 +2785,32 @@ static int32_t msgToPhysiHashJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_HASH_JOIN_CODE_JOIN_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType));
break;
case PHY_HASH_JOIN_CODE_JOIN_STYPE:
code = tlvDecodeEnum(pTlv, &pNode->subType, sizeof(pNode->subType));
break;
case PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pOnLeft);
break;
case PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pOnRight);
break;
case PHY_HASH_JOIN_CODE_LEFT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->leftPrimExpr);
break;
case PHY_HASH_JOIN_CODE_RIGHT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->rightPrimExpr);
break;
case PHY_HASH_JOIN_CODE_LEFT_PRIM_SLOTID:
code = tlvDecodeI32(pTlv, &pNode->leftPrimSlotId);
break;
case PHY_HASH_JOIN_CODE_RIGHT_PRIM_SLOTID:
code = tlvDecodeI32(pTlv, &pNode->rightPrimSlotId);
break;
case PHY_HASH_JOIN_CODE_TIME_RANGE_TARGET:
code = tlvDecodeI32(pTlv, &pNode->timeRangeTarget);
break;
case PHY_HASH_JOIN_CODE_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFilterConditions);
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFullOnCond);
break;
case PHY_HASH_JOIN_CODE_TARGETS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
@ -2612,6 +2827,18 @@ static int32_t msgToPhysiHashJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1:
code = tlvDecodeI32(pTlv, &pNode->inputStat[1].inputRowSize);
break;
case PHY_HASH_JOIN_CODE_LEFT_ON_COND:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pLeftOnCond);
break;
case PHY_HASH_JOIN_CODE_RIGHT_ON_COND:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pRightOnCond);
break;
case PHY_HASH_JOIN_CODE_TIME_RANGE_SKEY:
code = tlvDecodeI64(pTlv, &pNode->timeRange.skey);
break;
case PHY_HASH_JOIN_CODE_TIME_RANGE_EKEY:
code = tlvDecodeI64(pTlv, &pNode->timeRange.ekey);
break;
default:
break;
}
@ -2627,7 +2854,8 @@ enum {
PHY_AGG_CODE_GROUP_KEYS,
PHY_AGG_CODE_AGG_FUNCS,
PHY_AGG_CODE_MERGE_DATA_BLOCK,
PHY_AGG_CODE_GROUP_KEY_OPTIMIZE
PHY_AGG_CODE_GROUP_KEY_OPTIMIZE,
PHY_AGG_CODE_HAS_COUNT_LIKE_FUNCS,
};
static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2649,6 +2877,9 @@ static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_GROUP_KEY_OPTIMIZE, pNode->groupKeyOptimized);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_HAS_COUNT_LIKE_FUNCS, pNode->hasCountLikeFunc);
}
return code;
}
@ -2678,6 +2909,9 @@ static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_AGG_CODE_GROUP_KEY_OPTIMIZE:
code = tlvDecodeBool(pTlv, &pNode->groupKeyOptimized);
break;
case PHY_AGG_CODE_HAS_COUNT_LIKE_FUNCS:
code = tlvDecodeBool(pTlv, &pNode->hasCountLikeFunc);
break;
default:
break;
}
@ -4222,6 +4456,9 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_CASE_WHEN:
code = caseWhenNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_WINDOW_OFFSET:
code = windowOffsetNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = physiTagScanNodeToMsg(pObj, pEncoder);
break;
@ -4381,6 +4618,9 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_CASE_WHEN:
code = msgToCaseWhenNode(pDecoder, pObj);
break;
case QUERY_NODE_WINDOW_OFFSET:
code = msgToWindowOffsetNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = msgToPhysiTagScanNode(pDecoder, pObj);
break;

View File

@ -372,6 +372,14 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit
}
break;
}
case QUERY_NODE_WINDOW_OFFSET: {
SWindowOffsetNode* pWin = (SWindowOffsetNode*)pNode;
res = rewriteExpr(&pWin->pStartOffset, order, rewriter, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = rewriteExpr(&pWin->pEndOffset, order, rewriter, pContext);
}
break;
}
case QUERY_NODE_COUNT_WINDOW: {
SCountWindowNode* pEvent = (SCountWindowNode*)pNode;
res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext);
@ -415,7 +423,7 @@ void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void*
(void)rewriteExprs(pList, TRAVERSAL_POSTORDER, rewriter, pContext);
}
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext) {
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext) {
if (NULL == pSelect) {
return;
}
@ -451,6 +459,10 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
return;
}
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext) {
nodesWalkSelectStmtImpl(pSelect, clause, walker, pContext);
}
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext) {
if (NULL == pSelect) {
return;

View File

@ -44,6 +44,74 @@ struct SNodeAllocator {
static threadlocal SNodeAllocator* g_pNodeAllocator;
static int32_t g_allocatorReqRefPool = -1;
char* getJoinTypeString(EJoinType type) {
static char* joinType[] = {"", "INNER", "LEFT", "RIGHT", "FULL"};
return joinType[type];
}
char* getJoinSTypeString(EJoinSubType type) {
static char* joinSType[] = {"", "", "OUTER", "SEMI", "ANTI", "ANY", "ASOF", "WINDOW"};
return joinSType[type];
}
char* getFullJoinTypeString(EJoinType type, EJoinSubType stype) {
static char* joinFullType[][8] = {
{"INNER", "INNER", "INNER", "INNER", "INNER", "INNER ANY", "INNER", "INNER"},
{"LEFT", "LEFT", "LEFT OUTER", "LEFT SEMI", "LEFT ANTI", "LEFT ANY", "LEFT ASOF", "LEFT WINDOW"},
{"RIGHT", "RIGHT", "RIGHT OUTER", "RIGHT SEMI", "RIGHT ANTI", "RIGHT ANY", "RIGHT ASOF", "RIGHT WINDOW"},
{"FULL", "FULL", "FULL OUTER", "FULL", "FULL", "FULL ANY", "FULL", "FULL"}
};
return joinFullType[type][stype];
}
int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc) {
if (NULL == *ppSrc) {
return TSDB_CODE_SUCCESS;
}
if (NULL == *ppDst) {
*ppDst = *ppSrc;
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc) && ((SLogicConditionNode*)(*ppSrc))->condType == LOGIC_COND_TYPE_AND) {
TSWAP(*ppDst, *ppSrc);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppDst)) {
SLogicConditionNode* pDst = (SLogicConditionNode*)*ppDst;
if (pDst->condType == LOGIC_COND_TYPE_AND) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc) && ((SLogicConditionNode*)(*ppSrc))->condType == LOGIC_COND_TYPE_AND) {
nodesListStrictAppendList(pDst->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList);
((SLogicConditionNode*)(*ppSrc))->pParameterList = NULL;
} else {
nodesListStrictAppend(pDst->pParameterList, *ppSrc);
*ppSrc = NULL;
}
nodesDestroyNode(*ppSrc);
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
}
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
pLogicCond->condType = LOGIC_COND_TYPE_AND;
pLogicCond->pParameterList = nodesMakeList();
nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc);
nodesListStrictAppend(pLogicCond->pParameterList, *ppDst);
*ppDst = (SNode*)pLogicCond;
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) {
SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize);
if (NULL == pNewChunk) {
@ -308,6 +376,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SHintNode));
case QUERY_NODE_VIEW:
return makeNode(type, sizeof(SViewNode));
case QUERY_NODE_WINDOW_OFFSET:
return makeNode(type, sizeof(SWindowOffsetNode));
case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT:
@ -450,6 +520,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_ENCRYPTIONS_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT:
return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return makeNode(type, sizeof(SShowTableTagsStmt));
@ -492,6 +563,12 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateViewStmt));
case QUERY_NODE_DROP_VIEW_STMT:
return makeNode(type, sizeof(SDropViewStmt));
case QUERY_NODE_CREATE_TSMA_STMT:
return makeNode(type, sizeof(SCreateTSMAStmt));
case QUERY_NODE_DROP_TSMA_STMT:
return makeNode(type, sizeof(SDropTSMAStmt));
case QUERY_NODE_TSMA_OPTIONS:
return makeNode(type, sizeof(STSMAOptions));
case QUERY_NODE_LOGIC_PLAN_SCAN:
return makeNode(type, sizeof(SScanLogicNode));
case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -740,6 +817,8 @@ void nodesDestroyNode(SNode* pNode) {
taosMemoryFreeClear(pReal->pMeta);
taosMemoryFreeClear(pReal->pVgroupList);
taosArrayDestroyEx(pReal->pSmaIndexes, destroySmaIndex);
taosArrayDestroyP(pReal->tsmaTargetTbVgInfo, taosMemoryFree);
taosArrayDestroy(pReal->tsmaTargetTbInfo);
break;
}
case QUERY_NODE_TEMP_TABLE:
@ -747,6 +826,9 @@ void nodesDestroyNode(SNode* pNode) {
break;
case QUERY_NODE_JOIN_TABLE: {
SJoinTableNode* pJoin = (SJoinTableNode*)pNode;
nodesDestroyNode(pJoin->pWindowOffset);
nodesDestroyNode(pJoin->pJLimit);
nodesDestroyNode(pJoin->addPrimCond);
nodesDestroyNode(pJoin->pLeft);
nodesDestroyNode(pJoin->pRight);
nodesDestroyNode(pJoin->pOnCond);
@ -838,6 +920,12 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pOptions->pDeleteMark);
break;
}
case QUERY_NODE_TSMA_OPTIONS: {
STSMAOptions* pOptions = (STSMAOptions*)pNode;
nodesDestroyList(pOptions->pFuncs);
nodesDestroyNode(pOptions->pInterval);
break;
}
case QUERY_NODE_LEFT_VALUE: // no pointer field
case QUERY_NODE_COLUMN_REF: // no pointer field
break;
@ -878,6 +966,12 @@ void nodesDestroyNode(SNode* pNode) {
taosArrayDestroyEx(pView->pSmaIndexes, destroySmaIndex);
break;
}
case QUERY_NODE_WINDOW_OFFSET: {
SWindowOffsetNode* pWin = (SWindowOffsetNode*)pNode;
nodesDestroyNode(pWin->pStartOffset);
nodesDestroyNode(pWin->pEndOffset);
break;
}
case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList);
@ -1106,7 +1200,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_VIEWS_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: {
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName);
nodesDestroyNode(pStmt->pTbName);
@ -1193,6 +1288,15 @@ void nodesDestroyNode(SNode* pNode) {
}
case QUERY_NODE_DROP_VIEW_STMT:
break;
case QUERY_NODE_CREATE_TSMA_STMT: {
SCreateTSMAStmt* pStmt = (SCreateTSMAStmt*)pNode;
nodesDestroyNode((SNode*)pStmt->pOptions);
if (pStmt->pReq) {
tFreeSMCreateSmaReq(pStmt->pReq);
taosMemoryFreeClear(pStmt->pReq);
}
break;
}
case QUERY_NODE_LOGIC_PLAN_SCAN: {
SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
@ -1207,14 +1311,26 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pLogicNode->pTags);
nodesDestroyNode(pLogicNode->pSubtable);
taosArrayDestroyEx(pLogicNode->pFuncTypes, destroyFuncParam);
taosArrayDestroyP(pLogicNode->pTsmaTargetTbVgInfo, taosMemoryFree);
taosArrayDestroy(pLogicNode->pTsmaTargetTbInfo);
break;
}
case QUERY_NODE_LOGIC_PLAN_JOIN: {
SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyNode(pLogicNode->pWindowOffset);
nodesDestroyNode(pLogicNode->pJLimit);
nodesDestroyNode(pLogicNode->addPrimEqCond);
nodesDestroyNode(pLogicNode->pPrimKeyEqCond);
nodesDestroyNode(pLogicNode->pOtherOnCond);
nodesDestroyNode(pLogicNode->pColEqCond);
nodesDestroyNode(pLogicNode->pColOnCond);
nodesDestroyNode(pLogicNode->pTagEqCond);
nodesDestroyNode(pLogicNode->pTagOnCond);
nodesDestroyNode(pLogicNode->pFullOnCond);
nodesDestroyList(pLogicNode->pLeftEqNodes);
nodesDestroyList(pLogicNode->pRightEqNodes);
nodesDestroyNode(pLogicNode->pLeftOnCond);
nodesDestroyNode(pLogicNode->pRightOnCond);
break;
}
case QUERY_NODE_LOGIC_PLAN_AGG: {
@ -1358,10 +1474,17 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode* pPhyNode = (SSortMergeJoinPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pWindowOffset);
nodesDestroyNode(pPhyNode->pJLimit);
nodesDestroyNode(pPhyNode->leftPrimExpr);
nodesDestroyNode(pPhyNode->rightPrimExpr);
nodesDestroyList(pPhyNode->pEqLeft);
nodesDestroyList(pPhyNode->pEqRight);
nodesDestroyNode(pPhyNode->pPrimKeyCond);
nodesDestroyNode(pPhyNode->pOtherOnCond);
nodesDestroyNode(pPhyNode->pFullOnCond);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pColEqCond);
nodesDestroyNode(pPhyNode->pColOnCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: {
@ -1369,12 +1492,17 @@ void nodesDestroyNode(SNode* pNode) {
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pOnLeft);
nodesDestroyList(pPhyNode->pOnRight);
nodesDestroyNode(pPhyNode->pFilterConditions);
nodesDestroyNode(pPhyNode->leftPrimExpr);
nodesDestroyNode(pPhyNode->rightPrimExpr);
nodesDestroyNode(pPhyNode->pFullOnCond);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pPrimKeyCond);
nodesDestroyNode(pPhyNode->pColEqCond);
nodesDestroyNode(pPhyNode->pTagEqCond);
nodesDestroyNode(pPhyNode->pLeftOnCond);
nodesDestroyNode(pPhyNode->pRightOnCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
@ -1640,6 +1768,16 @@ int32_t nodesListMakeStrictAppendList(SNodeList** pTarget, SNodeList* pSrc) {
return nodesListStrictAppendList(*pTarget, pSrc);
}
int32_t nodesListMakePushFront(SNodeList** pList, SNode* pNode) {
if (*pList == NULL) {
*pList = nodesMakeList();
if (*pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return nodesListPushFront(*pList, pNode);
}
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) {
if (NULL == pList || NULL == pNode) {
@ -1656,6 +1794,7 @@ int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) {
p->pNext = pList->pHead;
}
pList->pHead = p;
pList->pTail = pList->pTail ? pList->pTail : p;
++(pList->length);
return TSDB_CODE_SUCCESS;
}
@ -2026,6 +2165,7 @@ bool nodesIsBitwiseOp(const SOperatorNode* pOp) {
typedef struct SCollectColumnsCxt {
int32_t errCode;
const char* pTableAlias;
SSHashObj* pMultiTableAlias;
ECollectColType collectType;
SNodeList* pCols;
SHashObj* pColHash;
@ -2067,6 +2207,19 @@ static EDealRes collectColumns(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static EDealRes collectColumnsExt(SNode* pNode, void* pContext) {
SCollectColumnsCxt* pCxt = (SCollectColumnsCxt*)pContext;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (isCollectType(pCxt->collectType, pCol->colType) && 0 != strcmp(pCol->colName, "*") &&
(NULL == pCxt->pMultiTableAlias || NULL != (pCxt->pTableAlias = tSimpleHashGet(pCxt->pMultiTableAlias, pCol->tableAlias, strlen(pCol->tableAlias))))) {
return doCollect(pCxt, pCol, pNode);
}
}
return DEAL_RES_CONTINUE;
}
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols) {
if (NULL == pSelect || NULL == pCols) {
@ -2098,6 +2251,38 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
return TSDB_CODE_SUCCESS;
}
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols) {
if (NULL == pSelect || NULL == pCols) {
return TSDB_CODE_FAILED;
}
SCollectColumnsCxt cxt = {
.errCode = TSDB_CODE_SUCCESS,
.pTableAlias = NULL,
.pMultiTableAlias = pMultiTableAlias,
.collectType = type,
.pCols = (NULL == *pCols ? nodesMakeList() : *pCols),
.pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)};
if (NULL == cxt.pCols || NULL == cxt.pColHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
*pCols = NULL;
nodesWalkSelectStmtImpl(pSelect, clause, collectColumnsExt, &cxt);
taosHashCleanup(cxt.pColHash);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pCols);
return cxt.errCode;
}
if (LIST_LENGTH(cxt.pCols) > 0) {
*pCols = cxt.pCols;
} else {
nodesDestroyList(cxt.pCols);
}
return TSDB_CODE_SUCCESS;
}
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols) {
if (NULL == pCols) {
return TSDB_CODE_FAILED;
@ -2438,7 +2623,6 @@ SValueNode* nodesMakeValueNodeFromString(char* literal) {
pValNode->datum.p = p;
pValNode->literal = tstrdup(literal);
pValNode->translate = true;
pValNode->isDuration = false;
pValNode->isNull = false;
}
return pValNode;
@ -2451,7 +2635,6 @@ SValueNode* nodesMakeValueNodeFromBool(bool b) {
pValNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
nodesSetValueNodeValue(pValNode, &b);
pValNode->translate = true;
pValNode->isDuration = false;
pValNode->isNull = false;
}
return pValNode;
@ -2466,3 +2649,70 @@ bool nodesIsTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
void nodesSortList(SNodeList** pList, int32_t (*comp)(SNode* pNode1, SNode* pNode2)) {
if ((*pList)->length == 1) return;
uint32_t inSize = 1;
SListCell* pHead = (*pList)->pHead;
while (1) {
SListCell* p = pHead;
pHead = NULL;
SListCell* pTail = NULL;
uint32_t nMerges = 0;
while (p) {
++nMerges;
SListCell* q = p;
uint32_t pSize = 0;
for (uint32_t i = 0; i < inSize; ++i) {
++pSize;
q = q->pNext;
if (!q) {
break;
}
}
uint32_t qSize = inSize;
while (pSize > 0 || (qSize > 0 && q)) {
SListCell* pCell;
if (pSize == 0) {
pCell = q;
q = q->pNext;
--qSize;
} else if (qSize == 0 || !q) {
pCell = p;
p = p->pNext;
--pSize;
} else if (comp(q->pNode, p->pNode) >= 0) {
pCell = p;
p = p->pNext;
--pSize;
} else {
pCell = q;
q = q->pNext;
--qSize;
}
if (pTail) {
pTail->pNext = pCell;
pCell->pPrev = pTail;
} else {
pHead = pCell;
pHead->pPrev = NULL;
}
pTail = pCell;
}
p = q;
}
pTail->pNext = NULL;
if (nMerges <= 1) {
(*pList)->pHead = pHead;
(*pList)->pTail = pTail;
return;
}
inSize *= 2;
}
}

View File

@ -56,6 +56,69 @@ TEST(NodesTest, traverseTest) {
nodesDestroyNode(pRoot);
}
int32_t compareValueNode(SNode* pNode1, SNode* pNode2) {
SValueNode* p1 = (SValueNode*)pNode1;
SValueNode* p2 = (SValueNode*)pNode2;
if (p1->datum.i < p2->datum.i)
return -1;
else if (p1->datum.i > p2->datum.i)
return 1;
else
return 0;
}
void assert_sort_result(SNodeList* pList) {
SNode* pNode;
int32_t i = 0;
FOREACH(pNode, pList) {
SValueNode* p = (SValueNode*)pNode;
ASSERT_EQ(p->datum.i, i++);
}
SListCell* pCell = pList->pHead;
ASSERT_TRUE(pCell->pPrev == NULL);
ASSERT_TRUE(pList->pTail->pNext == NULL);
int32_t len = 1;
while (pCell) {
if (pCell->pNext) {
ASSERT_TRUE(pCell->pNext->pPrev == pCell);
}
pCell = pCell->pNext;
if (pCell) len++;
}
ASSERT_EQ(len, pList->length);
}
TEST(NodesTest, sort) {
SValueNode *vn1 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn1->datum.i = 4;
SValueNode *vn2 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn2->datum.i = 3;
SValueNode *vn3 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn3->datum.i = 2;
SValueNode *vn4 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn4->datum.i = 1;
SValueNode *vn5 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn5->datum.i = 0;
SNodeList* l = NULL;
nodesListMakeAppend(&l, (SNode*)vn1);
nodesListMakeAppend(&l, (SNode*)vn2);
nodesListMakeAppend(&l, (SNode*)vn3);
nodesListMakeAppend(&l, (SNode*)vn4);
nodesListMakeAppend(&l, (SNode*)vn5);
nodesSortList(&l, compareValueNode);
assert_sort_result(l);
nodesDestroyList(l);
}
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

View File

@ -115,37 +115,39 @@ SNode* createValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken*
SNode* createRawValueNode(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pNode);
SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const SToken* pLiteral, SNode *pLeft, SNode *pRight);
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
SNode* createLogicConditionNode(SAstCreateContext* pCxt, ELogicConditionType type, SNode* pParam1, SNode* pParam2);
SNode* createOperatorNode(SAstCreateContext* pCxt, EOperatorType type, SNode* pLeft, SNode* pRight);
SNode* createBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createNotBetweenAnd(SAstCreateContext* pCxt, SNode* pExpr, SNode* pLeft, SNode* pRight);
SNode* createFunctionNode(SAstCreateContext* pCxt, const SToken* pFuncName, SNodeList* pParameterList);
SNode* createCastFunctionNode(SAstCreateContext* pCxt, SNode* pExpr, SDataType dt);
SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
SNode* createSessionWindowNode(SAstCreateContext* pCxt, SNode* pCol, SNode* pGap);
SNode* createStateWindowNode(SAstCreateContext* pCxt, SNode* pExpr);
SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNode* pEndCond);
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
SNode* createInterpTimePoint(SAstCreateContext* pCxt, SNode* pPoint);
SNode* createWhenThenNode(SAstCreateContext* pCxt, SNode* pWhen, SNode* pThen);
SNode* createCaseWhenNode(SAstCreateContext* pCxt, SNode* pCase, SNodeList* pWhenThenList, SNode* pElse);
SNode* addWhereClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pWhere);
SNode* addPartitionByClause(SAstCreateContext* pCxt, SNode* pStmt, SNodeList* pPartitionByList);
@ -158,8 +160,9 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
SNodeList* pHint);
SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit);
SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinOffset);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
@ -273,6 +276,15 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie
SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView);
SNode* createShowCompactDetailsStmt(SAstCreateContext* pCxt, SNode* pCompactIdNode);
SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type);
SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* tsmaName, SNode* pOptions,
SNode* pRealTable, SNode* pInterval);
SNode* createTSMAOptions(SAstCreateContext* pCxt, SNodeList* pFuncs);
SNode* createDefaultTSMAOptions(SAstCreateContext* pCxt);
SNode* createDropTSMAStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createShowCreateTSMAStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createShowTSMASStmt(SAstCreateContext* pCxt, SNode* dbName);
#ifdef __cplusplus
}
#endif

View File

@ -37,8 +37,9 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placeholderNo, SArray** pPlaceholderValues);
int32_t translateTable(STranslateContext* pCxt, SNode** pTable);
int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinParent);
int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput);
void tfreeSParseQueryRes(void* p);

Some files were not shown because too many files have changed in this diff Show More