xxq250
/
homework-jianmu
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Merge branch '3.0' into fix/3_liaohj

This commit is contained in:
Haojun Liao 2024-04-17 11:27:54 +08:00
parent 13e14d0f55 c43c0cf427
commit 24c000e052
182 changed files with 39949 additions and 7391 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Jenkinsfile2
View File

@ -454,7 +454,7 @@ pipeline {
cd ${WKC}/tests/parallel_test
export DEFAULT_RETRY_TIME=2
date
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 600 ''' + extra_param + '''
''' + timeout_cmd + ''' time ./run.sh -e -m /home/m.json -t cases.task -b ${BRANCH_NAME}_${BUILD_ID} -l ${WKDIR}/log -o 900 ''' + extra_param + '''
'''
}
}

1
include/common/systable.h
View File

@ -56,6 +56,7 @@ extern "C" {
#define TSDB_INS_TABLE_GRANTS_FULL "ins_grants_full"
#define TSDB_INS_TABLE_GRANTS_LOGS "ins_grants_logs"
#define TSDB_INS_TABLE_MACHINES "ins_machines"
#define TSDB_INS_TABLE_TSMAS "ins_tsmas"
#define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema"
#define TSDB_PERFS_TABLE_SMAS "perf_smas"

12
include/common/tcommon.h
View File

@ -414,6 +414,18 @@ typedef struct STUidTagInfo {
int32_t taosGenCrashJsonMsg(int signum, char **pMsg, int64_t clusterId, int64_t startTime);
#define TSMA_RES_STB_POSTFIX "_tsma_res_stb_"
#define MD5_OUTPUT_LEN 32
#define TSMA_RES_STB_EXTRA_COLUMN_NUM 4 // 3 columns: _wstart, _wend, _wduration, 1 tag: tbname
static inline bool isTsmaResSTb(const char* stbName) {
const char* pos = strstr(stbName, TSMA_RES_STB_POSTFIX);
if (pos && strlen(stbName) == (pos - stbName) + strlen(TSMA_RES_STB_POSTFIX)) {
return true;
}
return false;
}
#ifdef __cplusplus
}
#endif

8
include/common/tdatablock.h
View File

@ -37,6 +37,7 @@ typedef struct SBlockOrderInfo {
#define NBIT (3u)
#define BitPos(_n) ((_n) & ((1 << NBIT) - 1))
#define CharPos(r_) ((r_) >> NBIT)
#define BMCharPos(bm_, r_) ((bm_)[(r_) >> NBIT])
#define colDataIsNull_f(bm_, r_) ((BMCharPos(bm_, r_) & (1u << (7u - BitPos(r_)))) == (1u << (7u - BitPos(r_))))
@ -192,12 +193,14 @@ int32_t getJsonValueLen(const char* data);
int32_t colDataSetVal(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, bool isNull);
int32_t colDataReassignVal(SColumnInfoData* pColumnInfoData, uint32_t dstRowIdx, uint32_t srcRowIdx, const char* pData);
int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t rowIndex, const char* pData, uint32_t numOfRows, bool trimValue);
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows);
int32_t colDataCopyNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData,
uint32_t numOfRows, bool isNull);
int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity,
const SColumnInfoData* pSource, int32_t numOfRow2);
int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows,
const SDataBlockInfo* pBlockInfo);
int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows);
int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex);
int32_t blockDataUpdatePkRange(SSDataBlock* pDataBlock, int32_t pkColumnIndex, bool asc);
@ -210,6 +213,8 @@ size_t blockDataGetNumOfCols(const SSDataBlock* pBlock);
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock);
int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc);
int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows);
void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows);
int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startIndex, int32_t* stopIndex,
int32_t pageSize);
int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock);
@ -234,7 +239,10 @@ int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows);
void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows);
void blockDataCleanup(SSDataBlock* pDataBlock);
void blockDataReset(SSDataBlock* pDataBlock);
void blockDataEmpty(SSDataBlock* pDataBlock);
int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
bool clearPayload);
size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize, int32_t extraSize);

2
include/common/tglobal.h
View File

@ -201,6 +201,8 @@ extern char tsSmlTsDefaultName[];
extern int32_t tmqMaxTopicNum;
extern int32_t tmqRowSize;
extern int32_t tsMaxTsmaNum;
extern int32_t tsMaxTsmaCalcDelay;
// wal
extern int64_t tsWalFsyncDataSizeLimit;

121
include/common/tmsg.h
View File

@ -110,6 +110,7 @@ enum {
HEARTBEAT_KEY_TMQ,
HEARTBEAT_KEY_DYN_VIEW,
HEARTBEAT_KEY_VIEWINFO,
HEARTBEAT_KEY_TSMA,
};
typedef enum _mgmt_table {
@ -145,6 +146,7 @@ typedef enum _mgmt_table {
TSDB_MGMT_TABLE_STREAM_TASKS,
TSDB_MGMT_TABLE_PRIVILEGES,
TSDB_MGMT_TABLE_VIEWS,
TSDB_MGMT_TABLE_TSMAS,
TSDB_MGMT_TABLE_COMPACT,
TSDB_MGMT_TABLE_COMPACT_DETAIL,
TSDB_MGMT_TABLE_GRANTS_FULL,
@ -245,7 +247,9 @@ typedef enum ENodeType {
QUERY_NODE_EVENT_WINDOW,
QUERY_NODE_HINT,
QUERY_NODE_VIEW,
QUERY_NODE_WINDOW_OFFSET,
QUERY_NODE_COUNT_WINDOW,
QUERY_NODE_TSMA_OPTIONS,
// Statement nodes are used in parser and planner module.
QUERY_NODE_SET_OPERATOR = 100,
@ -367,6 +371,10 @@ typedef enum ENodeType {
QUERY_NODE_SHOW_GRANTS_FULL_STMT,
QUERY_NODE_SHOW_GRANTS_LOGS_STMT,
QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT,
QUERY_NODE_SHOW_TSMAS_STMT,
QUERY_NODE_CREATE_TSMA_STMT,
QUERY_NODE_SHOW_CREATE_TSMA_STMT,
QUERY_NODE_DROP_TSMA_STMT,
// logic plan node
QUERY_NODE_LOGIC_PLAN_SCAN = 1000,
@ -440,8 +448,8 @@ typedef enum ENodeType {
typedef struct {
int32_t vgId;
char* dbFName;
char* tbName;
const char* dbFName;
const char* tbName;
} SBuildTableInput;
typedef struct {
@ -1395,9 +1403,15 @@ int32_t tSerializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp);
int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp);
void tFreeSDnodeListRsp(SDnodeListRsp* pRsp);
typedef struct {
SArray* pTsmas; // SArray<STableTSMAInfo*>
} STableTSMAInfoRsp;
typedef struct {
SUseDbRsp* useDbRsp;
SDbCfgRsp* cfgRsp;
STableTSMAInfoRsp* pTsmaRsp;
int32_t dbTsmaVersion;
} SDbHbRsp;
typedef struct {
@ -2596,6 +2610,7 @@ typedef struct {
SArray* pVgroupVerList;
// 3.3.0.0
SArray* pCols; // array of SField
int64_t smaId;
} SCMCreateStreamReq;
typedef struct {
@ -3560,6 +3575,10 @@ typedef struct {
char* ast;
int64_t deleteMark;
int64_t lastTs;
int64_t normSourceTbUid; // the Uid of source tb if its a normal table, otherwise 0
SArray* pVgroupVerList;
int8_t recursiveTsma;
char baseTsmaName[TSDB_TABLE_FNAME_LEN]; // base tsma name for recursively created tsma
} SMCreateSmaReq;
int32_t tSerializeSMCreateSmaReq(void* buf, int32_t bufLen, SMCreateSmaReq* pReq);
@ -4183,6 +4202,104 @@ typedef struct {
int32_t tSerializeSViewMetaRsp(void* buf, int32_t bufLen, const SViewMetaRsp* pRsp);
int32_t tDeserializeSViewMetaRsp(void* buf, int32_t bufLen, SViewMetaRsp* pRsp);
void tFreeSViewMetaRsp(SViewMetaRsp* pRsp);
typedef struct {
char name[TSDB_TABLE_FNAME_LEN]; // table name or tsma name
bool fetchingWithTsmaName; // if we are fetching with tsma name
}STableTSMAInfoReq;
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq);
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq);
typedef struct {
int32_t funcId;
col_id_t colId;
} STableTSMAFuncInfo;
typedef struct {
char name[TSDB_TABLE_NAME_LEN];
uint64_t tsmaId;
char targetTb[TSDB_TABLE_NAME_LEN];
char targetDbFName[TSDB_DB_FNAME_LEN];
char tb[TSDB_TABLE_NAME_LEN];
char dbFName[TSDB_DB_FNAME_LEN];
uint64_t suid;
uint64_t destTbUid;
uint64_t dbId;
int32_t version;
int64_t interval;
int8_t unit;
SArray* pFuncs; // SArray<STableTSMAFuncInfo>
SArray* pTags; // SArray<SSchema>
SArray* pUsedCols; // SArray<SSchema>
char* ast;
int64_t streamUid;
int64_t reqTs;
int64_t rspTs;
int64_t delayDuration; // ms
bool fillHistoryFinished;
} STableTSMAInfo;
int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAInfoRsp* pRsp);
int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRsp* pRsp);
int32_t tCloneTbTSMAInfo(STableTSMAInfo* pInfo, STableTSMAInfo** pRes);
void tFreeTableTSMAInfo(void* p);
void tFreeAndClearTableTSMAInfo(void* p);
void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp* pRsp);
#define STSMAHbRsp STableTSMAInfoRsp
#define tSerializeTSMAHbRsp tSerializeTableTSMAInfoRsp
#define tDeserializeTSMAHbRsp tDeserializeTableTSMAInfoRsp
#define tFreeTSMAHbRsp tFreeTableTSMAInfoRsp
typedef struct SStreamProgressReq {
int64_t streamId;
int32_t vgId;
int32_t fetchIdx;
int32_t subFetchIdx;
} SStreamProgressReq;
int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProgressReq* pReq);
int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgressReq* pReq);
typedef struct SStreamProgressRsp {
int64_t streamId;
int32_t vgId;
bool fillHisFinished;
int64_t progressDelay;
int32_t fetchIdx;
int32_t subFetchIdx;
} SStreamProgressRsp;
int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProgressRsp* pRsp);
int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgressRsp* pRsp);
typedef struct SDropCtbWithTsmaSingleVgReq {
SVgroupInfo vgInfo;
SArray* pTbs; // SVDropTbReq
} SMDropTbReqsOnSingleVg;
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder* pEncoder, const SMDropTbReqsOnSingleVg* pReq);
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* pReq);
void tFreeSMDropTbReqOnSingleVg(void* p);
typedef struct SDropTbsReq {
SArray* pVgReqs; // SMDropTbReqsOnSingleVg
} SMDropTbsReq;
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq);
int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq);
void tFreeSMDropTbsReq(void*);
typedef struct SVFetchTtlExpiredTbsRsp {
SArray* pExpiredTbs; // SVDropTbReq
int32_t vgId;
} SVFetchTtlExpiredTbsRsp;
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredTbsRsp* pRsp);
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp* pRsp);
void tFreeFetchTtlExpiredTbsRsp(void* p);
#pragma pack(pop)

10
include/common/tmsgdef.h
View File

@ -223,6 +223,12 @@
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB, "s3migrate-db", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_S3MIGRATE_DB_TIMER, "s3migrate-db-tmr", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_MAX_MSG, "mnd-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TSMA, "create-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TSMA, "drop-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_STB_DROP, "drop-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_TSMA, "get-table-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_GET_TSMA, "get-tsma", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_MND_DROP_TB_WITH_TSMA, "drop-tb-with-tsma", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_MND_MSG)
TD_NEW_MSG_SEG(TDMT_VND_MSG) // 2<<8
@ -267,6 +273,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "vnode-compact", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "vnode-drop-ttl-stb", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_FETCH_TTL_EXPIRED_TBS, "vnode-fetch-ttl-expired-tbs", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_TRIM, "vnode-trim", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_COMMIT, "vnode-commit", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_CREATE_INDEX, "vnode-create-index", NULL, NULL)
@ -310,6 +317,8 @@
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RESUME, "stream-task-resume", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_STOP, "stream-task-stop", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_MAX_MSG, "stream-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_CREATE, "stream-create", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_STREAM_DROP, "stream-drop", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_MON_MSG) //5 << 8
@ -357,6 +366,7 @@
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_RESET, "vnode-stream-reset", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_CHECK, "vnode-stream-task-check", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_STREAM_MAX_MSG, "vnd-stream-max", NULL, NULL)
TD_DEF_MSG_TYPE(TDMT_VND_GET_STREAM_PROGRESS, "vnd-stream-progress", NULL, NULL)
TD_CLOSE_MSG_SEG(TDMT_END_VND_STREAM_MSG)
TD_NEW_MSG_SEG(TDMT_VND_TMQ_MSG) //8 << 8

3
include/common/ttime.h
View File

@ -79,7 +79,7 @@ int64_t taosTimeGetIntervalEnd(int64_t ts, const SInterval* pInterval);
int32_t taosTimeCountIntervalForFill(int64_t skey, int64_t ekey, int64_t interval, char unit, int32_t precision, int32_t order);
int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* ts, char* unit, int32_t timePrecision);
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision);
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow);
int32_t taosParseTime(const char* timestr, int64_t* pTime, int32_t len, int32_t timePrec, int8_t dayligth);
void deltaToUtcInitOnce();
@ -88,6 +88,7 @@ char getPrecisionUnit(int32_t precision);
int64_t convertTimePrecision(int64_t ts, int32_t fromPrecision, int32_t toPrecision);
int64_t convertTimeFromPrecisionToUnit(int64_t ts, int32_t fromPrecision, char toUnit);
int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec, int64_t* timeVal);
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision);
void taosFormatUtcTime(char* buf, int32_t bufLen, int64_t ts, int32_t precision);

350
include/common/ttokendef.h
View File

@ -206,175 +206,185 @@
#define TK_QTAGS 188
#define TK_AS 189
#define TK_SYSTEM 190
#define TK_INDEX 191
#define TK_FUNCTION 192
#define TK_INTERVAL 193
#define TK_COUNT 194
#define TK_LAST_ROW 195
#define TK_META 196
#define TK_ONLY 197
#define TK_TOPIC 198
#define TK_CONSUMER 199
#define TK_GROUP 200
#define TK_DESC 201
#define TK_DESCRIBE 202
#define TK_RESET 203
#define TK_QUERY 204
#define TK_CACHE 205
#define TK_EXPLAIN 206
#define TK_ANALYZE 207
#define TK_VERBOSE 208
#define TK_NK_BOOL 209
#define TK_RATIO 210
#define TK_NK_FLOAT 211
#define TK_OUTPUTTYPE 212
#define TK_AGGREGATE 213
#define TK_BUFSIZE 214
#define TK_LANGUAGE 215
#define TK_REPLACE 216
#define TK_STREAM 217
#define TK_INTO 218
#define TK_PAUSE 219
#define TK_RESUME 220
#define TK_TRIGGER 221
#define TK_AT_ONCE 222
#define TK_WINDOW_CLOSE 223
#define TK_IGNORE 224
#define TK_EXPIRED 225
#define TK_FILL_HISTORY 226
#define TK_UPDATE 227
#define TK_SUBTABLE 228
#define TK_UNTREATED 229
#define TK_KILL 230
#define TK_CONNECTION 231
#define TK_TRANSACTION 232
#define TK_BALANCE 233
#define TK_VGROUP 234
#define TK_LEADER 235
#define TK_MERGE 236
#define TK_REDISTRIBUTE 237
#define TK_SPLIT 238
#define TK_DELETE 239
#define TK_INSERT 240
#define TK_NK_BIN 241
#define TK_NK_HEX 242
#define TK_NULL 243
#define TK_NK_QUESTION 244
#define TK_NK_ALIAS 245
#define TK_NK_ARROW 246
#define TK_ROWTS 247
#define TK_QSTART 248
#define TK_QEND 249
#define TK_QDURATION 250
#define TK_WSTART 251
#define TK_WEND 252
#define TK_WDURATION 253
#define TK_IROWTS 254
#define TK_ISFILLED 255
#define TK_CAST 256
#define TK_NOW 257
#define TK_TODAY 258
#define TK_TIMEZONE 259
#define TK_CLIENT_VERSION 260
#define TK_SERVER_VERSION 261
#define TK_SERVER_STATUS 262
#define TK_CURRENT_USER 263
#define TK_CASE 264
#define TK_WHEN 265
#define TK_THEN 266
#define TK_ELSE 267
#define TK_BETWEEN 268
#define TK_IS 269
#define TK_NK_LT 270
#define TK_NK_GT 271
#define TK_NK_LE 272
#define TK_NK_GE 273
#define TK_NK_NE 274
#define TK_MATCH 275
#define TK_NMATCH 276
#define TK_CONTAINS 277
#define TK_IN 278
#define TK_JOIN 279
#define TK_INNER 280
#define TK_SELECT 281
#define TK_NK_HINT 282
#define TK_DISTINCT 283
#define TK_WHERE 284
#define TK_PARTITION 285
#define TK_BY 286
#define TK_SESSION 287
#define TK_STATE_WINDOW 288
#define TK_EVENT_WINDOW 289
#define TK_COUNT_WINDOW 290
#define TK_SLIDING 291
#define TK_FILL 292
#define TK_VALUE 293
#define TK_VALUE_F 294
#define TK_NONE 295
#define TK_PREV 296
#define TK_NULL_F 297
#define TK_LINEAR 298
#define TK_NEXT 299
#define TK_HAVING 300
#define TK_RANGE 301
#define TK_EVERY 302
#define TK_ORDER 303
#define TK_SLIMIT 304
#define TK_SOFFSET 305
#define TK_LIMIT 306
#define TK_OFFSET 307
#define TK_ASC 308
#define TK_NULLS 309
#define TK_ABORT 310
#define TK_AFTER 311
#define TK_ATTACH 312
#define TK_BEFORE 313
#define TK_BEGIN 314
#define TK_BITAND 315
#define TK_BITNOT 316
#define TK_BITOR 317
#define TK_BLOCKS 318
#define TK_CHANGE 319
#define TK_COMMA 320
#define TK_CONCAT 321
#define TK_CONFLICT 322
#define TK_COPY 323
#define TK_DEFERRED 324
#define TK_DELIMITERS 325
#define TK_DETACH 326
#define TK_DIVIDE 327
#define TK_DOT 328
#define TK_EACH 329
#define TK_FAIL 330
#define TK_FILE 331
#define TK_FOR 332
#define TK_GLOB 333
#define TK_ID 334
#define TK_IMMEDIATE 335
#define TK_IMPORT 336
#define TK_INITIALLY 337
#define TK_INSTEAD 338
#define TK_ISNULL 339
#define TK_MODULES 340
#define TK_NK_BITNOT 341
#define TK_NK_SEMI 342
#define TK_NOTNULL 343
#define TK_OF 344
#define TK_PLUS 345
#define TK_PRIVILEGE 346
#define TK_RAISE 347
#define TK_RESTRICT 348
#define TK_ROW 349
#define TK_SEMI 350
#define TK_STAR 351
#define TK_STATEMENT 352
#define TK_STRICT 353
#define TK_STRING 354
#define TK_TIMES 355
#define TK_VALUES 356
#define TK_VARIABLE 357
#define TK_WAL 358
#define TK_TSMA 191
#define TK_INTERVAL 192
#define TK_RECURSIVE 193
#define TK_TSMAS 194
#define TK_FUNCTION 195
#define TK_INDEX 196
#define TK_COUNT 197
#define TK_LAST_ROW 198
#define TK_META 199
#define TK_ONLY 200
#define TK_TOPIC 201
#define TK_CONSUMER 202
#define TK_GROUP 203
#define TK_DESC 204
#define TK_DESCRIBE 205
#define TK_RESET 206
#define TK_QUERY 207
#define TK_CACHE 208
#define TK_EXPLAIN 209
#define TK_ANALYZE 210
#define TK_VERBOSE 211
#define TK_NK_BOOL 212
#define TK_RATIO 213
#define TK_NK_FLOAT 214
#define TK_OUTPUTTYPE 215
#define TK_AGGREGATE 216
#define TK_BUFSIZE 217
#define TK_LANGUAGE 218
#define TK_REPLACE 219
#define TK_STREAM 220
#define TK_INTO 221
#define TK_PAUSE 222
#define TK_RESUME 223
#define TK_TRIGGER 224
#define TK_AT_ONCE 225
#define TK_WINDOW_CLOSE 226
#define TK_IGNORE 227
#define TK_EXPIRED 228
#define TK_FILL_HISTORY 229
#define TK_UPDATE 230
#define TK_SUBTABLE 231
#define TK_UNTREATED 232
#define TK_KILL 233
#define TK_CONNECTION 234
#define TK_TRANSACTION 235
#define TK_BALANCE 236
#define TK_VGROUP 237
#define TK_LEADER 238
#define TK_MERGE 239
#define TK_REDISTRIBUTE 240
#define TK_SPLIT 241
#define TK_DELETE 242
#define TK_INSERT 243
#define TK_NK_BIN 244
#define TK_NK_HEX 245
#define TK_NULL 246
#define TK_NK_QUESTION 247
#define TK_NK_ALIAS 248
#define TK_NK_ARROW 249
#define TK_ROWTS 250
#define TK_QSTART 251
#define TK_QEND 252
#define TK_QDURATION 253
#define TK_WSTART 254
#define TK_WEND 255
#define TK_WDURATION 256
#define TK_IROWTS 257
#define TK_ISFILLED 258
#define TK_CAST 259
#define TK_NOW 260
#define TK_TODAY 261
#define TK_TIMEZONE 262
#define TK_CLIENT_VERSION 263
#define TK_SERVER_VERSION 264
#define TK_SERVER_STATUS 265
#define TK_CURRENT_USER 266
#define TK_CASE 267
#define TK_WHEN 268
#define TK_THEN 269
#define TK_ELSE 270
#define TK_BETWEEN 271
#define TK_IS 272
#define TK_NK_LT 273
#define TK_NK_GT 274
#define TK_NK_LE 275
#define TK_NK_GE 276
#define TK_NK_NE 277
#define TK_MATCH 278
#define TK_NMATCH 279
#define TK_CONTAINS 280
#define TK_IN 281
#define TK_JOIN 282
#define TK_INNER 283
#define TK_LEFT 284
#define TK_RIGHT 285
#define TK_OUTER 286
#define TK_SEMI 287
#define TK_ANTI 288
#define TK_ASOF 289
#define TK_WINDOW 290
#define TK_WINDOW_OFFSET 291
#define TK_JLIMIT 292
#define TK_SELECT 293
#define TK_NK_HINT 294
#define TK_DISTINCT 295
#define TK_WHERE 296
#define TK_PARTITION 297
#define TK_BY 298
#define TK_SESSION 299
#define TK_STATE_WINDOW 300
#define TK_EVENT_WINDOW 301
#define TK_COUNT_WINDOW 302
#define TK_SLIDING 303
#define TK_FILL 304
#define TK_VALUE 305
#define TK_VALUE_F 306
#define TK_NONE 307
#define TK_PREV 308
#define TK_NULL_F 309
#define TK_LINEAR 310
#define TK_NEXT 311
#define TK_HAVING 312
#define TK_RANGE 313
#define TK_EVERY 314
#define TK_ORDER 315
#define TK_SLIMIT 316
#define TK_SOFFSET 317
#define TK_LIMIT 318
#define TK_OFFSET 319
#define TK_ASC 320
#define TK_NULLS 321
#define TK_ABORT 322
#define TK_AFTER 323
#define TK_ATTACH 324
#define TK_BEFORE 325
#define TK_BEGIN 326
#define TK_BITAND 327
#define TK_BITNOT 328
#define TK_BITOR 329
#define TK_BLOCKS 330
#define TK_CHANGE 331
#define TK_COMMA 332
#define TK_CONCAT 333
#define TK_CONFLICT 334
#define TK_COPY 335
#define TK_DEFERRED 336
#define TK_DELIMITERS 337
#define TK_DETACH 338
#define TK_DIVIDE 339
#define TK_DOT 340
#define TK_EACH 341
#define TK_FAIL 342
#define TK_FILE 343
#define TK_FOR 344
#define TK_GLOB 345
#define TK_ID 346
#define TK_IMMEDIATE 347
#define TK_IMPORT 348
#define TK_INITIALLY 349
#define TK_INSTEAD 350
#define TK_ISNULL 351
#define TK_MODULES 352
#define TK_NK_BITNOT 353
#define TK_NK_SEMI 354
#define TK_NOTNULL 355
#define TK_OF 356
#define TK_PLUS 357
#define TK_PRIVILEGE 358
#define TK_RAISE 359
#define TK_RESTRICT 360
#define TK_ROW 361
#define TK_STAR 362
#define TK_STATEMENT 363
#define TK_STRICT 364
#define TK_STRING 365
#define TK_TIMES 366
#define TK_VALUES 367
#define TK_VARIABLE 368
#define TK_WAL 369
#define TK_NK_SPACE 600
#define TK_NK_COMMENT 601
@ -388,6 +398,8 @@
#define TK_PARTITION_FIRST 609
#define TK_PARA_TABLES_SORT 610
#define TK_SMALLDATA_TS_SORT 611
#define TK_HASH_JOIN 612
#define TK_SKIP_TSMA 613
#define TK_NK_NIL 65535

27
include/libs/catalog/catalog.h
View File

@ -94,6 +94,8 @@ typedef struct SCatalogReq {
SArray* pTableCfg; // element is SNAME
SArray* pTableTag; // element is SNAME
SArray* pView; // element is STablesReq
SArray* pTableTSMAs; // element is STablesReq
SArray* pTSMAs; // element is STablesReq
bool qNodeRequired; // valid qnode
bool dNodeRequired; // valid dnode
bool svrVerRequired;
@ -122,6 +124,8 @@ typedef struct SMetaData {
SArray* pTableTag; // pRes = SArray<STagVal>*
SArray* pDnodeList; // pRes = SArray<SEpSet>*
SArray* pView; // pRes = SViewMeta*
SArray* pTableTsmas; // pRes = SArray<STableTSMAInfo*>
SArray* pTsmas; // pRes = SArray<STableTSMAInfo*>
SMetaRes* pSvrVer; // pRes = char*
} SMetaData;
@ -130,9 +134,11 @@ typedef struct SCatalogCfg {
uint32_t maxViewCacheNum;
uint32_t maxDBCacheNum;
uint32_t maxUserCacheNum;
uint32_t maxTSMACacheNum;
uint32_t dbRentSec;
uint32_t stbRentSec;
uint32_t viewRentSec;
uint32_t tsmaRentSec;
} SCatalogCfg;
typedef struct SSTableVersion {
@ -152,6 +158,7 @@ typedef struct SDbCacheInfo {
int32_t cfgVersion;
int32_t numOfTable; // unit is TSDB_TABLE_NUM_UNIT
int64_t stateTs;
int32_t tsmaVersion;
} SDbCacheInfo;
typedef struct SDynViewVersion {
@ -167,6 +174,14 @@ typedef struct SViewVersion {
int32_t version;
} SViewVersion;
typedef struct STSMAVersion {
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
char name[TSDB_TABLE_NAME_LEN];
uint64_t dbId;
uint64_t tsmaId;
int32_t version;
} STSMAVersion;
typedef struct STbSVersion {
char* tbFName;
@ -342,6 +357,8 @@ int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbCacheInfo** dbs, uint32_t* n
int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_t* num);
int32_t catalogGetExpiredTsmas(SCatalog* pCtg, STSMAVersion** tsmas, uint32_t* num);
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg);
int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* indexName, SIndexInfo* pInfo);
@ -388,6 +405,16 @@ int32_t ctgdEnableDebug(char* option, bool enable);
int32_t ctgdHandleDbgCommand(char* command);
int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma, int32_t tsmaVersion);
int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma);
int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma);
int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes);
int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma);
/**
* Destroy catalog and relase all resources
*/

21
include/libs/function/functionMgt.h
View File

@ -88,6 +88,7 @@ typedef enum EFunctionType {
FUNCTION_TYPE_LTRIM,
FUNCTION_TYPE_RTRIM,
FUNCTION_TYPE_SUBSTR,
FUNCTION_TYPE_MD5,
// conversion function
FUNCTION_TYPE_CAST = 2000,
@ -164,6 +165,18 @@ typedef enum EFunctionType {
FUNCTION_TYPE_STDDEV_MERGE,
FUNCTION_TYPE_IRATE_PARTIAL,
FUNCTION_TYPE_IRATE_MERGE,
FUNCTION_TYPE_AVG_STATE,
FUNCTION_TYPE_AVG_STATE_MERGE,
FUNCTION_TYPE_FIRST_STATE,
FUNCTION_TYPE_FIRST_STATE_MERGE,
FUNCTION_TYPE_LAST_STATE,
FUNCTION_TYPE_LAST_STATE_MERGE,
FUNCTION_TYPE_SPREAD_STATE,
FUNCTION_TYPE_SPREAD_STATE_MERGE,
FUNCTION_TYPE_STDDEV_STATE,
FUNCTION_TYPE_STDDEV_STATE_MERGE,
FUNCTION_TYPE_HYPERLOGLOG_STATE,
FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE,
// geometry functions
FUNCTION_TYPE_GEOM_FROM_TEXT = 4250,
@ -238,6 +251,7 @@ bool fmIsCumulativeFunc(int32_t funcId);
bool fmIsInterpPseudoColumnFunc(int32_t funcId);
bool fmIsGroupKeyFunc(int32_t funcId);
bool fmIsBlockDistFunc(int32_t funcId);
bool fmIsIgnoreNullFunc(int32_t funcId);
bool fmIsConstantResFunc(SFunctionNode* pFunc);
bool fmIsSkipScanCheckFunc(int32_t funcId);
bool fmIsPrimaryKeyFunc(int32_t funcId);
@ -270,6 +284,13 @@ bool fmIsInvertible(int32_t funcId);
char* fmGetFuncName(int32_t funcId);
bool fmIsTSMASupportedFunc(func_id_t funcId);
int32_t fmCreateStateFuncs(SNodeList* pFuncs);
int32_t fmCreateStateMergeFuncs(SNodeList* pFuncs);
int32_t fmGetFuncId(const char* name);
bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId);
bool fmIsCountLikeFunc(int32_t funcId);
#ifdef __cplusplus
}
#endif

30
include/libs/nodes/cmdnodes.h
View File

@ -213,11 +213,13 @@ typedef struct SDropTableClause {
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
bool ignoreNotExists;
SArray* pTsmas;
} SDropTableClause;
typedef struct SDropTableStmt {
ENodeType type;
SNodeList* pTables;
bool withTsma;
} SDropTableStmt;
typedef struct SDropSuperTableStmt {
@ -590,6 +592,34 @@ typedef struct SSplitVgroupStmt {
int32_t vgId;
} SSplitVgroupStmt;
typedef struct STSMAOptions {
ENodeType type;
SNodeList* pFuncs;
SNodeList* pCols;
SNode* pInterval;
uint8_t tsPrecision;
bool recursiveTsma; // true if create recursive tsma
} STSMAOptions;
typedef struct SCreateTSMAStmt {
ENodeType type;
bool ignoreExists;
char tsmaName[TSDB_TABLE_NAME_LEN];
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN]; // base tb name or base tsma name
char originalTbName[TSDB_TABLE_NAME_LEN];
STSMAOptions* pOptions;
SNode* pPrevQuery;
SMCreateSmaReq* pReq;
} SCreateTSMAStmt;
typedef struct SDropTSMAStmt {
ENodeType type;
bool ignoreNotExists;
char dbName[TSDB_DB_NAME_LEN];
char tsmaName[TSDB_TABLE_NAME_LEN];
} SDropTSMAStmt;
#ifdef __cplusplus
}
#endif

2
include/libs/nodes/nodes.h
View File

@ -123,6 +123,7 @@ int32_t nodesListAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListStrictAppendList(SNodeList* pTarget, SNodeList* pSrc);
int32_t nodesListMakeStrictAppendList(SNodeList** pTarget, SNodeList* pSrc);
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode);
int32_t nodesListMakePushFront(SNodeList** pList, SNode* pNode);
SListCell* nodesListErase(SNodeList* pList, SListCell* pCell);
void nodesListInsertList(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
void nodesListInsertListAfterPos(SNodeList* pTarget, SListCell* pPos, SNodeList* pSrc);
@ -169,6 +170,7 @@ int32_t nodesMsgToNode(const char* pStr, int32_t len, SNode** pNode);
int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len);
char* nodesGetNameFromColumnNode(SNode* pNode);
int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots);
void nodesSortList(SNodeList** pList, int32_t (*)(SNode* pNode1, SNode* pNode2));
#ifdef __cplusplus
}

65
include/libs/nodes/plannodes.h
View File

@ -66,6 +66,7 @@ typedef struct SLogicNode {
EOrder inputTsOrder;
EOrder outputTsOrder;
bool forceCreateNonBlockingOptr; // true if the operator can use non-blocking(pipeline) mode
bool splitDone;
} SLogicNode;
typedef enum EScanType {
@ -108,6 +109,9 @@ typedef struct SScanLogicNode {
int8_t igExpired;
int8_t igCheckUpdate;
SArray* pSmaIndexes;
SArray* pTsmas;
SArray* pTsmaTargetTbVgInfo;
SArray* pTsmaTargetTbInfo;
SNodeList* pGroupTags;
bool groupSort;
SNodeList* pTags; // for create stream
@ -123,20 +127,38 @@ typedef struct SScanLogicNode {
SArray* pFuncTypes; // for last, last_row
bool paraTablesSort; // for table merge scan
bool smallDataTsSort; // disable row id sort for table merge scan
bool needSplit;
} SScanLogicNode;
typedef struct SJoinLogicNode {
SLogicNode node;
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
EJoinAlgorithm joinAlgo;
SNode* addPrimEqCond;
SNode* pPrimKeyEqCond;
SNode* pColEqCond;
SNode* pColOnCond;
SNode* pTagEqCond;
SNode* pTagOnCond;
SNode* pOtherOnCond;
SNode* pFullOnCond; // except prim eq cond
SNodeList* pLeftEqNodes;
SNodeList* pRightEqNodes;
bool allEqTags;
bool isSingleTableJoin;
bool hasSubQuery;
bool isLowLevelJoin;
bool seqWinGroup;
bool grpJoin;
bool hashJoinHint;
// FOR HASH JOIN
int32_t timeRangeTarget; //table onCond filter
STimeWindow timeRange; //table onCond filter
SNode* pLeftOnCond; //table onCond filter
SNode* pRightOnCond; //table onCond filter
} SJoinLogicNode;
typedef struct SAggLogicNode {
@ -151,6 +173,7 @@ typedef struct SAggLogicNode {
bool isGroupTb;
bool isPartTb; // true if partition keys has tbname
bool hasGroup;
SNodeList *pTsmaSubplans;
} SAggLogicNode;
typedef struct SProjectLogicNode {
@ -234,7 +257,9 @@ typedef struct SMergeLogicNode {
SNodeList* pMergeKeys;
SNodeList* pInputs;
int32_t numOfChannels;
int32_t numOfSubplans;
int32_t srcGroupId;
int32_t srcEndGroupId;
bool colsMerge;
bool needSort;
bool groupSort;
@ -287,6 +312,7 @@ typedef struct SWindowLogicNode {
bool isPartTb;
int64_t windowCount;
int64_t windowSliding;
SNodeList* pTsmaSubplans;
} SWindowLogicNode;
typedef struct SFillLogicNode {
@ -482,21 +508,47 @@ typedef struct SInterpFuncPhysiNode {
typedef struct SSortMergeJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pOtherOnCond;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
int32_t asofOpType;
SNode* leftPrimExpr;
SNode* rightPrimExpr;
int32_t leftPrimSlotId;
int32_t rightPrimSlotId;
SNodeList* pEqLeft;
SNodeList* pEqRight;
SNode* pPrimKeyCond; //remove
SNode* pColEqCond; //remove
SNode* pColOnCond;
SNode* pFullOnCond;
SNodeList* pTargets;
SQueryStat inputStat[2];
bool seqWinGroup;
bool grpJoin;
} SSortMergeJoinPhysiNode;
typedef struct SHashJoinPhysiNode {
SPhysiNode node;
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
SNodeList* pOnLeft;
SNodeList* pOnRight;
SNode* pFilterConditions;
SNode* leftPrimExpr;
SNode* rightPrimExpr;
int32_t leftPrimSlotId;
int32_t rightPrimSlotId;
int32_t timeRangeTarget; //table onCond filter
STimeWindow timeRange; //table onCond filter
SNode* pLeftOnCond; //table onCond filter
SNode* pRightOnCond; //table onCond filter
SNode* pFullOnCond; //preFilter
SNodeList* pTargets;
SQueryStat inputStat[2];
// only in planner internal
SNode* pPrimKeyCond;
SNode* pColEqCond;
SNode* pTagEqCond;
@ -533,6 +585,7 @@ typedef struct SAggPhysiNode {
SNodeList* pAggFuncs;
bool mergeDataBlock;
bool groupKeyOptimized;
bool hasCountLikeFunc;
} SAggPhysiNode;
typedef struct SDownstreamSourceNode {
@ -562,7 +615,9 @@ typedef struct SMergePhysiNode {
SNodeList* pMergeKeys;
SNodeList* pTargets;
int32_t numOfChannels;
int32_t numOfSubplans;
int32_t srcGroupId;
int32_t srcEndGroupId;
bool groupSort;
bool ignoreGroupId;
bool inputWithGroupId;

68
include/libs/nodes/querynodes.h
View File

@ -23,6 +23,7 @@ extern "C" {
#include "nodes.h"
#include "tmsg.h"
#include "tvariant.h"
#include "tsimplehash.h"
#define TABLE_TOTAL_COL_NUM(pMeta) ((pMeta)->tableInfo.numOfColumns + (pMeta)->tableInfo.numOfTags)
#define TABLE_META_SIZE(pMeta) \
@ -79,6 +80,7 @@ typedef struct SColumnNode {
uint16_t projIdx; // the idx in project list, start from 1
EColumnType colType; // column or tag
bool hasIndex;
bool isPrimTs;
char dbName[TSDB_DB_NAME_LEN];
char tableName[TSDB_TABLE_NAME_LEN];
char tableAlias[TSDB_TABLE_NAME_LEN];
@ -103,10 +105,16 @@ typedef struct STargetNode {
SNode* pExpr;
} STargetNode;
#define VALUE_FLAG_IS_DURATION (1 << 0)
#define VALUE_FLAG_IS_TIME_OFFSET (1 << 1)
#define IS_DURATION_VAL(_flag) ((_flag) & VALUE_FLAG_IS_DURATION)
#define IS_TIME_OFFSET_VAL(_flag) ((_flag) & VALUE_FLAG_IS_TIME_OFFSET)
typedef struct SValueNode {
SExprNode node; // QUERY_NODE_VALUE
char* literal;
bool isDuration;
int32_t flag;
bool translate;
bool notReserved;
bool isNull;
@ -133,6 +141,8 @@ typedef enum EHintOption {
HINT_PARTITION_FIRST,
HINT_PARA_TABLES_SORT,
HINT_SMALLDATA_TS_SORT,
HINT_HASH_JOIN,
HINT_SKIP_TSMA,
} EHintOption;
typedef struct SHintNode {
@ -168,6 +178,8 @@ typedef struct SFunctionNode {
int32_t udfBufSize;
bool hasPk;
int32_t pkBytes;
bool hasOriginalFunc;
int32_t originalFuncId;
} SFunctionNode;
typedef struct STableNode {
@ -181,6 +193,11 @@ typedef struct STableNode {
struct STableMeta;
typedef struct STsmaTargetCTbInfo {
char tableName[TSDB_TABLE_NAME_LEN]; // child table or normal table name
uint64_t uid;
} STsmaTargetTbInfo;
typedef struct SRealTableNode {
STableNode table; // QUERY_NODE_REAL_TABLE
struct STableMeta* pMeta;
@ -189,6 +206,9 @@ typedef struct SRealTableNode {
double ratio;
SArray* pSmaIndexes;
int8_t cacheLastMode;
SArray* pTsmas;
SArray* tsmaTargetTbVgInfo; // SArray<SVgroupsInfo*>, used for child table or normal table only
SArray* tsmaTargetTbInfo; // SArray<STsmaTargetTbInfo>, used for child table or normal table only
} SRealTableNode;
typedef struct STempTableNode {
@ -206,12 +226,31 @@ typedef struct SViewNode {
int8_t cacheLastMode;
} SViewNode;
#define JOIN_JLIMIT_MAX_VALUE 1024
#define IS_INNER_NONE_JOIN(_type, _stype) ((_type) == JOIN_TYPE_INNER && (_stype) == JOIN_STYPE_NONE)
#define IS_SEMI_JOIN(_stype) ((_stype) == JOIN_STYPE_SEMI)
#define IS_WINDOW_JOIN(_stype) ((_stype) == JOIN_STYPE_WIN)
#define IS_ASOF_JOIN(_stype) ((_stype) == JOIN_STYPE_ASOF)
typedef enum EJoinType {
JOIN_TYPE_INNER = 1,
JOIN_TYPE_INNER = 0,
JOIN_TYPE_LEFT,
JOIN_TYPE_RIGHT,
JOIN_TYPE_FULL,
JOIN_TYPE_MAX_VALUE
} EJoinType;
typedef enum EJoinSubType {
JOIN_STYPE_NONE = 0,
JOIN_STYPE_OUTER,
JOIN_STYPE_SEMI,
JOIN_STYPE_ANTI,
JOIN_STYPE_ASOF,
JOIN_STYPE_WIN,
JOIN_STYPE_MAX_VALUE
} EJoinSubType;
typedef enum EJoinAlgorithm {
JOIN_ALGO_UNKNOWN = 0,
JOIN_ALGO_MERGE,
@ -225,8 +264,13 @@ typedef enum EDynQueryType {
typedef struct SJoinTableNode {
STableNode table; // QUERY_NODE_JOIN_TABLE
EJoinType joinType;
EJoinSubType subType;
SNode* pWindowOffset;
SNode* pJLimit;
SNode* addPrimCond;
bool hasSubQuery;
bool isLowLevelJoin;
SNode* pParent;
SNode* pLeft;
SNode* pRight;
SNode* pOnCond;
@ -305,6 +349,7 @@ typedef enum EFillMode {
typedef enum ETimeLineMode {
TIME_LINE_NONE = 1,
TIME_LINE_BLOCK,
TIME_LINE_MULTI,
TIME_LINE_GLOBAL,
} ETimeLineMode;
@ -338,6 +383,13 @@ typedef struct SCaseWhenNode {
SNodeList* pWhenThenList;
} SCaseWhenNode;
typedef struct SWindowOffsetNode {
ENodeType type; // QUERY_NODE_WINDOW_OFFSET
SNode* pStartOffset; // SValueNode
SNode* pEndOffset; // SValueNode
} SWindowOffsetNode;
typedef struct SSelectStmt {
ENodeType type; // QUERY_NODE_SELECT_STMT
bool isDistinct;
@ -362,6 +414,7 @@ typedef struct SSelectStmt {
uint8_t precision;
int32_t selectFuncNum;
int32_t returnRows; // EFuncReturnRows
ETimeLineMode timeLineCurMode;
ETimeLineMode timeLineResMode;
bool isEmptyResult;
bool isSubquery;
@ -385,6 +438,7 @@ typedef struct SSelectStmt {
bool onlyHasKeepOrderFunc;
bool groupSort;
bool tagScan;
bool joinContains;
} SSelectStmt;
typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType;
@ -400,6 +454,7 @@ typedef struct SSetOperator {
char stmtName[TSDB_TABLE_NAME_LEN];
uint8_t precision;
ETimeLineMode timeLineResMode;
bool joinContains;
} SSetOperator;
typedef enum ESqlClause {
@ -537,12 +592,15 @@ typedef struct SQuery {
bool stableQuery;
} SQuery;
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext);
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext);
typedef enum ECollectColType { COLLECT_COL_TYPE_COL = 1, COLLECT_COL_TYPE_TAG, COLLECT_COL_TYPE_ALL } ECollectColType;
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols);
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols);
typedef bool (*FFuncClassifier)(int32_t funcId);
@ -579,6 +637,12 @@ const char* logicConditionTypeStr(ELogicConditionType type);
bool nodesIsStar(SNode* pNode);
bool nodesIsTableStar(SNode* pNode);
char* getJoinTypeString(EJoinType type);
char* getJoinSTypeString(EJoinSubType type);
char* getFullJoinTypeString(EJoinType type, EJoinSubType stype);
int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc);
#ifdef __cplusplus
}
#endif

1
include/libs/parser/parser.h
View File

@ -158,6 +158,7 @@ int32_t rewriteToVnodeModifyOpStmt(SQuery* pQuery, SArray* pBufArray);
SArray* serializeVgroupsCreateTableBatch(SHashObj* pVgroupHashmap);
SArray* serializeVgroupsDropTableBatch(SHashObj* pVgroupHashmap);
void destoryCatalogReq(SCatalogReq *pCatalogReq);
bool isPrimaryKeyImpl(SNode* pExpr);
#ifdef __cplusplus
}

4
include/libs/qcom/query.h
View File

@ -272,6 +272,7 @@ void initQueryModuleMsgHandle();
const SSchema* tGetTbnameColumnSchema();
bool tIsValidSchema(struct SSchema* pSchema, int32_t numOfCols, int32_t numOfTags);
int32_t getAsofJoinReverseOp(EOperatorType op);
int32_t queryCreateCTableMetaFromMsg(STableMetaRsp* msg, SCTableMeta* pMeta);
int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STableMeta** pMeta);
@ -330,7 +331,8 @@ extern int32_t (*queryProcessMsgRsp[TDMT_MAX])(void* output, char* msg, int32_t
#define NEED_CLIENT_RM_TBLMETA_REQ(_type) \
((_type) == TDMT_VND_CREATE_TABLE || (_type) == TDMT_MND_CREATE_STB || (_type) == TDMT_VND_DROP_TABLE || \
(_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW)
(_type) == TDMT_MND_DROP_STB || (_type) == TDMT_MND_CREATE_VIEW || (_type) == TDMT_MND_DROP_VIEW || \
(_type) == TDMT_MND_CREATE_TSMA || (_type) == TDMT_MND_DROP_TSMA || (_type) == TDMT_MND_DROP_TB_WITH_TSMA)
#define NEED_SCHEDULER_REDIRECT_ERROR(_code) \
(SYNC_UNKNOWN_LEADER_REDIRECT_ERROR(_code) || SYNC_SELF_LEADER_REDIRECT_ERROR(_code) || \

11
include/libs/scalar/filter.h
View File

@ -31,6 +31,15 @@ enum {
FLT_OPTION_NEED_UNIQE = 4,
};
typedef enum EConditionType {
COND_TYPE_PRIMARY_KEY = 1,
COND_TYPE_TAG_INDEX,
COND_TYPE_TAG,
COND_TYPE_NORMAL
} EConditionType;
#define FILTER_RESULT_ALL_QUALIFIED 0x1
#define FILTER_RESULT_NONE_QUALIFIED 0x2
#define FILTER_RESULT_PARTIAL_QUALIFIED 0x3
@ -54,6 +63,8 @@ extern bool filterRangeExecute(SFilterInfo *info, SColumnDataAgg **pColsAgg,
/* condition split interface */
int32_t filterPartitionCond(SNode **pCondition, SNode **pPrimaryKeyCond, SNode **pTagIndexCond, SNode **pTagCond,
SNode **pOtherCond);
bool filterIsMultiTableColsCond(SNode *pCond);
EConditionType filterClassifyCondition(SNode *pNode);
#ifdef __cplusplus
}

2
include/libs/scalar/scalar.h
View File

@ -72,6 +72,7 @@ int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut
int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t rtrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t substrFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t md5Function(SScalarParam* pInput, int32_t inputNum, SScalarParam* pOutput);
/* Conversion functions */
int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
@ -82,6 +83,7 @@ int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarP
int32_t toJsonFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toTimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t toCharFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int64_t offsetFromTz(char *timezone, int64_t factor);
int32_t timeTruncateFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t timeDiffFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);
int32_t nowFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput);

2
include/os/osMath.h
View File

@ -31,7 +31,7 @@ extern "C" {
#define TSWAP(a, b) \
do { \
char *__tmp = alloca(sizeof(a)); \
char *__tmp = (char*)alloca(sizeof(a)); \
memcpy(__tmp, &(a), sizeof(a)); \
memcpy(&(a), &(b), sizeof(a)); \
memcpy(&(b), __tmp, sizeof(a)); \

27
include/util/taoserror.h
View File

@ -423,6 +423,8 @@ int32_t* taosGetErrno();
#define TSDB_CODE_MND_SMA_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0480)
#define TSDB_CODE_MND_SMA_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x0481)
#define TSDB_CODE_MND_INVALID_SMA_OPTION TAOS_DEF_ERROR_CODE(0, 0x0482)
#define TSDB_CODE_MND_INVALID_DROP_TSMA TAOS_DEF_ERROR_CODE(0, 0x0485)
#define TSDB_CODE_MND_MAX_TSMA_NUM_EXCEEDED TAOS_DEF_ERROR_CODE(0, 0x0486)
// mnode-tag-indxe
@ -547,6 +549,7 @@ int32_t* taosGetErrno();
#define TSDB_CODE_QRY_QWORKER_QUIT TAOS_DEF_ERROR_CODE(0, 0x0730)
#define TSDB_CODE_QRY_GEO_NOT_SUPPORT_ERROR TAOS_DEF_ERROR_CODE(0, 0x0731)
#define TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x0732)
#define TSDB_CODE_QRY_INVALID_JOIN_CONDITION TAOS_DEF_ERROR_CODE(0, 0x0733)
// grant
#define TSDB_CODE_GRANT_EXPIRED TAOS_DEF_ERROR_CODE(0, 0x0800)
@ -762,12 +765,18 @@ int32_t* taosGetErrno();
#define TSDB_CODE_PAR_VIEW_CONFLICT_WITH_TABLE TAOS_DEF_ERROR_CODE(0, 0x266E)
#define TSDB_CODE_PAR_ORDERBY_AMBIGUOUS TAOS_DEF_ERROR_CODE(0, 0x266F)
#define TSDB_CODE_PAR_NOT_SUPPORT_MULTI_RESULT TAOS_DEF_ERROR_CODE(0, 0x2670)
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2671)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2673)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x2674)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x2675)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x2676)
#define TSDB_CODE_PAR_GRP_WINDOW_NOT_ALLOWED TAOS_DEF_ERROR_CODE(0, 0x2671)
#define TSDB_CODE_PAR_INVALID_WJOIN_HAVING_EXPR TAOS_DEF_ERROR_CODE(0, 0x2672)
#define TSDB_CODE_PAR_INVALID_WIN_OFFSET_UNIT TAOS_DEF_ERROR_CODE(0, 0x2673)
#define TSDB_CODE_PAR_VALID_PRIM_TS_REQUIRED TAOS_DEF_ERROR_CODE(0, 0x2674)
#define TSDB_CODE_PAR_ORDERBY_UNKNOWN_EXPR TAOS_DEF_ERROR_CODE(0, 0x2675)
#define TSDB_CODE_PAR_NOT_WIN_FUNC TAOS_DEF_ERROR_CODE(0, 0x2676)
#define TSDB_CODE_PAR_TAG_IS_PRIMARY_KEY TAOS_DEF_ERROR_CODE(0, 0x2677)
#define TSDB_CODE_PAR_SECOND_COL_PK TAOS_DEF_ERROR_CODE(0, 0x2678)
#define TSDB_CODE_PAR_COL_PK_TYPE TAOS_DEF_ERROR_CODE(0, 0x2679)
#define TSDB_CODE_PAR_INVALID_PK_OP TAOS_DEF_ERROR_CODE(0, 0x267A)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NULL TAOS_DEF_ERROR_CODE(0, 0x267B)
#define TSDB_CODE_PAR_PRIMARY_KEY_IS_NONE TAOS_DEF_ERROR_CODE(0, 0x267C)
#define TSDB_CODE_PAR_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x26FF)
//planner
@ -816,6 +825,12 @@ int32_t* taosGetErrno();
#define TSDB_CODE_TSMA_INVALID_STAT TAOS_DEF_ERROR_CODE(0, 0x3103)
#define TSDB_CODE_TSMA_INVALID_PTR TAOS_DEF_ERROR_CODE(0, 0x3104)
#define TSDB_CODE_TSMA_INVALID_PARA TAOS_DEF_ERROR_CODE(0, 0x3105)
#define TSDB_CODE_TSMA_INVALID_TB TAOS_DEF_ERROR_CODE(0, 0x3106)
#define TSDB_CODE_TSMA_INVALID_INTERVAL TAOS_DEF_ERROR_CODE(0, 0x3107)
#define TSDB_CODE_TSMA_INVALID_FUNC_PARAM TAOS_DEF_ERROR_CODE(0, 0x3108)
#define TSDB_CODE_TSMA_UNSUPPORTED_FUNC TAOS_DEF_ERROR_CODE(0, 0x3109)
#define TSDB_CODE_TSMA_MUST_BE_DROPPED TAOS_DEF_ERROR_CODE(0, 0x3110)
#define TSDB_CODE_TSMA_NAME_TOO_LONG TAOS_DEF_ERROR_CODE(0, 0x3111)
//rsma
#define TSDB_CODE_RSMA_INVALID_ENV TAOS_DEF_ERROR_CODE(0, 0x3150)

3
include/util/tdef.h
View File

@ -149,7 +149,7 @@ typedef enum EOperatorType {
OP_TYPE_BIT_OR,
// binary comparison operator
OP_TYPE_GREATER_THAN = 40,
OP_TYPE_GREATER_THAN = 40, // MUST KEEP IT FIRST AT COMPARE SECTION
OP_TYPE_GREATER_EQUAL,
OP_TYPE_LOWER_THAN,
OP_TYPE_LOWER_EQUAL,
@ -170,6 +170,7 @@ typedef enum EOperatorType {
OP_TYPE_IS_NOT_TRUE,
OP_TYPE_IS_NOT_FALSE,
OP_TYPE_IS_NOT_UNKNOWN,
OP_TYPE_COMPARE_MAX_VALUE = 149, // MUST KEEP IT LAST AT COMPARE SECTION
// json operator
OP_TYPE_JSON_GET_VALUE = 150,

219
packaging/tools/quick_deploy.sh Normal file
View File

@ -0,0 +1,219 @@
#!/bin/bash
log_date_time=`date +%m%d%Y-%H%M`
log=${log_date_time}.log
dataDir=/data
logDir=/data/taos/log
echo "#########collect sysinfo############"
cat /etc/os-release
free -g
cat /proc/cpuinfo |grep processor
uname -a
df -h
lsblk
blkid
cat /etc/fstab
mount
cat /etc/sysctl.conf
cat /etc/security/limits.conf
sysctl -p
######check hostname #####
echo "#########hostname"
hostname
hostname -f
echo "#########/etc/hostname"
cat /etc/hostname
echo "#########/etc/hosts"
cat /etc/hosts
echo "#########Step 1############"
echo "$#"
######single node without config file#####
if [ "$#" -eq 0 ] ;then
echo "#########This is for single node !!"
echo "#########Please provide config.ini file for cluster installation !!"
first_ep=`hostname -f`:6030
second_ep=""
fi
echo "#########Step 2############"
######use config file to configre cluster!#####
echo $1
config_file=$1
if [ "$#" -eq 1 ] ;then
echo "#########Use $1 file to configure cluster."
echo "#########The configure of cluster"
cat $config_file
first_ep=`sed -n '1p' $config_file |awk '{print $2}'`:6030
second_ep=`sed -n '2p' $config_file |awk '{print $2}'`:6030
fi
echo "#########Step 3############"
################################################################
echo "#########create data directory"
mkdir -p ${dataDir}/taos/{data,core,tmp,soft}
mkdir -p ${logDir}
chmod 777 ${dataDir}/taos/tmp
chmod 777 ${logDir}
echo "#########install suggest package and disable firewall"
os=$(cat /etc/os-release| grep PRETTY_NAME | awk '{print $1}'|awk -F '=' '{print $2}' | sed 's/"//g')
if [ $os = 'Ubuntu' ]
then
echo "This is Ununtu!"
#apt install screen tmux gdb fio iperf3 sysstat net-tools ntp tree wget
ufw status
ufw stop
ufw disable
elif [ $os = 'CentOS' -o $os = 'Red' ]
then
echo "This is Centos/Red"
#yum install -y screen tmux gdb fio iperf3 sysstat net-tools ntp tree wget
systemctl status firewalld
systemctl stop firewalld
systemctl disable firewalld
else
echo "####$os"
fi
sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
echo "#########system tunning"
echo "fs.nr_open = 10485760" >>/etc/sysctl.conf
echo "net.core.somaxconn=10240" >> /etc/sysctl.conf
echo "net.core.netdev_max_backlog=20480" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_syn_backlog=10240" >> /etc/sysctl.conf
echo "net.ipv4.tcp_retries2=5" >> /etc/sysctl.conf
echo "net.ipv4.tcp_syn_retries=2" >> /etc/sysctl.conf
echo "net.ipv4.tcp_synack_retries=2" >> /etc/sysctl.conf
echo "net.ipv4.tcp_tw_reuse=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_tw_recycle=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_keepalive_time=600" >> /etc/sysctl.conf
echo "net.ipv4.tcp_abort_on_overflow=1" >> /etc/sysctl.conf
echo "net.ipv4.tcp_max_tw_buckets=5000" >> /etc/sysctl.conf
echo "net.ipv4.ip_local_port_range=10000 60999" >> /etc/sysctl.conf
echo "* soft ntaosc 65536" >>/etc/security/limits.conf
echo "* soft nofile 1048576" >>/etc/security/limits.conf
echo "* soft stack 65536" >>/etc/security/limits.conf
echo "* hard ntaosc 65536" >>/etc/security/limits.conf
echo "* hard nofile 1048576" >>/etc/security/limits.conf
echo "* hard stack 65536" >>/etc/security/limits.conf
echo "root soft ntaosc 65536" >>/etc/security/limits.conf
echo "root soft nofile 1048576" >>/etc/security/limits.conf
echo "root soft stack 65536" >>/etc/security/limits.conf
echo "root hard ntaosc 65536" >>/etc/security/limits.conf
echo "root hard nofile 1048576" >>/etc/security/limits.conf
echo "root hard stack 65536" >>/etc/security/limits.conf
echo "ulimit -c unlimited" >>/etc/profile
echo "kernel.core_pattern=${dataDir}/taos/core/core-%e-%p" >>/etc/sysctl.conf
sysctl -p
date
####to do check date timezone and ntp########
#cp -f /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
#NTP
#SWAP
#reboot
echo "#########install TDengine"
./install.sh -e no
pwd
echo "#########Create /etc/taos/taos.cfg"
mv /etc/taos/taos.cfg /etc/taos/taos.cfg.bak
cat /proc/cpuinfo | grep "processor" | wc -l
num=`cat /proc/cpuinfo | grep "processor" | wc -l`
cpunum=$(expr $num + $num)
host_name=`hostname -f`
#####vi /etc/taos/taos.cfg
echo "firstEp $first_ep
secondEp $second_ep
fqdn $host_name
supportVnodes $cpunum
logDir $logDir
dataDir ${dataDir}/taos/data
tempDir ${dataDir}/taos/tmp
keepColumnName 0
maxNumOfDistinctRes 10000000
timezone UTC-8
locale en_US.UTF-8
charset UTF-8
maxShellConns 100000
maxConnections 100000
audit 0
auditFqdn localhost
monitor 1
monitorFqdn localhost
logKeepDays 10
debugflag 131
shellActivityTimer 120
numOfRpcSessions 30000" >> /etc/taos/taos.cfg
########################
########################tunning taosadapter and taosx log directory
sed -i "/^\#*path/c\path\ =\ \"${logDir}\"" /etc/taos/taosadapter.toml
sed -i "/^\#*logs_home/c\logs_home\ =\ \"${logDir}\"" /etc/taos/taosx.toml
##########################single nodes
echo "#########Start toasd"
./start-all.sh
date
if [ "$#" -eq 0 ] ;then
sleep 5
taos -s "show dnodes;"
taos -s "show cluster\G;"
echo "########"
taosd -k
curl -u root:taosdata ${host_name}:6041/rest/sql -d "select server_version()"
echo -e "\r"
echo "#########Installation completed########"
echo "#########Suggest reboot OS##########"
date
exit
fi
##########################use $config_file to config cluster
if test "$host_name"x = `sed -n '1p' $config_file |awk '{print $2}'`x ;then
echo "#########Add dnodes and mnodes"
while read ip_address host_name
do
echo "$host_name"
taos -s "create dnode '$host_name:6030';"
sleep 5
done <$config_file
fi
host_name=`hostname -f`
if test "$host_name"x = `sed -n '3p' $config_file |awk '{print $2}'`x ;then
echo "#########Add mnodes"
taos -s "create mnode on dnode 2;"
sleep 5
taos -s "create mnode on dnode 3;"
fi
sleep 5
taos -s "show dnodes;"
taos -s "show mnodes;"
taos -s "show grants\G;"
taos -s "show cluster\G;"
echo "########"
taosd -k
echo "#########Test TDengine"
curl -u root:taosdata ${host_name}:6041/rest/sql -d "select server_version()"
echo -e "\r"
echo "#########Installation completed########"
echo "#########Suggest reboot OS##########"
date
cd ..

78
source/client/src/clientHb.c
View File

@ -234,6 +234,15 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog
catalogUpdateDbCfg(pCatalog, rsp->cfgRsp->db, rsp->cfgRsp->dbId, rsp->cfgRsp);
rsp->cfgRsp = NULL;
}
if (rsp->pTsmaRsp) {
if (rsp->pTsmaRsp->pTsmas) {
for (int32_t i = 0; i < rsp->pTsmaRsp->pTsmas->size; ++i) {
STableTSMAInfo* pTsma = taosArrayGetP(rsp->pTsmaRsp->pTsmas, i);
catalogAsyncUpdateTSMA(pCatalog, &pTsma, rsp->dbTsmaVersion);
}
taosArrayClear(rsp->pTsmaRsp->pTsmas);
}
}
}
_return:
@ -327,6 +336,33 @@ static int32_t hbProcessViewInfoRsp(void *value, int32_t valueLen, struct SCatal
return TSDB_CODE_SUCCESS;
}
static int32_t hbprocessTSMARsp(void* value, int32_t valueLen, struct SCatalog* pCatalog) {
int32_t code = 0;
STSMAHbRsp hbRsp = {0};
if (tDeserializeTSMAHbRsp(value, valueLen, &hbRsp)) {
terrno = TSDB_CODE_INVALID_MSG;
return -1;
}
int32_t numOfTsma = taosArrayGetSize(hbRsp.pTsmas);
for (int32_t i = 0; i < numOfTsma; ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(hbRsp.pTsmas, i);
if (!pTsmaInfo->pFuncs) {
tscDebug("hb to remove tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
catalogRemoveTSMA(pCatalog, pTsmaInfo);
tFreeAndClearTableTSMAInfo(pTsmaInfo);
} else {
tscDebug("hb to update tsma: %s.%s", pTsmaInfo->dbFName, pTsmaInfo->name);
catalogUpdateTSMA(pCatalog, &pTsmaInfo);
tFreeAndClearTableTSMAInfo(pTsmaInfo);
}
}
taosArrayDestroy(hbRsp.pTsmas);
return TSDB_CODE_SUCCESS;
}
static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *pCatalog, SAppHbMgr *pAppHbMgr) {
for (int32_t i = 0; i < kvNum; ++i) {
@ -379,6 +415,13 @@ static void hbProcessQueryRspKvs(int32_t kvNum, SArray* pKvs, struct SCatalog *p
break;
}
#endif
case HEARTBEAT_KEY_TSMA: {
if (kv->valueLen <= 0 || !kv->value) {
tscError("Invalid tsma info, len: %d, value: %p", kv->valueLen, kv->value);
}
hbprocessTSMARsp(kv->value, kv->valueLen, pCatalog);
break;
}
default:
tscError("invalid hb key type:%d", kv->key);
break;
@ -763,6 +806,7 @@ int32_t hbGetExpiredDBInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SCl
db->cfgVersion = htonl(db->cfgVersion);
db->numOfTable = htonl(db->numOfTable);
db->stateTs = htobe64(db->stateTs);
db->tsmaVersion = htonl(db->tsmaVersion);
}
SKv kv = {
@ -871,6 +915,39 @@ int32_t hbGetExpiredViewInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, S
return TSDB_CODE_SUCCESS;
}
int32_t hbGetExpiredTSMAInfo(SClientHbKey* connKey, struct SCatalog* pCatalog, SClientHbReq* pReq) {
int32_t code = 0;
uint32_t tsmaNum = 0;
STSMAVersion *tsmas = NULL;
code = catalogGetExpiredTsmas(pCatalog, &tsmas, &tsmaNum);
if (code) {
taosMemoryFree(tsmas);
return code;
}
if (tsmaNum <= 0) {
taosMemoryFree(tsmas);
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < tsmaNum; ++i) {
STSMAVersion* tsma = &tsmas[i];
tsma->dbId = htobe64(tsma->dbId);
tsma->tsmaId = htobe64(tsma->tsmaId);
tsma->version = htonl(tsma->version);
}
tscDebug("hb got %d expred tsmas, valueLen: %lu", tsmaNum, sizeof(STSMAVersion) * tsmaNum);
if (!pReq->info) {
pReq->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK);
}
SKv kv = {.key = HEARTBEAT_KEY_TSMA, .valueLen = sizeof(STSMAVersion) * tsmaNum, .value = tsmas};
taosHashPut(pReq->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv));
return TSDB_CODE_SUCCESS;
}
int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) {
SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId));
@ -935,6 +1012,7 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req
return code;
}
#endif
code = hbGetExpiredTSMAInfo(connKey, pCatalog, req);
} else {
req->app.appId = 0;
}

1
source/client/src/clientImpl.c
View File

@ -1041,6 +1041,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) {
if (code != TSDB_CODE_SUCCESS && NEED_CLIENT_HANDLE_ERROR(code) && pRequest->sqlstr != NULL) {
tscDebug("0x%" PRIx64 " client retry to handle the error, code:%s, tryCount:%d, reqId:0x%" PRIx64, pRequest->self,
tstrerror(code), pRequest->retry, pRequest->requestId);
removeMeta(pTscObj, pRequest->targetTableList, IS_VIEW_REQUEST(pRequest->type));
restartAsyncQuery(pRequest, code);
return;
}

2
source/client/src/clientMain.c
View File

@ -936,6 +936,8 @@ int32_t cloneCatalogReq(SCatalogReq **ppTarget, SCatalogReq *pSrc) {
pTarget->pTableCfg = taosArrayDup(pSrc->pTableCfg, NULL);
pTarget->pTableTag = taosArrayDup(pSrc->pTableTag, NULL);
pTarget->pView = taosArrayDup(pSrc->pView, NULL);
pTarget->pTableTSMAs = taosArrayDup(pSrc->pTableTSMAs, NULL);
pTarget->pTSMAs = taosArrayDup(pSrc->pTSMAs, NULL);
pTarget->qNodeRequired = pSrc->qNodeRequired;
pTarget->dNodeRequired = pSrc->dNodeRequired;
pTarget->svrVerRequired = pSrc->svrVerRequired;

13
source/common/src/systable.c
View File

@ -372,6 +372,18 @@ static const SSysDbTableSchema userCompactsDetailSchema[] = {
{.name = "finished", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false},
{.name = "start_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
};
static const SSysDbTableSchema tsmaSchema[] = {
{.name = "tsma_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "table_name", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_db", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "target_stb", .bytes = SYSTABLE_SCH_TABLE_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "stream_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_time", .bytes = 8, .type = TSDB_DATA_TYPE_TIMESTAMP, .sysInfo = false},
{.name = "interval", .bytes = 64 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "create_sql", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
{.name = "func_list", .bytes = TSDB_SHOW_SQL_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false},
};
static const SSysDbTableSchema userGrantsFullSchema[] = {
{.name = "grant_name", .bytes = 32 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true},
@ -427,6 +439,7 @@ static const SSysTableMeta infosMeta[] = {
{TSDB_INS_TABLE_GRANTS_LOGS, userGrantsLogsSchema, tListLen(userGrantsLogsSchema), true},
{TSDB_INS_TABLE_MACHINES, userMachinesSchema, tListLen(userMachinesSchema), true},
{TSDB_INS_TABLE_ARBGROUPS, arbGroupsSchema, tListLen(arbGroupsSchema), true},
{TSDB_INS_TABLE_TSMAS, tsmaSchema, tListLen(tsmaSchema), false},
};
static const SSysDbTableSchema connectionsSchema[] = {

249
source/common/src/tdatablock.c
View File

@ -191,15 +191,17 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
}
size_t start = 1;
// the first item
memcpy(pColumnInfoData->pData, pData, itemLen);
int32_t t = 0;
int32_t count = log(numOfRows) / log(2);
uint32_t startOffset = (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) ? pColumnInfoData->varmeta.length : (currentRow * itemLen);
// the first item
memcpy(pColumnInfoData->pData + startOffset, pData, itemLen);
while (t < count) {
int32_t xlen = 1 << t;
memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData,
memcpy(pColumnInfoData->pData + start * itemLen + startOffset,
pColumnInfoData->pData + startOffset,
xlen * itemLen);
t += 1;
start += xlen;
@ -207,7 +209,8 @@ static int32_t doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t cur
// the tail part
if (numOfRows > start) {
memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData,
memcpy(pColumnInfoData->pData + start * itemLen + startOffset,
pColumnInfoData->pData + startOffset,
(numOfRows - start) * itemLen);
}
@ -226,9 +229,13 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
bool trimValue) {
int32_t len = pColumnInfoData->info.bytes;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
if (pColumnInfoData->info.type == TSDB_DATA_TYPE_JSON) {
len = getJsonValueLen(pData);
} else {
len = varDataTLen(pData);
if (pColumnInfoData->varmeta.allocLen < (numOfRows + currentRow) * len) {
int32_t code = colDataReserve(pColumnInfoData, (numOfRows + currentRow) * len);
}
if (pColumnInfoData->varmeta.allocLen < (numOfRows * len + pColumnInfoData->varmeta.length)) {
int32_t code = colDataReserve(pColumnInfoData, (numOfRows * len + pColumnInfoData->varmeta.length));
if (code != TSDB_CODE_SUCCESS) {
return code;
}
@ -239,10 +246,12 @@ int32_t colDataSetNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
}
void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow, uint32_t numOfRows) {
pColumnInfoData->hasNull = true;
if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) {
memset(&pColumnInfoData->varmeta.offset[currentRow], -1, sizeof(int32_t) * numOfRows);
} else {
if (numOfRows < sizeof(char) * 2) {
if (numOfRows < 16) {
for (int32_t i = 0; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
}
@ -256,8 +265,9 @@ void colDataSetNItemsNull(SColumnInfoData* pColumnInfoData, uint32_t currentRow,
}
}
memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, (numOfRows - i) / sizeof(char));
i += (numOfRows - i) / sizeof(char) * sizeof(char);
int32_t bytes = (numOfRows - i) / 8;
memset(&BMCharPos(pColumnInfoData->nullbitmap, currentRow + i), 0xFF, bytes);
i += bytes * 8;
for (; i < numOfRows; ++i) {
colDataSetNull_f(pColumnInfoData->nullbitmap, currentRow + i);
@ -481,6 +491,111 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p
return 0;
}
int32_t colDataAssignNRows(SColumnInfoData* pDst, int32_t dstIdx, const SColumnInfoData* pSrc, int32_t srcIdx, int32_t numOfRows) {
if (pDst->info.type != pSrc->info.type || pDst->info.bytes != pSrc->info.bytes || pSrc->reassigned) {
return TSDB_CODE_FAILED;
}
if (numOfRows <= 0) {
return numOfRows;
}
if (IS_VAR_DATA_TYPE(pDst->info.type)) {
int32_t allLen = 0;
void* srcAddr = NULL;
if (pSrc->hasNull) {
for (int32_t i = 0; i < numOfRows; ++i) {
if (colDataIsNull_var(pSrc, srcIdx + i)) {
pDst->varmeta.offset[dstIdx + i] = -1;
pDst->hasNull = true;
continue;
}
char* pData = colDataGetVarData(pSrc, srcIdx + i);
if (NULL == srcAddr) {
srcAddr = pData;
}
int32_t dataLen = 0;
if (pSrc->info.type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
pDst->varmeta.offset[dstIdx + i] = pDst->varmeta.length + allLen;
allLen += dataLen;
}
} else {
for (int32_t i = 0; i < numOfRows; ++i) {
char* pData = colDataGetVarData(pSrc, srcIdx + i);
int32_t dataLen = 0;
if (pSrc->info.type == TSDB_DATA_TYPE_JSON) {
dataLen = getJsonValueLen(pData);
} else {
dataLen = varDataTLen(pData);
}
pDst->varmeta.offset[dstIdx + i] = pDst->varmeta.length + allLen;
allLen += dataLen;
}
}
if (allLen > 0) {
// copy data
if (pDst->varmeta.allocLen < pDst->varmeta.length + allLen) {
char* tmp = taosMemoryRealloc(pDst->pData, pDst->varmeta.length + allLen);
if (tmp == NULL) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pDst->pData = tmp;
pDst->varmeta.allocLen = pDst->varmeta.length + allLen;
}
if (pSrc->hasNull) {
memcpy(pDst->pData + pDst->varmeta.length, srcAddr, allLen);
} else {
memcpy(pDst->pData + pDst->varmeta.length, colDataGetVarData(pSrc, srcIdx), allLen);
}
pDst->varmeta.length = pDst->varmeta.length + allLen;
}
} else {
if (pSrc->hasNull) {
if (BitPos(dstIdx) == BitPos(srcIdx)) {
for (int32_t i = 0; i < numOfRows; ++i) {
if (0 == BitPos(dstIdx) && (i + (1 << NBIT) <= numOfRows)) {
BMCharPos(pDst->nullbitmap, dstIdx + i) = BMCharPos(pSrc->nullbitmap, srcIdx + i);
if (BMCharPos(pDst->nullbitmap, dstIdx + i)) {
pDst->hasNull = true;
}
i += (1 << NBIT) - 1;
} else {
if (colDataIsNull_f(pSrc->nullbitmap, srcIdx + i)) {
colDataSetNull_f(pDst->nullbitmap, dstIdx + i);
pDst->hasNull = true;
} else {
colDataClearNull_f(pDst->nullbitmap, dstIdx + i);
}
}
}
} else {
for (int32_t i = 0; i < numOfRows; ++i) {
if (colDataIsNull_f(pSrc->nullbitmap, srcIdx + i)) {
colDataSetNull_f(pDst->nullbitmap, dstIdx + i);
pDst->hasNull = true;
} else {
colDataClearNull_f(pDst->nullbitmap, dstIdx + i);
}
}
}
}
if (pSrc->pData != NULL) {
memcpy(pDst->pData + pDst->info.bytes * dstIdx, pSrc->pData + pSrc->info.bytes * srcIdx, pDst->info.bytes * numOfRows);
}
}
return 0;
}
size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { return taosArrayGetSize(pBlock->pDataBlock); }
size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; }
@ -574,6 +689,60 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) {
return TSDB_CODE_SUCCESS;
}
int32_t blockDataMergeNRows(SSDataBlock* pDest, const SSDataBlock* pSrc, int32_t srcIdx, int32_t numOfRows) {
if (pDest->info.rows + numOfRows > pDest->info.capacity) {
ASSERT(0);
return TSDB_CODE_FAILED;
}
size_t numOfCols = taosArrayGetSize(pDest->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i);
SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i);
colDataAssignNRows(pCol2, pDest->info.rows, pCol1, srcIdx, numOfRows);
}
pDest->info.rows += numOfRows;
return TSDB_CODE_SUCCESS;
}
void blockDataShrinkNRows(SSDataBlock* pBlock, int32_t numOfRows) {
if (numOfRows >= pBlock->info.rows) {
blockDataCleanup(pBlock);
return;
}
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i);
if (IS_VAR_DATA_TYPE(pCol->info.type)) {
pCol->varmeta.length = pCol->varmeta.offset[pBlock->info.rows - numOfRows];
memset(pCol->varmeta.offset + pBlock->info.rows - numOfRows, 0, sizeof(*pCol->varmeta.offset) * numOfRows);
} else {
int32_t i = pBlock->info.rows - numOfRows;
for (; i < pBlock->info.rows; ++i) {
if (BitPos(i)) {
colDataClearNull_f(pCol->nullbitmap, i);
} else {
break;
}
}
int32_t bytes = (pBlock->info.rows - i) / 8;
memset(&BMCharPos(pCol->nullbitmap, i), 0, bytes);
i += bytes * 8;
for (; i < pBlock->info.rows; ++i) {
colDataClearNull_f(pCol->nullbitmap, i);
}
}
}
pBlock->info.rows -= numOfRows;
}
size_t blockDataGetSize(const SSDataBlock* pBlock) {
size_t total = 0;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
@ -653,25 +822,26 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3
return NULL;
}
SSDataBlock* pDst = createDataBlock();
SSDataBlock* pDst = createOneDataBlock(pBlock, false);
if (pDst == NULL) {
return NULL;
}
pDst->info = pBlock->info;
pDst->info.rows = 0;
pDst->info.capacity = 0;
pDst->info.rowSize = 0;
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData colInfo = {0};
SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i);
colInfo.info = pSrcCol->info;
blockDataAppendColInfo(pDst, &colInfo);
}
blockDataEnsureCapacity(pDst, rowCount);
/* may have disorder varchar data, TODO
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
colDataAssignNRows(pDstCol, 0, pColData, startIndex, rowCount);
}
*/
size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i);
SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i);
@ -1206,18 +1376,43 @@ void blockDataEmpty(SSDataBlock* pDataBlock) {
pInfo->window.skey = 0;
}
void blockDataReset(SSDataBlock* pDataBlock) {
SDataBlockInfo* pInfo = &pDataBlock->info;
if (pInfo->capacity == 0) {
return;
}
size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock);
for (int32_t i = 0; i < numOfCols; ++i) {
SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i);
p->hasNull = false;
p->reassigned = false;
if (IS_VAR_DATA_TYPE(p->info.type)) {
p->varmeta.length = 0;
}
}
pInfo->rows = 0;
pInfo->dataLoad = 0;
pInfo->window.ekey = 0;
pInfo->window.skey = 0;
pInfo->id.uid = 0;
pInfo->id.groupId = 0;
}
/*
* NOTE: the type of the input column may be TSDB_DATA_TYPE_NULL, which is used to denote
* the all NULL value in this column. It is an internal representation of all NULL value column, and no visible to
* any users. The length of TSDB_DATA_TYPE_NULL is 0, and it is an special case.
*/
static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo* pBlockInfo, uint32_t numOfRows,
bool clearPayload) {
if (numOfRows <= 0 || numOfRows <= pBlockInfo->capacity) {
if (numOfRows <= 0 || pBlockInfo && numOfRows <= pBlockInfo->capacity) {
return TSDB_CODE_SUCCESS;
}
int32_t existedRows = pBlockInfo->rows;
int32_t existedRows = pBlockInfo ? pBlockInfo->rows : 0;
if (IS_VAR_DATA_TYPE(pColumn->info.type)) {
char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows);
@ -2505,6 +2700,8 @@ void trimDataBlock(SSDataBlock* pBlock, int32_t totalRows, const bool* pBoolList
int32_t numOfRows = 0;
if (IS_VAR_DATA_TYPE(pDst->info.type)) {
pDst->varmeta.length = 0;
} else {
memset(pDst->nullbitmap, 0, bmLen);
}
}
return;

21
source/common/src/tglobal.c
View File

@ -298,6 +298,9 @@ int32_t tsS3UploadDelaySec = 60;
bool tsExperimental = true;
int32_t tsMaxTsmaNum = 8;
int32_t tsMaxTsmaCalcDelay = 600;
#ifndef _STORAGE
int32_t taosSetTfsCfg(SConfig *pCfg) {
SConfigItem *pItem = cfgGetItem(pCfg, "dataDir");
@ -553,6 +556,11 @@ static int32_t taosAddClientCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "monitorInterval", tsMonitorInterval, 1, 200000, CFG_SCOPE_BOTH, CFG_DYN_NONE) != 0) return -1;
if (cfgAddBool(pCfg, "multiResultFunctionStarReturnTags", tsMultiResultFunctionStarReturnTags, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0) return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "maxTsmaCalcDelay", tsMaxTsmaCalcDelay, 600, 86400, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) !=
0)
return -1;
return 0;
}
@ -607,8 +615,6 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "minIntervalTime", tsMinIntervalTime, 1, 1000000, CFG_SCOPE_CLIENT, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "countAlwaysReturnValue", tsCountAlwaysReturnValue, 0, 1, CFG_SCOPE_BOTH, CFG_DYN_CLIENT) != 0)
return -1;
if (cfgAddInt32(pCfg, "queryBufferSize", tsQueryBufferSize, -1, 500000000000, CFG_SCOPE_SERVER, CFG_DYN_NONE) != 0)
return -1;
if (cfgAddInt32(pCfg, "queryRspPolicy", tsQueryRspPolicy, 0, 1, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0) return -1;
@ -732,6 +738,7 @@ static int32_t taosAddServerCfg(SConfig *pCfg) {
if (cfgAddInt32(pCfg, "tmqRowSize", tmqRowSize, 1, 1000000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) != 0)
return -1;
if (cfgAddInt32(pCfg, "maxTsmaNum", tsMaxTsmaNum, 0, 12, CFG_SCOPE_SERVER, CFG_DYN_SERVER) != 0) return -1;
if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, CFG_SCOPE_SERVER, CFG_DYN_ENT_SERVER) !=
0)
return -1;
@ -1109,6 +1116,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) {
return -1;
}
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32;
@ -1122,6 +1130,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) {
tsExperimental = cfgGetItem(pCfg, "experimental")->bval;
tsMultiResultFunctionStarReturnTags = cfgGetItem(pCfg, "multiResultFunctionStarReturnTags")->bval;
tsMaxTsmaCalcDelay = cfgGetItem(pCfg, "maxTsmaCalcDelay")->i32;
return 0;
}
@ -1153,7 +1162,6 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tsStatusInterval = cfgGetItem(pCfg, "statusInterval")->i32;
tsMinSlidingTime = cfgGetItem(pCfg, "minSlidingTime")->i32;
tsMinIntervalTime = cfgGetItem(pCfg, "minIntervalTime")->i32;
tsCountAlwaysReturnValue = cfgGetItem(pCfg, "countAlwaysReturnValue")->i32;
tsQueryBufferSize = cfgGetItem(pCfg, "queryBufferSize")->i32;
tsNumOfRpcThreads = cfgGetItem(pCfg, "numOfRpcThreads")->i32;
@ -1203,6 +1211,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) {
tmqMaxTopicNum = cfgGetItem(pCfg, "tmqMaxTopicNum")->i32;
tmqRowSize = cfgGetItem(pCfg, "tmqRowSize")->i32;
tsMaxTsmaNum = cfgGetItem(pCfg, "maxTsmaNum")->i32;
tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32;
tsCompactPullupInterval = cfgGetItem(pCfg, "compactPullupInterval")->i32;
@ -1548,7 +1557,8 @@ static int32_t taosCfgDynamicOptionsForServer(SConfig *pCfg, char *name) {
{"s3PageCacheSize", &tsS3PageCacheSize},
{"s3UploadDelaySec", &tsS3UploadDelaySec},
{"supportVnodes", &tsNumOfSupportVnodes},
{"experimental", &tsExperimental}};
{"experimental", &tsExperimental},
{"maxTsmaNum", &tsMaxTsmaNum}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false);
@ -1766,7 +1776,8 @@ static int32_t taosCfgDynamicOptionsForClient(SConfig *pCfg, char *name) {
{"slowLogThreshold", &tsSlowLogThreshold},
{"useAdapter", &tsUseAdapter},
{"experimental", &tsExperimental},
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags} };
{"multiResultFunctionStarReturnTags", &tsMultiResultFunctionStarReturnTags},
{"maxTsmaCalcDelay", &tsMaxTsmaCalcDelay}};
if (taosCfgSetOption(debugOptions, tListLen(debugOptions), pItem, true) != 0) {
taosCfgSetOption(options, tListLen(options), pItem, false);

503
source/common/src/tmsg.c
View File

@ -67,6 +67,8 @@
static int32_t tDecodeSVAlterTbReqCommon(SDecoder *pDecoder, SVAlterTbReq *pReq);
static int32_t tDecodeSBatchDeleteReqCommon(SDecoder *pDecoder, SBatchDeleteReq *pReq);
static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoRsp *pRsp);
static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pRsp);
int32_t tInitSubmitMsgIter(const SSubmitReq *pMsg, SSubmitMsgIter *pIter) {
if (pMsg == NULL) {
@ -887,6 +889,16 @@ int32_t tSerializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pReq
}
if (tEncodeI64(&encoder, pReq->deleteMark) < 0) return -1;
if (tEncodeI64(&encoder, pReq->lastTs) < 0) return -1;
if (tEncodeI64(&encoder, pReq->normSourceTbUid) < 0) return -1;
if (tEncodeI32(&encoder, taosArrayGetSize(pReq->pVgroupVerList)) < 0) return -1;
for(int32_t i = 0; i < taosArrayGetSize(pReq->pVgroupVerList); ++i) {
SVgroupVer* p = taosArrayGet(pReq->pVgroupVerList, i);
if (tEncodeI32(&encoder, p->vgId) < 0) return -1;
if (tEncodeI64(&encoder, p->ver) < 0) return -1;
}
if (tEncodeI8(&encoder, pReq->recursiveTsma) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->baseTsmaName) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -937,6 +949,29 @@ int32_t tDeserializeSMCreateSmaReq(void *buf, int32_t bufLen, SMCreateSmaReq *pR
}
if (tDecodeI64(&decoder, &pReq->deleteMark) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->lastTs) < 0) return -1;
if (tDecodeI64(&decoder, &pReq->normSourceTbUid) < 0) return -1;
int32_t numOfVgVer;
if (tDecodeI32(&decoder, &numOfVgVer) < 0) return -1;
if (numOfVgVer > 0) {
pReq->pVgroupVerList = taosArrayInit(numOfVgVer, sizeof(SVgroupVer));
if (pReq->pVgroupVerList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < numOfVgVer; ++i) {
SVgroupVer v = {0};
if (tDecodeI32(&decoder, &v.vgId) < 0) return -1;
if (tDecodeI64(&decoder, &v.ver) < 0) return -1;
if (taosArrayPush(pReq->pVgroupVerList, &v) == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
}
}
if (tDecodeI8(&decoder, &pReq->recursiveTsma) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->baseTsmaName) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
@ -947,6 +982,7 @@ void tFreeSMCreateSmaReq(SMCreateSmaReq *pReq) {
taosMemoryFreeClear(pReq->tagsFilter);
taosMemoryFreeClear(pReq->sql);
taosMemoryFreeClear(pReq->ast);
taosArrayDestroy(pReq->pVgroupVerList);
}
int32_t tSerializeSMDropSmaReq(void *buf, int32_t bufLen, SMDropSmaReq *pReq) {
@ -3632,6 +3668,14 @@ int32_t tSerializeSDbHbRspImp(SEncoder *pEncoder, const SDbHbRsp *pRsp) {
if (tEncodeI8(pEncoder, 0) < 0) return -1;
}
if (pRsp->pTsmaRsp) {
if (tEncodeI8(pEncoder, 1) < 0) return -1;
if (tEncodeTableTSMAInfoRsp(pEncoder, pRsp->pTsmaRsp) < 0) return -1;
} else {
if (tEncodeI8(pEncoder, 0) < 0) return -1;
}
if (tEncodeI32(pEncoder, pRsp->dbTsmaVersion) < 0) return -1;
return 0;
}
@ -3712,6 +3756,17 @@ int32_t tDeserializeSDbHbRspImp(SDecoder *decoder, SDbHbRsp *pRsp) {
if (NULL == pRsp->cfgRsp) return -1;
if (tDeserializeSDbCfgRspImpl(decoder, pRsp->cfgRsp) < 0) return -1;
}
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI8(decoder, &flag) < 0) return -1;
if (flag) {
pRsp->pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (!pRsp->pTsmaRsp) return -1;
if (tDecodeTableTSMAInfoRsp(decoder, pRsp->pTsmaRsp) < 0) return -1;
}
}
if (!tDecodeIsEnd(decoder)) {
if (tDecodeI32(decoder, &pRsp->dbTsmaVersion) < 0) return -1;
}
return 0;
}
@ -3761,6 +3816,10 @@ void tFreeSDbHbRsp(SDbHbRsp *pDbRsp) {
tFreeSDbCfgRsp(pDbRsp->cfgRsp);
taosMemoryFree(pDbRsp->cfgRsp);
}
if (pDbRsp->pTsmaRsp) {
tFreeTableTSMAInfoRsp(pDbRsp->pTsmaRsp);
taosMemoryFree(pDbRsp->pTsmaRsp);
}
}
void tFreeSDbHbBatchRsp(SDbHbBatchRsp *pRsp) {
@ -7763,6 +7822,7 @@ int32_t tSerializeSCMCreateStreamReq(void *buf, int32_t bufLen, const SCMCreateS
if (tEncodeCStr(&encoder, pField->name) < 0) return -1;
}
if (tEncodeI64(&encoder, pReq->smaId) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
@ -7889,6 +7949,9 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea
}
}
}
if (!tDecodeIsEnd(&decoder)) {
if (tDecodeI64(&decoder, &pReq->smaId)< 0) return -1;
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
@ -9936,3 +9999,443 @@ void tFreeSViewHbRsp(SViewHbRsp *pRsp) {
taosArrayDestroy(pRsp->pViewRsp);
}
int32_t tSerializeTableTSMAInfoReq(void* buf, int32_t bufLen, const STableTSMAInfoReq* pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeCStr(&encoder, pReq->name) < 0) return -1;
if (tEncodeI8(&encoder, pReq->fetchingWithTsmaName) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeTableTSMAInfoReq(void* buf, int32_t bufLen, STableTSMAInfoReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1;
if (tDecodeI8(&decoder, (uint8_t*)&pReq->fetchingWithTsmaName) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
static int32_t tEncodeTableTSMAInfo(SEncoder* pEncoder, const STableTSMAInfo* pTsmaInfo) {
if (tEncodeCStr(pEncoder, pTsmaInfo->name) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->tsmaId) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->tb) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->dbFName) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->suid) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->destTbUid) < 0) return -1;
if (tEncodeU64(pEncoder, pTsmaInfo->dbId) < 0) return -1;
if (tEncodeI32(pEncoder, pTsmaInfo->version) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->targetTb) < 0) return -1;
if (tEncodeCStr(pEncoder, pTsmaInfo->targetDbFName) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->interval) < 0) return -1;
if (tEncodeI8(pEncoder, pTsmaInfo->unit) < 0) return -1;
int32_t size = pTsmaInfo->pFuncs ? pTsmaInfo->pFuncs->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAFuncInfo* pFuncInfo = taosArrayGet(pTsmaInfo->pFuncs, i);
if (tEncodeI32(pEncoder, pFuncInfo->funcId) < 0) return -1;
if (tEncodeI16(pEncoder, pFuncInfo->colId) < 0) return -1;
}
size = pTsmaInfo->pTags ? pTsmaInfo->pTags->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
const SSchema* pSchema = taosArrayGet(pTsmaInfo->pTags, i);
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
}
size = pTsmaInfo->pUsedCols ? pTsmaInfo->pUsedCols->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
const SSchema* pSchema = taosArrayGet(pTsmaInfo->pUsedCols, i);
if (tEncodeSSchema(pEncoder, pSchema) < 0) return -1;
}
if (tEncodeCStr(pEncoder, pTsmaInfo->ast) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->streamUid) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->reqTs) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->rspTs) < 0) return -1;
if (tEncodeI64(pEncoder, pTsmaInfo->delayDuration) < 0) return -1;
if (tEncodeI8(pEncoder, pTsmaInfo->fillHistoryFinished) < 0) return -1;
return 0;
}
static int32_t tDecodeTableTSMAInfo(SDecoder* pDecoder, STableTSMAInfo* pTsmaInfo) {
if (tDecodeCStrTo(pDecoder, pTsmaInfo->name) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->tsmaId) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->tb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->dbFName) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->suid) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->destTbUid) < 0) return -1;
if (tDecodeU64(pDecoder, &pTsmaInfo->dbId) < 0) return -1;
if (tDecodeI32(pDecoder, &pTsmaInfo->version) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->targetTb) < 0) return -1;
if (tDecodeCStrTo(pDecoder, pTsmaInfo->targetDbFName) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->interval) < 0) return -1;
if (tDecodeI8(pDecoder, &pTsmaInfo->unit) < 0) return -1;
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pTsmaInfo->pFuncs = taosArrayInit(size, sizeof(STableTSMAFuncInfo));
if (!pTsmaInfo->pFuncs) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAFuncInfo funcInfo = {0};
if (tDecodeI32(pDecoder, &funcInfo.funcId) < 0) return -1;
if (tDecodeI16(pDecoder, &funcInfo.colId) < 0) return -1;
if (!taosArrayPush(pTsmaInfo->pFuncs, &funcInfo)) return -1;
}
}
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pTsmaInfo->pTags = taosArrayInit(size, sizeof(SSchema));
if (!pTsmaInfo->pTags) return -1;
for (int32_t i = 0; i < size; ++i) {
SSchema schema = {0};
if(tDecodeSSchema(pDecoder, &schema) < 0) return -1;
taosArrayPush(pTsmaInfo->pTags, &schema);
}
}
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size > 0) {
pTsmaInfo->pUsedCols = taosArrayInit(size, sizeof(SSchema));
if (!pTsmaInfo->pUsedCols) return -1;
for (int32_t i = 0; i < size; ++i) {
SSchema schema = {0};
if (tDecodeSSchema(pDecoder, &schema) < 0) return -1;
taosArrayPush(pTsmaInfo->pUsedCols, &schema);
}
}
if (tDecodeCStrAlloc(pDecoder, &pTsmaInfo->ast) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->streamUid) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->reqTs) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->rspTs) < 0) return -1;
if (tDecodeI64(pDecoder, &pTsmaInfo->delayDuration) < 0) return -1;
if (tDecodeI8(pDecoder, (int8_t*)&pTsmaInfo->fillHistoryFinished) < 0) return -1;
return 0;
}
static int32_t tEncodeTableTSMAInfoRsp(SEncoder *pEncoder, const STableTSMAInfoRsp *pRsp) {
int32_t size = pRsp->pTsmas ? pRsp->pTsmas->size : 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAInfo* pInfo = taosArrayGetP(pRsp->pTsmas, i);
if (tEncodeTableTSMAInfo(pEncoder, pInfo) < 0) return -1;
}
return 0;
}
static int32_t tDecodeTableTSMAInfoRsp(SDecoder* pDecoder, STableTSMAInfoRsp* pRsp) {
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
if (size <= 0) return 0;
pRsp->pTsmas = taosArrayInit(size, POINTER_BYTES);
if (!pRsp->pTsmas) return -1;
for (int32_t i = 0; i < size; ++i) {
STableTSMAInfo *pTsma = taosMemoryCalloc(1, sizeof(STableTSMAInfo));
if (!pTsma) return -1;
taosArrayPush(pRsp->pTsmas, &pTsma);
if (tDecodeTableTSMAInfo(pDecoder, pTsma) < 0) return -1;
}
return 0;
}
int32_t tSerializeTableTSMAInfoRsp(void* buf, int32_t bufLen, const STableTSMAInfoRsp* pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeTableTSMAInfoRsp(&encoder, pRsp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeTableTSMAInfoRsp(void* buf, int32_t bufLen, STableTSMAInfoRsp* pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeTableTSMAInfoRsp(&decoder, pRsp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeTableTSMAInfo(void* p) {
STableTSMAInfo *pTsmaInfo = p;
if (pTsmaInfo) {
taosArrayDestroy(pTsmaInfo->pFuncs);
taosArrayDestroy(pTsmaInfo->pTags);
taosArrayDestroy(pTsmaInfo->pUsedCols);
taosMemoryFree(pTsmaInfo->ast);
}
}
void tFreeAndClearTableTSMAInfo(void* p) {
STableTSMAInfo* pTsmaInfo = (STableTSMAInfo*)p;
if (pTsmaInfo) {
tFreeTableTSMAInfo(pTsmaInfo);
taosMemoryFree(pTsmaInfo);
}
}
int32_t tCloneTbTSMAInfo(STableTSMAInfo* pInfo, STableTSMAInfo** pRes) {
int32_t code = TSDB_CODE_SUCCESS;
if (NULL == pInfo) {
return TSDB_CODE_SUCCESS;
}
STableTSMAInfo* pRet = taosMemoryCalloc(1, sizeof(STableTSMAInfo));
if (!pRet) return TSDB_CODE_OUT_OF_MEMORY;
*pRet = *pInfo;
if (pInfo->pFuncs) {
pRet->pFuncs = taosArrayDup(pInfo->pFuncs, NULL);
if (!pRet->pFuncs) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->pTags && code == TSDB_CODE_SUCCESS) {
pRet->pTags = taosArrayDup(pInfo->pTags, NULL);
if (!pRet->pTags) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->pUsedCols && code == TSDB_CODE_SUCCESS) {
pRet->pUsedCols = taosArrayDup(pInfo->pUsedCols, NULL);
if (!pRet->pUsedCols) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (pInfo->ast && code == TSDB_CODE_SUCCESS) {
pRet->ast = taosStrdup(pInfo->ast);
if (!pRet->ast) code = TSDB_CODE_OUT_OF_MEMORY;
}
if (code) tFreeTableTSMAInfo(pRet);
*pRes = pRet;
return code;
}
void tFreeTableTSMAInfoRsp(STableTSMAInfoRsp *pRsp) {
if (pRsp && pRsp->pTsmas) {
taosArrayDestroyP(pRsp->pTsmas, tFreeAndClearTableTSMAInfo);
}
}
static int32_t tEncodeStreamProgressReq(SEncoder *pEncoder, const SStreamProgressReq *pReq) {
if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->vgId) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->fetchIdx) < 0) return -1;
if (tEncodeI32(pEncoder, pReq->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tSerializeStreamProgressReq(void* buf, int32_t bufLen, const SStreamProgressReq* pReq) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeStreamProgressReq(&encoder, pReq) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
static int32_t tDecodeStreamProgressReq(SDecoder* pDecoder, SStreamProgressReq* pReq) {
if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->vgId) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->fetchIdx) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tDeserializeStreamProgressReq(void* buf, int32_t bufLen, SStreamProgressReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, (char *)buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeStreamProgressReq(&decoder, pReq) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
static int32_t tEncodeStreamProgressRsp(SEncoder* pEncoder, const SStreamProgressRsp* pRsp) {
if (tEncodeI64(pEncoder, pRsp->streamId) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->vgId) < 0) return -1;
if (tEncodeI8(pEncoder, pRsp->fillHisFinished) < 0) return -1;
if (tEncodeI64(pEncoder, pRsp->progressDelay) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->fetchIdx) < 0) return -1;
if (tEncodeI32(pEncoder, pRsp->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tSerializeStreamProgressRsp(void* buf, int32_t bufLen, const SStreamProgressRsp* pRsp) {
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
if (tStartEncode(&encoder) < 0) return -1;
if (tEncodeStreamProgressRsp(&encoder, pRsp) < 0) return -1;
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
static int32_t tDecodeStreamProgressRsp(SDecoder* pDecoder, SStreamProgressRsp* pRsp) {
if (tDecodeI64(pDecoder, &pRsp->streamId) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->vgId) < 0) return -1;
if (tDecodeI8(pDecoder, (int8_t*)&pRsp->fillHisFinished) < 0) return -1;
if (tDecodeI64(pDecoder, &pRsp->progressDelay) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->fetchIdx) < 0) return -1;
if (tDecodeI32(pDecoder, &pRsp->subFetchIdx) < 0) return -1;
return 0;
}
int32_t tDeserializeSStreamProgressRsp(void* buf, int32_t bufLen, SStreamProgressRsp* pRsp) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
if (tStartDecode(&decoder) < 0) return -1;
if (tDecodeStreamProgressRsp(&decoder, pRsp) < 0) return -1;
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
int32_t tEncodeSMDropTbReqOnSingleVg(SEncoder *pEncoder, const SMDropTbReqsOnSingleVg *pReq) {
const SVgroupInfo* pVgInfo = &pReq->vgInfo;
if (tEncodeI32(pEncoder, pVgInfo->vgId) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashBegin) < 0) return -1;
if (tEncodeU32(pEncoder, pVgInfo->hashEnd) < 0) return -1;
if (tEncodeSEpSet(pEncoder, &pVgInfo->epSet) < 0) return -1;
if (tEncodeI32(pEncoder, pVgInfo->numOfTable) < 0) return -1;
int32_t size = pReq->pTbs ? pReq->pTbs->size: 0;
if (tEncodeI32(pEncoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
const SVDropTbReq* pInfo = taosArrayGet(pReq->pTbs, i);
if (tEncodeSVDropTbReq(pEncoder, pInfo) < 0) return -1;
}
return 0;
}
int32_t tDecodeSMDropTbReqOnSingleVg(SDecoder* pDecoder, SMDropTbReqsOnSingleVg* pReq) {
if (tDecodeI32(pDecoder, &pReq->vgInfo.vgId) < 0) return -1;
if (tDecodeU32(pDecoder, &pReq->vgInfo.hashBegin) < 0) return -1;
if (tDecodeU32(pDecoder, &pReq->vgInfo.hashEnd) < 0) return -1;
if (tDecodeSEpSet(pDecoder, &pReq->vgInfo.epSet) < 0) return -1;
if (tDecodeI32(pDecoder, &pReq->vgInfo.numOfTable) < 0) return -1;
int32_t size = 0;
if (tDecodeI32(pDecoder, &size) < 0) return -1;
pReq->pTbs = taosArrayInit(size, sizeof(SVDropTbReq));
if (!pReq->pTbs) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
SVDropTbReq pTbReq = {0};
for (int32_t i = 0; i < size; ++i) {
if (tDecodeSVDropTbReq(pDecoder, &pTbReq) < 0) return -1;
taosArrayPush(pReq->pTbs, &pTbReq);
}
return 0;
}
void tFreeSMDropTbReqOnSingleVg(void *p) {
SMDropTbReqsOnSingleVg* pReq = p;
taosArrayDestroy(pReq->pTbs);
}
int32_t tSerializeSMDropTbsReq(void* buf, int32_t bufLen, const SMDropTbsReq* pReq){
SEncoder encoder = {0};
tEncoderInit(&encoder, buf, bufLen);
tStartEncode(&encoder);
int32_t size = pReq->pVgReqs ? pReq->pVgReqs->size : 0;
if (tEncodeI32(&encoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
SMDropTbReqsOnSingleVg* pVgReq = taosArrayGet(pReq->pVgReqs, i);
if (tEncodeSMDropTbReqOnSingleVg(&encoder, pVgReq) < 0) return -1;
}
tEndEncode(&encoder);
int32_t tlen = encoder.pos;
tEncoderClear(&encoder);
return tlen;
}
int32_t tDeserializeSMDropTbsReq(void* buf, int32_t bufLen, SMDropTbsReq* pReq) {
SDecoder decoder = {0};
tDecoderInit(&decoder, buf, bufLen);
tStartDecode(&decoder);
int32_t size = 0;
if (tDecodeI32(&decoder, &size) < 0) return -1;
pReq->pVgReqs = taosArrayInit(size, sizeof(SMDropTbReqsOnSingleVg));
if (!pReq->pVgReqs) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return -1;
}
for (int32_t i = 0; i < size; ++i) {
SMDropTbReqsOnSingleVg vgReq = {0};
tDecodeSMDropTbReqOnSingleVg(&decoder, &vgReq);
taosArrayPush(pReq->pVgReqs, &vgReq);
}
tEndDecode(&decoder);
tDecoderClear(&decoder);
return 0;
}
void tFreeSMDropTbsReq(void* p) {
SMDropTbsReq* pReq = p;
taosArrayDestroyEx(pReq->pVgReqs, tFreeSMDropTbReqOnSingleVg);
}
int32_t tEncodeVFetchTtlExpiredTbsRsp(SEncoder* pCoder, const SVFetchTtlExpiredTbsRsp* pRsp) {
if (tEncodeI32(pCoder, pRsp->vgId) < 0) return -1;
int32_t size = pRsp->pExpiredTbs ? pRsp->pExpiredTbs->size : 0;
if (tEncodeI32(pCoder, size) < 0) return -1;
for (int32_t i = 0; i < size; ++i) {
if (tEncodeSVDropTbReq(pCoder, taosArrayGet(pRsp->pExpiredTbs, i)) < 0) return -1;
}
return 0;
}
int32_t tDecodeVFetchTtlExpiredTbsRsp(SDecoder* pCoder, SVFetchTtlExpiredTbsRsp* pRsp) {
if (tDecodeI32(pCoder, &pRsp->vgId) < 0) return -1;
int32_t size = 0;
if (tDecodeI32(pCoder, &size) < 0) return -1;
if (size > 0) {
pRsp->pExpiredTbs = taosArrayInit(size, sizeof(SVDropTbReq));
if (!pRsp->pExpiredTbs) return TSDB_CODE_OUT_OF_MEMORY;
SVDropTbReq tb = {0};
for (int32_t i = 0; i < size; ++i) {
if (tDecodeSVDropTbReq(pCoder, &tb) < 0) return -1;
taosArrayPush(pRsp->pExpiredTbs, &tb);
}
}
return 0;
}
void tFreeFetchTtlExpiredTbsRsp(void* p) {
SVFetchTtlExpiredTbsRsp* pRsp = p;
taosArrayDestroy(pRsp->pExpiredTbs);
}

6
source/common/src/ttime.c
View File

@ -581,7 +581,7 @@ int32_t convertStringToTimestamp(int16_t type, char* inputData, int64_t timePrec
return TSDB_CODE_SUCCESS;
}
static int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
int32_t getDuration(int64_t val, char unit, int64_t* result, int32_t timePrecision) {
switch (unit) {
case 's':
if (val > INT64_MAX / MILLISECOND_PER_SECOND) {
@ -663,12 +663,12 @@ int32_t parseAbsoluteDuration(const char* token, int32_t tokenlen, int64_t* dura
return getDuration(timestamp, *unit, duration, timePrecision);
}
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision) {
int32_t parseNatualDuration(const char* token, int32_t tokenLen, int64_t* duration, char* unit, int32_t timePrecision, bool negativeAllow) {
errno = 0;
/* get the basic numeric value */
*duration = taosStr2Int64(token, NULL, 10);
if (*duration < 0 || errno != 0) {
if ((*duration < 0 && !negativeAllow) || errno != 0) {
return -1;
}

12
source/dnode/mgmt/mgmt_mnode/src/mmHandle.c
View File

@ -164,6 +164,14 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_PAUSE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RESUME_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STB_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STB_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_TB_WITH_TSMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TABLE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_RETRIEVE_IP_WHITE, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_GET_USER_WHITELIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER;
@ -222,6 +230,10 @@ SArray *mmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_PAUSE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RESUME_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_STOP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_CREATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_STREAM_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_UPDATE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

1
source/dnode/mgmt/mgmt_snode/src/smHandle.c
View File

@ -91,6 +91,7 @@ SArray *smGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, smPutNodeMsgToStreamQueue, 1) == NULL) goto _OVER;
code = 0;
_OVER:

2
source/dnode/mgmt/mgmt_vnode/src/vmHandle.c
View File

@ -884,6 +884,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_SCH_TASK_NOTIFY, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TTL_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_FETCH_TTL_EXPIRED_TBS, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_STB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_CREATE_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
@ -930,6 +931,7 @@ SArray *vmGetMsgHandles() {
if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_RESET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_HEARTBEAT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_MND_STREAM_REQ_CHKPT_RSP, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_GET_STREAM_PROGRESS, vmPutMsgToStreamQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER;
if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER;

1
source/dnode/mnode/impl/inc/mndDb.h
View File

@ -29,6 +29,7 @@ void mndReleaseDb(SMnode *pMnode, SDbObj *pDb);
int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs, void **ppRsp, int32_t *pRspLen);
int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUseDbReq *pReq);
bool mndIsDbReady(SMnode *pMnode, SDbObj *pDb);
void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList);
SSdbRaw *mndDbActionEncode(SDbObj *pDb);
const char *mndGetDbStr(const char *src);

3
source/dnode/mnode/impl/inc/mndDef.h
View File

@ -398,6 +398,7 @@ typedef struct {
SRWLatch lock;
int64_t stateTs;
int64_t compactStartTime;
int32_t tsmaVersion;
} SDbObj;
typedef struct {
@ -457,12 +458,14 @@ typedef struct {
int32_t tagsFilterLen;
int32_t sqlLen;
int32_t astLen;
int32_t version;
char* expr;
char* tagsFilter;
char* sql;
char* ast;
SSchemaWrapper schemaRow; // for dstVgroup
SSchemaWrapper schemaTag; // for dstVgroup
char baseSmaName[TSDB_TABLE_FNAME_LEN];
} SSmaObj;
typedef struct {

3
source/dnode/mnode/impl/inc/mndSma.h
View File

@ -29,6 +29,9 @@ void mndReleaseSma(SMnode *pMnode, SSmaObj *pSma);
int32_t mndDropSmasByStb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *pStb);
int32_t mndDropSmasByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb);
int32_t mndGetTableSma(SMnode *pMnode, char *tbFName, STableIndexRsp *rsp, bool *exist);
int32_t mndValidateTSMAInfo(SMnode *pMnode, STSMAVersion *pTsmaVersions, int32_t numOfTsmas, void **ppRsp,
int32_t *pRspLen);
int32_t mndGetDbTsmas(SMnode *pMnode, const char *dbFName, uint64_t dbUid, STableTSMAInfoRsp *pRsp, bool *exist);
#ifdef __cplusplus
}

22
source/dnode/mnode/impl/src/mndDb.c
View File

@ -36,7 +36,7 @@
#include "tjson.h"
#define DB_VER_NUMBER 1
#define DB_RESERVE_SIZE 32
#define DB_RESERVE_SIZE 28
static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw);
static int32_t mndDbActionInsert(SSdb *pSdb, SDbObj *pDb);
@ -146,6 +146,7 @@ SSdbRaw *mndDbActionEncode(SDbObj *pDb) {
SDB_SET_INT32(pRaw, dataPos, pDb->cfg.s3KeepLocal, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.s3Compact, _OVER)
SDB_SET_INT8(pRaw, dataPos, pDb->cfg.withArbitrator, _OVER)
SDB_SET_INT32(pRaw, dataPos, pDb->tsmaVersion, _OVER);
SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
SDB_SET_DATALEN(pRaw, dataPos, _OVER)
@ -241,6 +242,7 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) {
SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.s3KeepLocal, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.s3Compact, _OVER)
SDB_GET_INT8(pRaw, dataPos, &pDb->cfg.withArbitrator, _OVER)
SDB_GET_INT32(pRaw, dataPos, &pDb->tsmaVersion, _OVER);
SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER)
taosInitRWLatch(&pDb->lock);
@ -347,6 +349,7 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) {
pOld->cfg.s3Compact = pNew->cfg.s3Compact;
pOld->cfg.withArbitrator = pNew->cfg.withArbitrator;
pOld->compactStartTime = pNew->compactStartTime;
pOld->tsmaVersion = pNew->tsmaVersion;
taosWUnLockLatch(&pOld->lock);
return 0;
}
@ -681,6 +684,7 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate,
dbObj.uid = mndGenerateUid(dbObj.name, TSDB_DB_FNAME_LEN);
dbObj.cfgVersion = 1;
dbObj.vgVersion = 1;
dbObj.tsmaVersion = 1;
memcpy(dbObj.createUser, pUser->user, TSDB_USER_LEN);
dbObj.cfg = (SDbCfg){
.numOfVgroups = pCreate->numOfVgroups,
@ -1521,7 +1525,7 @@ static int32_t mndGetDBTableNum(SDbObj *pDb, SMnode *pMnode) {
return numOfTables;
}
static void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
void mndBuildDBVgroupInfo(SDbObj *pDb, SMnode *pMnode, SArray *pVgList) {
int32_t vindex = 0;
SSdb *pSdb = pMnode->pSdb;
@ -1682,6 +1686,7 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
pDbCacheInfo->cfgVersion = htonl(pDbCacheInfo->cfgVersion);
pDbCacheInfo->numOfTable = htonl(pDbCacheInfo->numOfTable);
pDbCacheInfo->stateTs = be64toh(pDbCacheInfo->stateTs);
pDbCacheInfo->tsmaVersion = htonl(pDbCacheInfo->tsmaVersion);
SDbHbRsp rsp = {0};
@ -1720,7 +1725,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
int32_t numOfTable = mndGetDBTableNum(pDb, pMnode);
if (pDbCacheInfo->vgVersion >= pDb->vgVersion && pDbCacheInfo->cfgVersion >= pDb->cfgVersion &&
numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs) {
numOfTable == pDbCacheInfo->numOfTable && pDbCacheInfo->stateTs == pDb->stateTs &&
pDbCacheInfo->tsmaVersion >= pDb->tsmaVersion) {
mTrace("db:%s, valid dbinfo, vgVersion:%d cfgVersion:%d stateTs:%" PRId64
" numOfTables:%d, not changed vgVersion:%d cfgVersion:%d stateTs:%" PRId64 " numOfTables:%d",
pDbCacheInfo->dbFName, pDbCacheInfo->vgVersion, pDbCacheInfo->cfgVersion, pDbCacheInfo->stateTs,
@ -1739,6 +1745,16 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbCacheInfo *pDbs, int32_t numOfDbs,
mndDumpDbCfgInfo(rsp.cfgRsp, pDb);
}
if (pDbCacheInfo->tsmaVersion != pDb->tsmaVersion) {
rsp.pTsmaRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (rsp.pTsmaRsp) rsp.pTsmaRsp->pTsmas = taosArrayInit(4, POINTER_BYTES);
if (rsp.pTsmaRsp && rsp.pTsmaRsp->pTsmas) {
rsp.dbTsmaVersion = pDb->tsmaVersion;
bool exist = false;
mndGetDbTsmas(pMnode, 0, pDb->uid, rsp.pTsmaRsp, &exist);
}
}
if (pDbCacheInfo->vgVersion < pDb->vgVersion || numOfTable != pDbCacheInfo->numOfTable ||
pDbCacheInfo->stateTs != pDb->stateTs) {
rsp.useDbRsp = taosMemoryCalloc(1, sizeof(SUseDbRsp));

11
source/dnode/mnode/impl/src/mndProfile.c
View File

@ -25,6 +25,7 @@
#include "mndStb.h"
#include "mndUser.h"
#include "mndView.h"
#include "mndSma.h"
#include "tglobal.h"
#include "tversion.h"
@ -606,6 +607,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb
break;
}
#endif
case HEARTBEAT_KEY_TSMA: {
void * rspMsg = NULL;
int32_t rspLen = 0;
mndValidateTSMAInfo(pMnode, kv->value, kv->valueLen / sizeof(STSMAVersion), &rspMsg, &rspLen);
if (rspMsg && rspLen > 0) {
SKv kv = {.key = HEARTBEAT_KEY_TSMA, .valueLen = rspLen, .value = rspMsg};
taosArrayPush(hbRsp.info, &kv);
}
break;
}
default:
mError("invalid kv key:%d", kv->key);
hbRsp.status = TSDB_CODE_APP_ERROR;

2
source/dnode/mnode/impl/src/mndScheduler.c
View File

@ -109,7 +109,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
mDebug("mndSetSinkTaskInfo to sma or table, taskId:%s", pTask->id.idStr);
if (pStream->smaId != 0) {
if (pStream->smaId != 0 && pStream->subTableWithoutMd5 != 1) {
pInfo->type = TASK_OUTPUT__SMA;
pInfo->smaSink.smaId = pStream->smaId;
} else {

2
source/dnode/mnode/impl/src/mndShow.c
View File

@ -131,6 +131,8 @@ static int32_t convertToRetrieveType(char *name, int32_t len) {
type = TSDB_MGMT_TABLE_GRANTS_LOGS;
} else if (strncasecmp(name, TSDB_INS_TABLE_MACHINES, len) == 0) {
type = TSDB_MGMT_TABLE_MACHINES;
} else if (strncasecmp(name, TSDB_INS_TABLE_TSMAS, len) == 0) {
type = TSDB_MGMT_TABLE_TSMAS;
} else {
mError("invalid show name:%s len:%d", name, len);
}

1109
source/dnode/mnode/impl/src/mndSma.c
View File

File diff suppressed because it is too large Load Diff

394
source/dnode/mnode/impl/src/mndStb.c
View File

@ -62,6 +62,10 @@ static int32_t mndAlterStbAndUpdateTagIdxImp(SMnode *pMnode, SRpcMsg *pReq, SDb
static int32_t mndProcessCreateIndexReq(SRpcMsg *pReq);
static int32_t mndProcessDropIndexReq(SRpcMsg *pReq);
static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessDropTbWithTsma(SRpcMsg* pReq);
static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pReq);
int32_t mndInitStb(SMnode *pMnode) {
SSdbTable table = {
.sdbType = SDB_STB,
@ -87,6 +91,11 @@ int32_t mndInitStb(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_S3MIGRATE_RSP, mndProcessS3MigrateDbRsp);
mndSetMsgHandle(pMnode, TDMT_MND_S3MIGRATE_DB_TIMER, mndProcessS3MigrateDbTimer);
mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq);
mndSetMsgHandle(pMnode, TDMT_MND_STB_DROP, mndProcessDropStbReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_MND_STB_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_DROP_TB_WITH_TSMA, mndProcessDropTbWithTsma);
mndSetMsgHandle(pMnode, TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP, mndProcessFetchTtlExpiredTbs);
mndSetMsgHandle(pMnode, TDMT_VND_DROP_TABLE_RSP, mndTransProcessRsp);
// mndSetMsgHandle(pMnode, TDMT_MND_SYSTABLE_RETRIEVE, mndProcessRetrieveStbReq);
// mndSetMsgHandle(pMnode, TDMT_MND_CREATE_INDEX, mndProcessCreateIndexReq);
@ -936,7 +945,7 @@ static int32_t mndProcessTtlTimer(SRpcMsg *pReq) {
pHead->vgId = htonl(pVgroup->vgId);
tSerializeSVDropTtlTableReq((char *)pHead + sizeof(SMsgHead), reqLen, &ttlReq);
SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen, .info = pReq->info};
SRpcMsg rpcMsg = {.msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS, .pCont = pHead, .contLen = contLen, .info = pReq->info};
SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup);
int32_t code = tmsgSendReq(&epSet, &rpcMsg);
if (code != 0) {
@ -2532,11 +2541,6 @@ static int32_t mndCheckDropStbForStream(SMnode *pMnode, const char *stbFullName,
pIter = sdbFetch(pSdb, SDB_STREAM, pIter, (void **)&pStream);
if (pIter == NULL) break;
if (pStream->smaId != 0) {
sdbRelease(pSdb, pStream);
continue;
}
if (pStream->targetStbUid == suid) {
sdbCancelFetch(pSdb, pIter);
sdbRelease(pSdb, pStream);
@ -3227,6 +3231,11 @@ static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc
continue;
}
if (isTsmaResSTb(pStb->name)) {
sdbRelease(pSdb, pStb);
continue;
}
cols = 0;
SName name = {0};
@ -3693,3 +3702,376 @@ static int32_t mndProcessDropIndexReq(SRpcMsg *pReq) {
_OVER:
return code;
}*/
static int32_t mndProcessDropStbReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessDropStbReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}
typedef struct SVDropTbVgReqs {
SVDropTbBatchReq req;
SVgroupInfo info;
} SVDropTbVgReqs;
typedef struct SMDropTbDbInfo {
SArray *dbVgInfos;
int32_t hashPrefix;
int32_t hashSuffix;
int32_t hashMethod;
} SMDropTbDbInfo;
typedef struct SMDropTbTsmaInfo {
char tsmaResTbDbFName[TSDB_DB_FNAME_LEN];
char tsmaResTbNamePrefix[TSDB_TABLE_NAME_LEN];
int32_t suid;
SMDropTbDbInfo dbInfo; // reference to DbInfo in pDbMap
} SMDropTbTsmaInfo;
typedef struct SMDropTbTsmaInfos {
SArray* pTsmaInfos; // SMDropTbTsmaInfo
} SMDropTbTsmaInfos;
typedef struct SMndDropTbsWithTsmaCtx {
SHashObj* pTsmaMap; // <suid, SMDropTbTsmaInfos>
SHashObj* pDbMap; // <dbuid, SMDropTbDbInfo>
SHashObj* pVgMap; // <vgId, SVDropTbVgReqs>
SArray* pResTbNames; // SArray<char*>
} SMndDropTbsWithTsmaCtx;
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode* pMnode, SMndDropTbsWithTsmaCtx* pCtx, SArray* pTbs, int32_t vgId);
static void mndDestroyDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx* p) {
if (!p) return;
if (p->pDbMap) {
void* pIter = taosHashIterate(p->pDbMap, NULL);
while (pIter) {
SMDropTbDbInfo* pInfo = pIter;
taosArrayDestroy(pInfo->dbVgInfos);
pIter = taosHashIterate(p->pDbMap, pIter);
}
taosHashCleanup(p->pDbMap);
}
if (p->pResTbNames) {
taosArrayDestroyP(p->pResTbNames, taosMemoryFree);
}
if (p->pTsmaMap) {
void* pIter = taosHashIterate(p->pTsmaMap, NULL);
while (pIter) {
SMDropTbTsmaInfos* pInfos = pIter;
taosArrayDestroy(pInfos->pTsmaInfos);
pIter = taosHashIterate(p->pTsmaMap, pIter);
}
taosHashCleanup(p->pTsmaMap);
}
if (p->pVgMap) {
void* pIter = taosHashIterate(p->pVgMap, NULL);
while (pIter) {
SVDropTbVgReqs *pReqs = pIter;
taosArrayDestroy(pReqs->req.pArray);
pIter = taosHashIterate(p->pVgMap, pIter);
}
taosHashCleanup(p->pVgMap);
}
taosMemoryFree(p);
}
static int32_t mndInitDropTbsWithTsmaCtx(SMndDropTbsWithTsmaCtx** ppCtx) {
int32_t code = 0;
SMndDropTbsWithTsmaCtx* pCtx = taosMemoryCalloc(1, sizeof(SMndDropTbsWithTsmaCtx));
if (!pCtx) return TSDB_CODE_OUT_OF_MEMORY;
pCtx->pTsmaMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
if (!pCtx->pTsmaMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pCtx->pDbMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK);
if (!pCtx->pDbMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
pCtx->pResTbNames = taosArrayInit(TARRAY_MIN_SIZE, POINTER_BYTES);
pCtx->pVgMap = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), false, HASH_NO_LOCK);
if (!pCtx->pVgMap) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
*ppCtx = pCtx;
_end:
if (code) mndDestroyDropTbsWithTsmaCtx(pCtx);
return code;
}
static void* mndBuildVDropTbsReq(SMnode* pMnode, const SVgroupInfo* pVgInfo, const SVDropTbBatchReq* pReq, int32_t *len) {
int32_t contLen = 0;
int32_t ret = 0;
SMsgHead *pHead = NULL;
SEncoder encoder = {0};
tEncodeSize(tEncodeSVDropTbBatchReq, pReq, contLen, ret);
if (ret < 0) return NULL;
contLen += sizeof(SMsgHead);
pHead = taosMemoryMalloc(contLen);
if (pHead == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return NULL;
}
pHead->contLen = htonl(contLen);
pHead->vgId = htonl(pVgInfo->vgId);
void *pBuf = POINTER_SHIFT(pHead, sizeof(SMsgHead));
tEncoderInit(&encoder, pBuf, contLen - sizeof(SMsgHead));
tEncodeSVDropTbBatchReq(&encoder, pReq);
tEncoderClear(&encoder);
*len = contLen;
return pHead;
}
static int32_t mndSetDropTbsRedoActions(SMnode* pMnode, STrans* pTrans, const SVDropTbVgReqs* pVgReqs, void* pCont, int32_t contLen) {
STransAction action = {0};
action.epSet = pVgReqs->info.epSet;
action.pCont = pCont;
action.contLen = contLen;
action.msgType = TDMT_VND_DROP_TABLE;
action.acceptableCode = TSDB_CODE_TDB_TABLE_NOT_EXIST;
return mndTransAppendRedoAction(pTrans, &action);
}
static int32_t mndCreateDropTbsTxnPrepare(SRpcMsg* pRsp, SMndDropTbsWithTsmaCtx* pCtx) {
SMnode *pMnode = pRsp->info.node;
STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pRsp, "drop-tbs");
mndTransSetChangeless(pTrans);
if (pTrans == NULL) goto _OVER;
if (mndTransCheckConflict(pMnode, pTrans) != 0) goto _OVER;
void* pIter = taosHashIterate(pCtx->pVgMap, NULL);
while (pIter) {
const SVDropTbVgReqs* pVgReqs = pIter;
int32_t len = 0;
void* p = mndBuildVDropTbsReq(pMnode, &pVgReqs->info, &pVgReqs->req, &len);
if (!p || mndSetDropTbsRedoActions(pMnode, pTrans, pVgReqs, p, len) != 0) {
taosHashCancelIterate(pCtx->pVgMap, pIter);
goto _OVER;
}
pIter = taosHashIterate(pCtx->pVgMap, pIter);
}
if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER;
_OVER:
mndTransDrop(pTrans);
return terrno;
}
static int32_t mndProcessDropTbWithTsma(SRpcMsg* pReq) {
int32_t code = -1;
SMnode *pMnode = pReq->info.node;
SDbObj *pDb = NULL;
SStbObj *pStb = NULL;
SMDropTbsReq dropReq = {0};
bool locked = false;
if (tDeserializeSMDropTbsReq(pReq->pCont, pReq->contLen, &dropReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _OVER;
}
SMndDropTbsWithTsmaCtx* pCtx = NULL;
terrno = mndInitDropTbsWithTsmaCtx(&pCtx);
if (terrno) goto _OVER;
for (int32_t i = 0; i < dropReq.pVgReqs->size; ++i) {
SMDropTbReqsOnSingleVg* pReq = taosArrayGet(dropReq.pVgReqs, i);
terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, pReq->pTbs, pReq->vgInfo.vgId);
if (terrno) goto _OVER;
}
if (mndCreateDropTbsTxnPrepare(pReq, pCtx) == 0)
code = 0;
_OVER:
tFreeSMDropTbsReq(&dropReq);
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
return code;
}
static int32_t mndDropTbAdd(SMnode *pMnode, SHashObj *pVgHashMap, const SVgroupInfo *pVgInfo, char *name, tb_uid_t suid,
bool ignoreNotExists) {
SVDropTbReq req = {.name = name, .suid = suid, .igNotExists = ignoreNotExists};
SVDropTbVgReqs * pReqs = taosHashGet(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId));
SVDropTbVgReqs reqs = {0};
if (pReqs == NULL) {
reqs.info = *pVgInfo;
reqs.req.pArray = taosArrayInit(TARRAY_MIN_SIZE, sizeof(SVDropTbReq));
taosArrayPush(reqs.req.pArray, &req);
taosHashPut(pVgHashMap, &pVgInfo->vgId, sizeof(pVgInfo->vgId), &reqs, sizeof(reqs));
} else {
taosArrayPush(pReqs->req.pArray, &req);
}
return 0;
}
static int32_t mndGetDbVgInfoForTsma(SMnode* pMnode, const char* dbname, SMDropTbTsmaInfo* pInfo) {
int32_t code = 0;
SDbObj* pDb = mndAcquireDb(pMnode, dbname);
if (!pDb) {
code = TSDB_CODE_MND_DB_NOT_EXIST;
goto _end;
}
pInfo->dbInfo.dbVgInfos = taosArrayInit(pDb->cfg.numOfVgroups, sizeof(SVgroupInfo));
if ( !pInfo->dbInfo.dbVgInfos) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
mndBuildDBVgroupInfo(pDb, pMnode, pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.hashPrefix = pDb->cfg.hashPrefix;
pInfo->dbInfo.hashSuffix = pDb->cfg.hashSuffix;
pInfo->dbInfo.hashMethod = pDb->cfg.hashMethod;
_end:
if (pDb) mndReleaseDb(pMnode, pDb);
if (code && pInfo->dbInfo.dbVgInfos) {
taosArrayDestroy(pInfo->dbInfo.dbVgInfos);
pInfo->dbInfo.dbVgInfos = NULL;
}
return code;
}
int32_t vgHashValCmp(const void* lp, const void* rp) {
uint32_t* key = (uint32_t*)lp;
SVgroupInfo* pVg = (SVgroupInfo*)rp;
if (*key < pVg->hashBegin) {
return -1;
} else if (*key > pVg->hashEnd) {
return 1;
}
return 0;
}
static int32_t mndDropTbAddTsmaResTbsForSingleVg(SMnode* pMnode, SMndDropTbsWithTsmaCtx* pCtx, SArray* pTbs, int32_t vgId) {
int32_t code = 0;
SVgObj* pVgObj = mndAcquireVgroup(pMnode, vgId);
if (!pVgObj) {
code = 0;
goto _end;
}
SVgroupInfo vgInfo = {.hashBegin = pVgObj->hashBegin, .hashEnd = pVgObj->hashEnd, .numOfTable = pVgObj->numOfTables, .vgId = pVgObj->vgId};
vgInfo.epSet = mndGetVgroupEpset(pMnode, pVgObj);
mndReleaseVgroup(pMnode, pVgObj);
// get all stb uids
for (int32_t i = 0; i < pTbs->size; ++i) {
const SVDropTbReq* pTb = taosArrayGet(pTbs, i);
if (taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid))) {
} else {
SMDropTbTsmaInfos infos = {0};
infos.pTsmaInfos = taosArrayInit(2, sizeof(SMDropTbTsmaInfo));
if (!infos.pTsmaInfos) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
taosHashPut(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid), &infos, sizeof(infos));
}
}
void *pIter = NULL;
SSmaObj *pSma = NULL;
char buf[TSDB_TABLE_FNAME_LEN] = {0};
// get used tsmas and it's dbs
while (1) {
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void **)&pSma);
if (!pIter) break;
SMDropTbTsmaInfos* pInfos = taosHashGet(pCtx->pTsmaMap, &pSma->stbUid, sizeof(pSma->stbUid));
if (pInfos) {
SMDropTbTsmaInfo info = {0};
int32_t len = sprintf(buf, "%s", pSma->name);
len = taosCreateMD5Hash(buf, len);
sprintf(info.tsmaResTbDbFName, "%s", pSma->db);
snprintf(info.tsmaResTbNamePrefix, TSDB_TABLE_NAME_LEN, "%s", buf);
SMDropTbDbInfo* pDbInfo = taosHashGet(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN);
info.suid = pSma->dstTbUid;
if (!pDbInfo) {
code = mndGetDbVgInfoForTsma(pMnode, pSma->db, &info);
if (code != TSDB_CODE_SUCCESS) {
sdbCancelFetch(pMnode->pSdb, pIter);
sdbRelease(pMnode->pSdb, pSma);
goto _end;
}
taosHashPut(pCtx->pDbMap, pSma->db, TSDB_DB_FNAME_LEN, &info.dbInfo, sizeof(SMDropTbDbInfo));
} else {
info.dbInfo = *pDbInfo;
}
taosArrayPush(pInfos->pTsmaInfos, &info);
}
sdbRelease(pMnode->pSdb, pSma);
}
// generate vg req map
for (int32_t i = 0; i < pTbs->size; ++i) {
SVDropTbReq* pTb = taosArrayGet(pTbs, i);
mndDropTbAdd(pMnode, pCtx->pVgMap, &vgInfo, pTb->name, pTb->suid, pTb->igNotExists);
SMDropTbTsmaInfos *pInfos = taosHashGet(pCtx->pTsmaMap, &pTb->suid, sizeof(pTb->suid));
SArray *pVgInfos = NULL;
char buf[TSDB_TABLE_FNAME_LEN];
for (int32_t j = 0; j < pInfos->pTsmaInfos->size; ++j) {
SMDropTbTsmaInfo *pInfo = taosArrayGet(pInfos->pTsmaInfos, j);
int32_t len = sprintf(buf, "%s.%s_%s", pInfo->tsmaResTbDbFName, pInfo->tsmaResTbNamePrefix, pTb->name);
uint32_t hashVal =
taosGetTbHashVal(buf, len, pInfo->dbInfo.hashMethod, pInfo->dbInfo.hashPrefix, pInfo->dbInfo.hashSuffix);
const SVgroupInfo *pVgInfo = taosArraySearch(pInfo->dbInfo.dbVgInfos, &hashVal, vgHashValCmp, TD_EQ);
void* p = taosStrdup(buf + strlen(pInfo->tsmaResTbDbFName) + TSDB_NAME_DELIMITER_LEN);
taosArrayPush(pCtx->pResTbNames, &p);
mndDropTbAdd(pMnode, pCtx->pVgMap, pVgInfo, p, pInfo->suid, true);
}
}
_end:
return code;
}
static int32_t mndProcessFetchTtlExpiredTbs(SRpcMsg *pRsp) {
int32_t code = -1;
SDecoder decoder = {0};
SMnode *pMnode = pRsp->info.node;
SVFetchTtlExpiredTbsRsp rsp = {0};
SMndDropTbsWithTsmaCtx *pCtx = NULL;
if (pRsp->code != TSDB_CODE_SUCCESS) goto _end;
if (pRsp->contLen == 0) {
code = 0;
goto _end;
}
tDecoderInit(&decoder, pRsp->pCont, pRsp->contLen);
terrno = tDecodeVFetchTtlExpiredTbsRsp(&decoder, &rsp);
if (terrno) goto _end;
terrno = mndInitDropTbsWithTsmaCtx(&pCtx);
if (terrno) goto _end;
terrno = mndDropTbAddTsmaResTbsForSingleVg(pMnode, pCtx, rsp.pExpiredTbs, rsp.vgId);
if (terrno) goto _end;
if (mndCreateDropTbsTxnPrepare(pRsp, pCtx) == 0)
code = 0;
_end:
if (pCtx) mndDestroyDropTbsWithTsmaCtx(pCtx);
tDecoderClear(&decoder);
tFreeFetchTtlExpiredTbsRsp(&rsp);
return code;
}

56
source/dnode/mnode/impl/src/mndStream.c
View File

@ -41,6 +41,10 @@ static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream);
static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream);
static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReq(SRpcMsg *pReq);
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq);
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
static int32_t mndProcessStreamCheckpointInCandid(SRpcMsg *pReq);
@ -102,6 +106,13 @@ int32_t mndInitStream(SMnode *pMnode) {
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_UPDATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_TASK_RESET_RSP, mndTransProcessRsp);
// for msgs inside mnode
// TODO change the name
mndSetMsgHandle(pMnode, TDMT_STREAM_CREATE, mndProcessCreateStreamReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_STREAM_CREATE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_STREAM_DROP, mndProcessDropStreamReqFromMNode);
mndSetMsgHandle(pMnode, TDMT_STREAM_DROP_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_VND_STREAM_CHECK_POINT_SOURCE_RSP, mndTransProcessRsp);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_CHECKPOINT_TIMER, mndProcessStreamCheckpointTmr);
mndSetMsgHandle(pMnode, TDMT_MND_STREAM_BEGIN_CHECKPOINT, mndProcessStreamDoCheckpoint);
@ -335,7 +346,10 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj->createTime = taosGetTimestampMs();
pObj->updateTime = pObj->createTime;
pObj->version = 1;
pObj->smaId = 0;
if (pCreate->smaId > 0) {
pObj->subTableWithoutMd5 = 1;
}
pObj->smaId = pCreate->smaId;
pObj->indexForMultiAggBalance = -1;
pObj->uid = mndGenerateUid(pObj->name, strlen(pObj->name));
@ -1213,6 +1227,24 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) {
}
}
if (pStream->smaId != 0) {
void *pIter = NULL;
SSmaObj *pSma = NULL;
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
while (pIter) {
if (pSma && pSma->uid == pStream->smaId) {
sdbRelease(pMnode->pSdb, pSma);
sdbRelease(pMnode->pSdb, pStream);
sdbCancelFetch(pMnode->pSdb, pIter);
tFreeMDropStreamReq(&dropReq);
terrno = TSDB_CODE_TSMA_MUST_BE_DROPPED;
return -1;
}
if (pSma) sdbRelease(pMnode->pSdb, pSma);
pIter = sdbFetch(pMnode->pSdb, SDB_SMA, pIter, (void**)&pSma);
}
}
if (mndCheckDbPrivilegeByName(pMnode, pReq->info.conn.user, MND_OPER_WRITE_DB, pStream->targetDb) != 0) {
sdbRelease(pMnode->pSdb, pStream);
tFreeMDropStreamReq(&dropReq);
@ -2372,3 +2404,25 @@ int32_t mndProcessStreamReqCheckpoint(SRpcMsg *pReq) {
return 0;
}
static int32_t mndProcessCreateStreamReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessCreateStreamReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}
static int32_t mndProcessDropStreamReqFromMNode(SRpcMsg *pReq) {
int32_t code = mndProcessDropStreamReq(pReq);
if (code != 0 && code != TSDB_CODE_ACTION_IN_PROGRESS) {
pReq->info.rsp = rpcMallocCont(1);
pReq->info.rspLen = 1;
pReq->info.noResp = false;
pReq->code = code;
}
return code;
}

5
source/dnode/mnode/sdb/inc/sdb.h
View File

@ -352,6 +352,11 @@ void sdbTraverse(SSdb *pSdb, ESdbType type, sdbTraverseFp fp, void *p1, void *p2
*/
int32_t sdbGetSize(SSdb *pSdb, ESdbType type);
/**
* @brief get valid number of rows, removed rows are ignored
*/
int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type);
/**
* @brief Get the max id of the table, keyType of table should be INT32
*

12
source/dnode/mnode/sdb/src/sdbHash.c
View File

@ -510,3 +510,15 @@ int64_t sdbGetTableVer(SSdb *pSdb, ESdbType type) {
return pSdb->tableVer[type];
}
bool countValid(SMnode *pMnode, void *pObj, void *p1, void *p2, void *p3) {
int32_t* pInt = p1;
(*pInt) += 1;
return true;
}
int32_t sdbGetValidSize(SSdb* pSdb, ESdbType type) {
int32_t num = 0;
sdbTraverse(pSdb, type, countValid, &num, 0, 0);
return num;
}

2
source/dnode/vnode/src/inc/tsdb.h
View File

@ -126,7 +126,7 @@ int32_t tsdbRowCompareWithoutVersion(const void *p1, const void *p2);
int32_t tsdbRowKeyCmpr(const STsdbRowKey *key1, const STsdbRowKey *key2);
void tsdbRowGetKey(TSDBROW *row, STsdbRowKey *key);
void tColRowGetKey(SBlockData *pBlock, int32_t irow, SRowKey *key);
int32_t tRowKeyAssign(SRowKey *pDst, SRowKey *pSrc);
// STSDBRowIter
int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema);

1
source/dnode/vnode/src/inc/vnd.h
View File

@ -120,6 +120,7 @@ void vnodeQueryClose(SVnode* pVnode);
int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
int vnodeGetTableCfg(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg);
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct);
// vnodeCommit.c
int32_t vnodeBegin(SVnode* pVnode);

1
source/dnode/vnode/src/inc/vnodeInt.h
View File

@ -294,6 +294,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg);
int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg);
int32_t tqStreamProgressRetrieveReq(STQ* pTq, SRpcMsg* pMsg);
// sma
int32_t smaInit();

38
source/dnode/vnode/src/tq/tq.c
View File

@ -1031,6 +1031,44 @@ int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) {
return 0;
}
int32_t tqStreamProgressRetrieveReq(STQ *pTq, SRpcMsg *pMsg) {
char* msgStr = pMsg->pCont;
char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead));
int32_t msgLen = pMsg->contLen - sizeof(SMsgHead);
int32_t code = 0;
SStreamProgressReq req;
char* pRspBuf = taosMemoryCalloc(1, sizeof(SMsgHead) + sizeof(SStreamProgressRsp));
SStreamProgressRsp* pRsp = POINTER_SHIFT(pRspBuf, sizeof(SMsgHead));
if (!pRspBuf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER;
}
code = tDeserializeStreamProgressReq(msgBody, msgLen, &req);
if (code == TSDB_CODE_SUCCESS) {
code = tqGetStreamExecInfo(pTq->pVnode, req.streamId, &pRsp->progressDelay, &pRsp->fillHisFinished);
}
if (code == TSDB_CODE_SUCCESS) {
pRsp->fetchIdx = req.fetchIdx;
pRsp->subFetchIdx = req.subFetchIdx;
pRsp->vgId = req.vgId;
pRsp->streamId = req.streamId;
tSerializeStreamProgressRsp(pRsp, sizeof(SStreamProgressRsp) + sizeof(SMsgHead), pRsp);
SRpcMsg rsp = {.info = pMsg->info, .code = 0};
rsp.pCont = pRspBuf;
pRspBuf = NULL;
rsp.contLen = sizeof(SMsgHead) + sizeof(SStreamProgressRsp);
tmsgSendRsp(&rsp);
}
_OVER:
if (pRspBuf) {
taosMemoryFree(pRspBuf);
}
return code;
}
int32_t tqProcessTaskCheckPointSourceReq(STQ* pTq, SRpcMsg* pMsg, SRpcMsg* pRsp) {
int32_t vgId = TD_VID(pTq->pVnode);
SStreamMeta* pMeta = pTq->pStreamMeta;

2
source/dnode/vnode/src/tq/tqPush.c
View File

@ -45,7 +45,7 @@ int32_t tqPushMsg(STQ* pTq, tmsg_t msgType) {
// 1. the vnode has already been restored.
// 2. the vnode should be the leader.
// 3. the stream is not suspended yet.
if ((!tsDisableStream) && (numOfTasks > 0) && (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE)) {
if ((!tsDisableStream) && (numOfTasks > 0) /* && (msgType == TDMT_VND_SUBMIT || msgType == TDMT_VND_DELETE)*/) {
tqScanWalAsync(pTq, true);
}

13
source/dnode/vnode/src/tq/tqUtil.c
View File

@ -531,6 +531,11 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
*fhFinished = !HAS_RELATED_FILLHISTORY_TASK(pTask);
int64_t ver = walReaderGetCurrentVer(pTask->exec.pWalReader);
if (ver == -1) {
ver = pTask->chkInfo.processedVer;
} else {
ver--;
}
SVersionRange verRange = {0};
walReaderValidVersionRange(pTask->exec.pWalReader, &verRange.minVer, &verRange.maxVer);
@ -545,14 +550,18 @@ int32_t tqGetStreamExecInfo(SVnode* pVnode, int64_t streamId, int64_t* pDelay, b
int64_t latest = 0;
code = walFetchHead(pReader, ver);
if (code != TSDB_CODE_SUCCESS) {
if (code == TSDB_CODE_SUCCESS) {
cur = pReader->pHead->head.ingestTs;
}
if (ver == verRange.maxVer) {
latest = cur;
} else {
code = walFetchHead(pReader, verRange.maxVer);
if (code != TSDB_CODE_SUCCESS) {
if (code == TSDB_CODE_SUCCESS) {
latest = pReader->pHead->head.ingestTs;
}
}
if (pDelay != NULL) { // delay in ms
*pDelay = (latest - cur) / 1000;

34
source/dnode/vnode/src/tsdb/tsdbCommit2.c
View File

@ -37,7 +37,6 @@ typedef struct {
int64_t cid;
int64_t now;
TSKEY nextKey;
TSKEY maxDelKey;
int32_t fid;
int32_t expLevel;
SDiskID did;
@ -161,15 +160,13 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
int64_t numRecord = 0;
SMetaInfo info;
if (committer->ctx->fset == NULL && !committer->ctx->hasTSData) {
if (committer->ctx->maxKey < committer->ctx->maxDelKey) {
committer->ctx->nextKey = committer->ctx->maxKey + 1;
} else {
committer->ctx->nextKey = TSKEY_MAX;
}
return 0;
if (committer->tsdb->imem->nDel == 0) {
goto _exit;
}
// do not need to write tomb data if there is no ts data
bool skip = (committer->ctx->fset == NULL && !committer->ctx->hasTSData);
committer->ctx->tbid->suid = 0;
committer->ctx->tbid->uid = 0;
for (STombRecord *record; (record = tsdbIterMergerGetTombRecord(committer->tombIterMerger));) {
@ -196,10 +193,12 @@ static int32_t tsdbCommitTombData(SCommitter2 *committer) {
record->skey = TMAX(record->skey, committer->ctx->minKey);
record->ekey = TMIN(record->ekey, committer->ctx->maxKey);
if (!skip) {
numRecord++;
code = tsdbFSetWriteTombRecord(committer->writer, record);
TSDB_CHECK_CODE(code, lino, _exit);
}
}
code = tsdbIterMergerNext(committer->tombIterMerger);
TSDB_CHECK_CODE(code, lino, _exit);
@ -480,27 +479,10 @@ static int32_t tsdbOpenCommitter(STsdb *tsdb, SCommitInfo *info, SCommitter2 *co
STbData *tbData = TCONTAINER_OF(node, STbData, rbtn);
for (SDelData *delData = tbData->pHead; delData; delData = delData->pNext) {
if (delData->sKey < committer->ctx->nextKey) {
committer->ctx->nextKey = delData->sKey;
committer->ctx->nextKey = TMIN(committer->ctx->nextKey, delData->sKey);
}
}
}
}
committer->ctx->maxDelKey = TSKEY_MIN;
TSKEY minKey = TSKEY_MAX;
TSKEY maxKey = TSKEY_MIN;
if (TARRAY2_SIZE(committer->fsetArr) > 0) {
STFileSet *fset = TARRAY2_LAST(committer->fsetArr);
tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &committer->ctx->maxDelKey);
fset = TARRAY2_FIRST(committer->fsetArr);
tsdbFidKeyRange(fset->fid, committer->minutes, committer->precision, &minKey, &maxKey);
}
if (committer->ctx->nextKey < TMIN(tsdb->imem->minKey, minKey)) {
committer->ctx->nextKey = TMIN(tsdb->imem->minKey, minKey);
}
_exit:
if (code) {

3
source/dnode/vnode/src/tsdb/tsdbRead2.c
View File

@ -124,7 +124,7 @@ int32_t pkCompEx(SRowKey* p1, SRowKey* p2) {
if (p1->pks[0].val == p2->pks[0].val) {
return 0;
} else {
return p1->pks[0].val > p2->pks[0].val? 1:-1;
return tValueCompare(&p1->pks[0], &p2->pks[0]);
}
}
}
@ -5037,7 +5037,6 @@ int32_t tsdbReaderReset2(STsdbReader* pReader, SQueryTableDataCond* pCond) {
bool asc = ASCENDING_TRAVERSE(pReader->info.order);
int32_t step = asc ? 1 : -1;
int64_t ts = 0;
if (asc) {
ts = (pReader->info.window.skey > INT64_MIN) ? pReader->info.window.skey - 1 : pReader->info.window.skey;

45
source/dnode/vnode/src/tsdb/tsdbReadUtil.c
View File

@ -136,20 +136,51 @@ int32_t initRowKey(SRowKey* pKey, int64_t ts, int32_t numOfPks, int32_t type, in
if (numOfPks > 0) {
pKey->pks[0].type = type;
if (IS_NUMERIC_TYPE(pKey->pks[0].type)) {
if (IS_NUMERIC_TYPE(type)) {
if (asc) {
switch(pKey->pks[0].type) {
case TSDB_DATA_TYPE_BIGINT:pKey->pks[0].val = INT64_MIN;break;
case TSDB_DATA_TYPE_INT:pKey->pks[0].val = INT32_MIN;break;
case TSDB_DATA_TYPE_SMALLINT:pKey->pks[0].val = INT16_MIN;break;
case TSDB_DATA_TYPE_TINYINT:pKey->pks[0].val = INT8_MIN;break;
switch(type) {
case TSDB_DATA_TYPE_BIGINT: {
pKey->pks[0].val = INT64_MIN;
break;
}
case TSDB_DATA_TYPE_INT:{
int32_t min = INT32_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_SMALLINT:{
int16_t min = INT16_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_TINYINT:{
int8_t min = INT8_MIN;
memcpy(&pKey->pks[0].val, &min, tDataTypes[type].bytes);
break;
}
case TSDB_DATA_TYPE_UTINYINT:
case TSDB_DATA_TYPE_USMALLINT:
case TSDB_DATA_TYPE_UINT:
case TSDB_DATA_TYPE_UBIGINT: {
pKey->pks[0].val = 0;
break;
}
default:
ASSERT(0);
}
} else {
switch(pKey->pks[0].type) {
switch(type) {
case TSDB_DATA_TYPE_BIGINT:pKey->pks[0].val = INT64_MAX;break;
case TSDB_DATA_TYPE_INT:pKey->pks[0].val = INT32_MAX;break;
case TSDB_DATA_TYPE_SMALLINT:pKey->pks[0].val = INT16_MAX;break;
case TSDB_DATA_TYPE_TINYINT:pKey->pks[0].val = INT8_MAX;break;
case TSDB_DATA_TYPE_UBIGINT:pKey->pks[0].val = UINT64_MAX;break;
case TSDB_DATA_TYPE_UINT:pKey->pks[0].val = UINT32_MAX;break;
case TSDB_DATA_TYPE_USMALLINT:pKey->pks[0].val = UINT16_MAX;break;
case TSDB_DATA_TYPE_UTINYINT:pKey->pks[0].val = UINT8_MAX;break;
default:
ASSERT(0);
}
}
} else {

54
source/dnode/vnode/src/vnd/vnodeQuery.c
View File

@ -323,6 +323,9 @@ int32_t vnodeGetBatchMeta(SVnode *pVnode, SRpcMsg *pMsg) {
case TDMT_VND_TABLE_CFG:
vnodeGetTableCfg(pVnode, &reqMsg, false);
break;
case TDMT_VND_GET_STREAM_PROGRESS:
vnodeGetStreamProgress(pVnode, &reqMsg, false);
break;
default:
qError("invalid req msgType %d", req->msgType);
reqMsg.code = TSDB_CODE_INVALID_MSG;
@ -711,3 +714,54 @@ void *vnodeGetIvtIdx(void *pVnode) {
int32_t vnodeGetTableSchema(void *pVnode, int64_t uid, STSchema **pSchema, int64_t *suid) {
return tsdbGetTableSchema(((SVnode *)pVnode)->pMeta, uid, pSchema, suid);
}
int32_t vnodeGetStreamProgress(SVnode* pVnode, SRpcMsg* pMsg, bool direct) {
int32_t code = 0;
SStreamProgressReq req;
SStreamProgressRsp rsp = {0};
SRpcMsg rpcMsg = {.info = pMsg->info, .code = 0};
char * buf = NULL;
int32_t rspLen = 0;
code = tDeserializeStreamProgressReq(pMsg->pCont, pMsg->contLen, &req);
if (code == TSDB_CODE_SUCCESS) {
rsp.fetchIdx = req.fetchIdx;
rsp.subFetchIdx = req.subFetchIdx;
rsp.vgId = req.vgId;
rsp.streamId = req.streamId;
rspLen = tSerializeStreamProgressRsp(0, 0, &rsp);
if (direct) {
buf = rpcMallocCont(rspLen);
} else {
buf = taosMemoryCalloc(1, rspLen);
}
if (!buf) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _OVER;
}
}
if (code == TSDB_CODE_SUCCESS) {
code = tqGetStreamExecInfo(pVnode, req.streamId, &rsp.progressDelay, &rsp.fillHisFinished);
}
if (code == TSDB_CODE_SUCCESS) {
tSerializeStreamProgressRsp(buf, rspLen, &rsp);
rpcMsg.pCont = buf;
buf = NULL;
rpcMsg.contLen = rspLen;
rpcMsg.code = code;
rpcMsg.msgType = pMsg->msgType;
if (direct) {
tmsgSendRsp(&rpcMsg);
} else {
*pMsg = rpcMsg;
}
}
_OVER:
if (buf) {
taosMemoryFree(buf);
}
return code;
}

78
source/dnode/vnode/src/vnd/vnodeSvr.c
View File

@ -50,6 +50,7 @@ static int32_t vnodeProcessArbCheckSyncReq(SVnode *pVnode, void *pReq, int32_t l
static int32_t vnodePreCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeCheckAssignedLogSyncd(SVnode *pVnode, char *member0Token, char *member1Token);
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp);
extern int32_t vnodeProcessKillCompactReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp);
extern int32_t vnodeQueryCompactProgress(SVnode *pVnode, SRpcMsg *pMsg);
@ -484,6 +485,7 @@ int32_t vnodePreProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg) {
case TDMT_VND_ALTER_TABLE: {
code = vnodePreProcessAlterTableMsg(pVnode, pMsg);
} break;
case TDMT_VND_FETCH_TTL_EXPIRED_TBS:
case TDMT_VND_DROP_TTL_TABLE: {
code = vnodePreProcessDropTtlMsg(pVnode, pMsg);
} break;
@ -563,6 +565,9 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t ver, SRpcMsg
case TDMT_VND_DROP_TTL_TABLE:
if (vnodeProcessDropTtlTbReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_FETCH_TTL_EXPIRED_TBS:
if (vnodeProcessFetchTtlExpiredTbs(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
case TDMT_VND_TRIM:
if (vnodeProcessTrimReq(pVnode, ver, pReq, len, pRsp) < 0) goto _err;
break;
@ -838,6 +843,8 @@ int32_t vnodeProcessStreamMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo)
return tqProcessStreamReqCheckpointRsp(pVnode->pTq, pMsg);
case TDMT_STREAM_TASK_CHECKPOINT_READY_RSP:
return tqProcessTaskCheckpointReadyRsp(pVnode->pTq, pMsg);
case TDMT_VND_GET_STREAM_PROGRESS:
return tqStreamProgressRetrieveReq(pVnode->pTq, pMsg);
default:
vError("unknown msg type:%d in stream queue", pMsg->msgType);
return TSDB_CODE_APP_ERROR;
@ -924,6 +931,75 @@ end:
return ret;
}
static int32_t vnodeProcessFetchTtlExpiredTbs(SVnode* pVnode, int64_t ver, void* pReq, int32_t len, SRpcMsg* pRsp) {
int32_t code = -1;
SMetaReader mr = {0};
SVDropTtlTableReq ttlReq = {0};
SVFetchTtlExpiredTbsRsp rsp = {0};
SEncoder encoder = {0};
SArray* pNames = NULL;
pRsp->msgType = TDMT_VND_FETCH_TTL_EXPIRED_TBS_RSP;
pRsp->code = TSDB_CODE_SUCCESS;
pRsp->pCont = NULL;
pRsp->contLen = 0;
if (tDeserializeSVDropTtlTableReq(pReq, len, &ttlReq) != 0) {
terrno = TSDB_CODE_INVALID_MSG;
goto _end;
}
ASSERT(ttlReq.nUids == taosArrayGetSize(ttlReq.pTbUids));
tb_uid_t suid;
char ctbName[TSDB_TABLE_NAME_LEN];
SVDropTbReq expiredTb = {.igNotExists = true};
metaReaderDoInit(&mr, pVnode->pMeta, 0);
rsp.vgId = TD_VID(pVnode);
rsp.pExpiredTbs = taosArrayInit(ttlReq.nUids, sizeof(SVDropTbReq));
if (!rsp.pExpiredTbs) goto _end;
pNames = taosArrayInit(ttlReq.nUids, TSDB_TABLE_NAME_LEN);
if (!pNames) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
goto _end;
}
char buf[TSDB_TABLE_NAME_LEN];
for (int32_t i = 0; i < ttlReq.nUids; ++i) {
tb_uid_t* uid = taosArrayGet(ttlReq.pTbUids, i);
expiredTb.suid = *uid;
terrno = metaReaderGetTableEntryByUid(&mr, *uid);
if (terrno < 0) goto _end;
strncpy(buf, mr.me.name, TSDB_TABLE_NAME_LEN);
void* p = taosArrayPush(pNames, buf);
expiredTb.name = p;
if (mr.me.type == TSDB_CHILD_TABLE) {
expiredTb.suid = mr.me.ctbEntry.suid;
}
taosArrayPush(rsp.pExpiredTbs, &expiredTb);
}
int32_t ret = 0;
tEncodeSize(tEncodeVFetchTtlExpiredTbsRsp, &rsp, pRsp->contLen, ret);
pRsp->pCont = rpcMallocCont(pRsp->contLen);
if (pRsp->pCont == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
code = -1;
goto _end;
}
tEncoderInit(&encoder, pRsp->pCont, pRsp->contLen);
terrno = tEncodeVFetchTtlExpiredTbsRsp(&encoder, &rsp);
tEncoderClear(&encoder);
if (terrno == 0) code = 0;
_end:
metaReaderClear(&mr);
tFreeFetchTtlExpiredTbsRsp(&rsp);
taosArrayDestroy(ttlReq.pTbUids);
if (pNames) taosArrayDestroy(pNames);
pRsp->code = terrno;
return code;
}
static int32_t vnodeProcessCreateStbReq(SVnode *pVnode, int64_t ver, void *pReq, int32_t len, SRpcMsg *pRsp) {
SVCreateStbReq req = {0};
SDecoder coder;
@ -2093,6 +2169,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in
tDecoderInit(pCoder, pReq, len);
tDecodeDeleteRes(pCoder, pRes);
if (pRes->affectedRows > 0) {
for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) {
uint64_t uid = *(uint64_t *)taosArrayGet(pRes->uidList, iUid);
code = tsdbDeleteTableData(pVnode->pTsdb, ver, pRes->suid, uid, pRes->skey, pRes->ekey);
@ -2100,6 +2177,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t ver, void *pReq, in
code = metaUpdateChangeTimeWithLock(pVnode->pMeta, uid, pRes->ctimeMs);
if (code) goto _err;
}
}
code = tdProcessRSmaDelete(pVnode->pSma, ver, pRes, pReq, len);

99
source/libs/catalog/inc/catalogInt.h
View File

@ -31,6 +31,7 @@ extern "C" {
#define CTG_DEFAULT_CACHE_DB_NUMBER 20
#define CTG_DEFAULT_CACHE_TBLMETA_NUMBER 1000
#define CTG_DEFAULT_CACHE_VIEW_NUMBER 256
#define CTG_DEFAULT_CACHE_TSMA_NUMBER 10
#define CTG_DEFAULT_RENT_SECOND 10
#define CTG_DEFAULT_RENT_SLOT_SIZE 10
#define CTG_DEFAULT_MAX_RETRY_TIMES 3
@ -69,6 +70,7 @@ typedef enum {
CTG_CI_UDF,
CTG_CI_SVR_VER,
CTG_CI_VIEW,
CTG_CI_TBL_TSMA,
CTG_CI_MAX_VALUE,
} CTG_CACHE_ITEM;
@ -85,6 +87,7 @@ enum {
CTG_RENT_DB = 1,
CTG_RENT_STABLE,
CTG_RENT_VIEW,
CTG_RENT_TSMA,
};
enum {
@ -101,6 +104,8 @@ enum {
CTG_OP_DROP_TB_INDEX,
CTG_OP_UPDATE_VIEW_META,
CTG_OP_DROP_VIEW_META,
CTG_OP_UPDATE_TB_TSMA,
CTG_OP_DROP_TB_TSMA,
CTG_OP_CLEAR_CACHE,
CTG_OP_MAX
};
@ -123,6 +128,8 @@ typedef enum {
CTG_TASK_GET_TB_HASH_BATCH,
CTG_TASK_GET_TB_TAG,
CTG_TASK_GET_VIEW,
CTG_TASK_GET_TB_TSMA,
CTG_TASK_GET_TSMA,
} CTG_TASK_TYPE;
typedef enum {
@ -255,8 +262,41 @@ typedef struct SCtgViewsCtx {
SArray* pFetchs;
} SCtgViewsCtx;
typedef enum {
FETCH_TSMA_SOURCE_TB_META,
FETCH_TB_TSMA,
FETCH_TSMA_STREAM_PROGRESS,
} CTG_TSMA_FETCH_TYPE;
typedef struct SCtgTSMAFetch {
CTG_TSMA_FETCH_TYPE fetchType;
int32_t dbIdx;
int32_t tbIdx;
int32_t fetchIdx;
int32_t resIdx;
// tb meta
int32_t flag;
int32_t vgId;
// stream progress
int32_t subFetchNum;
int32_t finishedSubFetchNum;
int32_t vgNum;
// tb tsma
SName tsmaSourceTbName;
} SCtgTSMAFetch;
typedef struct SCtgTbTSMACtx {
int32_t fetchNum;
SArray* pNames; // SArray<STablesReq>
SArray* pResList;
SArray* pFetches;
} SCtgTbTSMACtx;
typedef STableIndexRsp STableIndex;
typedef STableTSMAInfo STSMACache;
typedef struct SCtgTbCache {
SRWLatch metaLock;
@ -280,6 +320,10 @@ typedef struct SCtgViewCache {
SViewMeta* pMeta;
} SCtgViewCache;
typedef struct SCtgTSMACache {
SRWLatch tsmaLock;
SArray* pTsmas; // SArray<STSMACache*>
} SCtgTSMACache;
typedef struct SCtgDBCache {
SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads
@ -290,6 +334,8 @@ typedef struct SCtgDBCache {
SHashObj* viewCache; // key:viewname, value:SCtgViewCache
SHashObj* tbCache; // key:tbname, value:SCtgTbCache
SHashObj* stbCache; // key:suid, value:char*
SHashObj* tsmaCache; // key:tbname, value: SCtgTSMACache
int32_t tsmaVersion;
uint64_t dbCacheNum[CTG_CI_MAX_VALUE];
uint64_t dbCacheSize;
} SCtgDBCache;
@ -325,6 +371,7 @@ typedef struct SCatalog {
SCtgRentMgmt dbRent;
SCtgRentMgmt stbRent;
SCtgRentMgmt viewRent;
SCtgRentMgmt tsmaRent;
SCtgCacheStat cacheStat;
} SCatalog;
@ -370,6 +417,8 @@ typedef struct SCtgJob {
int32_t tbCfgNum;
int32_t svrVerNum;
int32_t viewNum;
int32_t tbTsmaNum;
int32_t tsmaNum; // currently, only 1 is possible
} SCtgJob;
typedef struct SCtgMsgCtx {
@ -548,6 +597,24 @@ typedef struct SCtgDropViewMetaMsg {
uint64_t viewId;
} SCtgDropViewMetaMsg;
typedef struct SCtgUpdateTbTSMAMsg {
SCatalog* pCtg;
STableTSMAInfo* pTsma;
int32_t dbTsmaVersion;
uint64_t dbId;
} SCtgUpdateTbTSMAMsg;
typedef struct SCtgDropTbTSMAMsg {
SCatalog* pCtg;
char dbFName[TSDB_DB_FNAME_LEN];
char tbName[TSDB_TABLE_NAME_LEN];
char tsmaName[TSDB_TABLE_NAME_LEN];
uint64_t tsmaId;
uint64_t dbId;
uint64_t tbId;
bool dropAllForTb;
} SCtgDropTbTSMAMsg;
typedef struct SCtgCacheOperation {
int32_t opId;
@ -726,7 +793,9 @@ typedef struct SCtgCacheItemInfo {
(CTG_FLAG_IS_UNKNOWN_STB(_flag) || (CTG_FLAG_IS_STB(_flag) && (tbType) == TSDB_SUPER_TABLE) || \
(CTG_FLAG_IS_NOT_STB(_flag) && (tbType) != TSDB_SUPER_TABLE))
#define CTG_IS_BATCH_TASK(_taskType) ((CTG_TASK_GET_TB_META_BATCH == (_taskType)) || (CTG_TASK_GET_TB_HASH_BATCH == (_taskType)) || (CTG_TASK_GET_VIEW == (_taskType)))
#define CTG_IS_BATCH_TASK(_taskType) \
((CTG_TASK_GET_TB_META_BATCH == (_taskType)) || (CTG_TASK_GET_TB_HASH_BATCH == (_taskType)) || \
(CTG_TASK_GET_VIEW == (_taskType)) || (CTG_TASK_GET_TB_TSMA == (_taskType)))
#define CTG_GET_TASK_MSGCTX(_task, _id) \
(CTG_IS_BATCH_TASK((_task)->type) ? taosArrayGet((_task)->msgCtxs, (_id)) : &(_task)->msgCtx)
@ -917,7 +986,7 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation* operation);
int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char* dbFName, SCtgDBCache** pCache);
void ctgReleaseDBCache(SCatalog* pCtg, SCtgDBCache* dbCache);
void ctgRUnlockVgInfo(SCtgDBCache* dbCache);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32_t* exist);
int32_t ctgTbMetaExistInCache(SCatalog* pCtg, const char* dbFName, const char* tbName, int32_t* exist);
int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta);
int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType,
uint64_t* suid, char* stbName);
@ -943,10 +1012,12 @@ int32_t ctgMetaRentGet(SCtgRentMgmt* mgmt, void** res, uint32_t* num, int32_t si
int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortCompare, __compar_fn_t searchCompare);
void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache);
void ctgRemoveTSMARent(SCatalog* pCtg, SCtgDBCache* dbCache);
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache);
int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName, uint64_t dbId, uint64_t viewId,
SCtgViewCache *pCache);
int32_t ctgUpdateRentTSMAVersion(SCatalog* pCtg, char* dbFName, const STSMACache* pCache);
int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq);
int32_t ctgUpdateViewMetaToCache(SCatalog *pCtg, SViewMetaRsp *pRsp, bool syncReq);
int32_t ctgStartUpdateThread();
@ -976,7 +1047,7 @@ int32_t ctgGetUdfInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const ch
SCtgTask* pTask);
int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, SGetUserAuthRsp* out,
SCtgTask* pTask);
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName,
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, const char* tbName,
STableMetaOutput* out, SCtgTaskReq* tReq);
int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, STableMetaOutput* out,
SCtgTaskReq* tReq);
@ -1018,9 +1089,11 @@ void ctgFreeDbCache(SCtgDBCache* dbCache);
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2);
int32_t ctgViewVersionSortCompare(const void* key1, const void* key2);
int32_t ctgDbCacheInfoSortCompare(const void* key1, const void* key2);
int32_t ctgTSMAVersionSortCompare(const void* key1, const void* key2);
int32_t ctgStbVersionSearchCompare(const void* key1, const void* key2);
int32_t ctgDbCacheInfoSearchCompare(const void* key1, const void* key2);
int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2);
int32_t ctgTSMAVersionSearchCompare(const void* key1, const void* key2);
void ctgFreeSTableMetaOutput(STableMetaOutput* pOutput);
int32_t ctgUpdateMsgCtx(SCtgMsgCtx* pCtx, int32_t reqType, void* out, char* target);
int32_t ctgAddMsgCtx(SArray* pCtxs, int32_t reqType, void* out, char* target);
@ -1073,6 +1146,26 @@ int32_t ctgGetCachedStbNameFromSuid(SCatalog* pCtg, char* dbFName, uint64_t suid
int32_t ctgGetTbTagCb(SCtgTask* pTask);
int32_t ctgGetUserCb(SCtgTask* pTask);
int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx,
SArray* pList);
int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaName);
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType);
int32_t ctgUpdateTbTSMAEnqueue(SCatalog* pCtg, STSMACache** pTsma, int32_t tsmaVersion, bool syncOp);
int32_t ctgDropTSMAForTbEnqueue(SCatalog* pCtg, SName* pName, bool syncOp);
int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp);
int32_t ctgOpDropTbTSMA(SCtgCacheOperation* operation);
int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation* operation);
uint64_t ctgGetTbTSMACacheSize(STSMACache* pTsmaInfo);
void ctgFreeTbTSMAInfo(void* p);
bool hasOutOfDateTSMACache(SArray* pTsmas);
bool isCtgTSMACacheOutOfDate(STSMACache* pTsmaCache);
int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTbName,
SVgroupInfo* vgroupInfo, SStreamProgressRsp* out, SCtgTaskReq* tReq,
void* bInput);
int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag,
CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName);
extern SCatalogMgmt gCtgMgmt;
extern SCtgDebug gCTGDebug;
extern SCtgAsyncFps gCtgAsyncFps[];

137
source/libs/catalog/src/catalog.c
View File

@ -759,6 +759,10 @@ int32_t catalogInit(SCatalogCfg* cfg) {
gCtgMgmt.cfg.maxViewCacheNum = CTG_DEFAULT_CACHE_VIEW_NUMBER;
}
if (gCtgMgmt.cfg.maxTSMACacheNum == 0) {
gCtgMgmt.cfg.maxTSMACacheNum = CTG_DEFAULT_CACHE_TSMA_NUMBER;
}
if (gCtgMgmt.cfg.dbRentSec == 0) {
gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND;
}
@ -770,13 +774,19 @@ int32_t catalogInit(SCatalogCfg* cfg) {
if (gCtgMgmt.cfg.viewRentSec == 0) {
gCtgMgmt.cfg.viewRentSec = CTG_DEFAULT_RENT_SECOND;
}
if (gCtgMgmt.cfg.tsmaRentSec == 0) {
gCtgMgmt.cfg.tsmaRentSec = CTG_DEFAULT_RENT_SECOND;
}
} else {
gCtgMgmt.cfg.maxDBCacheNum = CTG_DEFAULT_CACHE_DB_NUMBER;
gCtgMgmt.cfg.maxTblCacheNum = CTG_DEFAULT_CACHE_TBLMETA_NUMBER;
gCtgMgmt.cfg.maxViewCacheNum = CTG_DEFAULT_CACHE_VIEW_NUMBER;
gCtgMgmt.cfg.maxTSMACacheNum = CTG_DEFAULT_CACHE_TSMA_NUMBER;
gCtgMgmt.cfg.dbRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.stbRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.viewRentSec = CTG_DEFAULT_RENT_SECOND;
gCtgMgmt.cfg.tsmaRentSec = CTG_DEFAULT_RENT_SECOND;
}
gCtgMgmt.pCluster = taosHashInit(CTG_DEFAULT_CACHE_CLUSTER_NUMBER, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT),
@ -862,6 +872,7 @@ int32_t catalogGetHandle(uint64_t clusterId, SCatalog** catalogHandle) {
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion)));
CTG_ERR_JRET(ctgMetaRentInit(&clusterCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion)));
clusterCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
false, HASH_ENTRY_LOCK);
@ -1562,6 +1573,16 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_
CTG_API_LEAVE(TSDB_CODE_SUCCESS);
}
int32_t catalogGetExpiredTsmas(SCatalog* pCtg, STSMAVersion** tsmas, uint32_t* num) {
CTG_API_ENTER();
if (!pCtg || !tsmas || !num) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
CTG_API_LEAVE(ctgMetaRentGet(&pCtg->tsmaRent, (void**)tsmas, num, sizeof(STSMAVersion)));
}
int32_t catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, SDbCfgInfo* pDbCfg) {
CTG_API_ENTER();
@ -1769,6 +1790,122 @@ int32_t catalogGetViewMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const SName*
CTG_API_LEAVE(TSDB_CODE_OPS_NOT_SUPPORT);
}
int32_t catalogAsyncUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** ppTsma, int32_t tsmaVersion) {
CTG_API_ENTER();
if (!pCtg || !ppTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, ppTsma, tsmaVersion, false));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogUpdateTSMA(SCatalog* pCtg, STableTSMAInfo** pTsma) {
CTG_API_ENTER();
if (!pCtg || !pTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, pTsma, 0, true));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogRemoveTSMA(SCatalog* pCtg, const STableTSMAInfo* pTsma) {
CTG_API_ENTER();
int32_t code = 0;
if (!pCtg || !pTsma) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
if (!pCtg->dbCache) {
return TSDB_CODE_SUCCESS;
}
CTG_ERR_JRET(ctgDropTbTSMAEnqueue(pCtg, pTsma, true));
_return:
CTG_API_LEAVE(code);
}
int32_t ctgGetTbTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, SArray** ppRes) {
STableTSMAInfoRsp tsmasRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTableName, &tsmasRsp, NULL, TDMT_MND_GET_TABLE_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
assert(tsmasRsp.pTsmas);
assert(tsmasRsp.pTsmas->size > 0);
*ppRes = tsmasRsp.pTsmas;
tsmasRsp.pTsmas = NULL;
for (int32_t i = 0; i < (*ppRes)->size; ++i) {
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, taosArrayGet((*ppRes), i), 0, false));
}
return TSDB_CODE_SUCCESS;
_return:
if (tsmasRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmasRsp);
}
CTG_RET(code);
}
int32_t catalogGetTableTsmas(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes) {
CTG_API_ENTER();
if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pRes) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetTbTsmas(pCtg, pConn, (SName*)pTableName, pRes));
_return:
CTG_API_LEAVE(code);
}
int32_t ctgGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
STableTSMAInfoRsp tsmaRsp = {0};
int32_t code = ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, &tsmaRsp, NULL, TDMT_MND_GET_TSMA);
if (code == TSDB_CODE_MND_SMA_NOT_EXIST) {
code = 0;
goto _return;
}
CTG_ERR_JRET(code);
ASSERT(tsmaRsp.pTsmas && tsmaRsp.pTsmas->size == 1);
*pTsma = taosArrayGetP(tsmaRsp.pTsmas, 0);
taosArrayDestroy(tsmaRsp.pTsmas);
tsmaRsp.pTsmas = NULL;
_return:
if (tsmaRsp.pTsmas) {
tFreeTableTSMAInfoRsp(&tsmaRsp);
}
CTG_RET(code);
}
int32_t catalogGetTsma(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTsmaName, STableTSMAInfo** pTsma) {
CTG_API_ENTER();
if (!pCtg || !pConn || !pTsmaName) {
CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT);
}
int32_t code = 0;
CTG_ERR_JRET(ctgGetTsma(pCtg, pConn, pTsmaName, pTsma));
_return:
CTG_API_LEAVE(code);
}
int32_t catalogClearCache(void) {
CTG_API_ENTER_NOLOCK();

438
source/libs/catalog/src/ctgAsync.c
View File

@ -476,6 +476,37 @@ int32_t ctgHandleForceUpdateView(SCatalog* pCtg, const SCatalogReq* pReq) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetTbTSMATask(SCtgJob* pJob, int32_t taskId, void* param) {
SCtgTask task = {0};
task.type = CTG_TASK_GET_TB_TSMA;
task.taskId = taskId;
task.pJob = pJob;
SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx));
if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
task.taskCtx = pTaskCtx;
pTaskCtx->pNames = param;
pTaskCtx->pResList = taosArrayInit(pJob->tbTsmaNum, sizeof(SMetaRes));
taosArrayPush(pJob->pTasks, &task);
return TSDB_CODE_SUCCESS;
}
int32_t ctgInitGetTSMATask(SCtgJob* pJob, int32_t taskId, void* param) {
SCtgTask task = {0};
task.type = CTG_TASK_GET_TSMA;
task.taskId = taskId;
task.pJob = pJob;
SCtgTbTSMACtx* pTaskCtx = taosMemoryCalloc(1, sizeof(SCtgTbTSMACtx));
if (!pTaskCtx) CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
task.taskCtx = pTaskCtx;
pTaskCtx->pNames = param;
pTaskCtx->pResList = taosArrayInit(pJob->tsmaNum, sizeof(SMetaRes));
taosArrayPush(pJob->pTasks, &task);
return 0;
}
int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, const SCatalogReq* pReq) {
SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
@ -573,6 +604,14 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob* pJob, con
ctgDropTbIndexEnqueue(pCtg, name, true);
}
for (int32_t i = 0; i < pJob->tbTsmaNum; ++i) {
STablesReq* pTbReq = taosArrayGet(pReq->pTableTSMAs, i);
for (int32_t j = 0; j < pTbReq->pTables->size; ++j) {
SName* name = taosArrayGet(pTbReq->pTables, j);
ctgDropTSMAForTbEnqueue(pCtg, name, true);
}
}
// REFRESH VIEW META
return ctgHandleForceUpdateView(pCtg, pReq);
}
@ -614,9 +653,11 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg);
int32_t tbTagNum = (int32_t)taosArrayGetSize(pReq->pTableTag);
int32_t viewNum = (int32_t)ctgGetTablesReqNum(pReq->pView);
int32_t tbTsmaNum = (int32_t)taosArrayGetSize(pReq->pTableTSMAs);
int32_t tsmaNum = (int32_t)taosArrayGetSize(pReq->pTSMAs);
int32_t taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + svrVerNum + dbCfgNum + indexNum +
userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum;
userNum + dbInfoNum + tbIndexNum + tbCfgNum + tbTagNum + viewNum + tbTsmaNum;
*job = taosMemoryCalloc(1, sizeof(SCtgJob));
if (NULL == *job) {
@ -649,6 +690,8 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
pJob->svrVerNum = svrVerNum;
pJob->tbTagNum = tbTagNum;
pJob->viewNum = viewNum;
pJob->tbTsmaNum = tbTsmaNum;
pJob->tsmaNum = tsmaNum;
#if CTG_BATCH_FETCH
pJob->pBatchs =
@ -741,6 +784,13 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo* pConn, SCtgJob** job, const
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_VIEW, pReq->pView, NULL));
}
if (tbTsmaNum > 0) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_TSMA, pReq->pTableTSMAs, NULL));
}
if (tsmaNum > 0) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TSMA, pReq->pTSMAs, NULL));
}
if (qnodeNum) {
CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL));
}
@ -2527,6 +2577,7 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
SCtgJob* pJob = pTask->pJob;
bool tbMetaDone = false;
/*
ctgIsTaskDone(pJob, CTG_TASK_GET_TB_META_BATCH, &tbMetaDone);
if (tbMetaDone) {
CTG_ERR_RET(ctgBuildViewNullRes(pTask, pCtx));
@ -2535,6 +2586,7 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
*/
int32_t dbNum = taosArrayGetSize(pCtx->pNames);
int32_t fetchIdx = 0;
@ -2572,6 +2624,386 @@ int32_t ctgLaunchGetViewsTask(SCtgTask* pTask) {
return TSDB_CODE_SUCCESS;
}
int32_t ctgAsyncRefreshTbTsma(SCtgTaskReq* pReq, const SCtgTSMAFetch* pFetch) {
int32_t code = 0;
SCtgTask* pTask = pReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SCtgTbTSMACtx* pTaskCtx = pTask->taskCtx;
SCtgDBCache* pDbCache = NULL;
STablesReq* pTbReq = taosArrayGet(pTaskCtx->pNames, pFetch->dbIdx);
ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache);
if (pDbCache) {
ctgReleaseVgInfoToCache(pCtg, pDbCache);
} else {
SBuildUseDBInput input = {0};
tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, pReq));
}
_return:
return code;
}
int32_t ctgLaunchGetTbTSMATask(SCtgTask* pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgTbTSMACtx* pCtx = (SCtgTbTSMACtx*)pTask->taskCtx;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SArray* pRes = NULL;
SCtgJob* pJob = pTask->pJob;
int32_t dbNum = taosArrayGetSize(pCtx->pNames);
int32_t fetchIdx = 0, baseResIdx = 0;
for (int32_t idx = 0; idx < dbNum; ++idx) {
STablesReq* pReq = taosArrayGet(pCtx->pNames, idx);
CTG_ERR_RET(ctgGetTbTSMAFromCache(pCtg, pCtx, idx, &fetchIdx, baseResIdx, pReq->pTables));
baseResIdx += taosArrayGetSize(pReq->pTables);
}
pCtx->fetchNum = taosArrayGetSize(pCtx->pFetches);
if (pCtx->fetchNum <= 0) {
TSWAP(pTask->res, pCtx->pResList);
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
return TSDB_CODE_SUCCESS;
}
pTask->msgCtxs = taosArrayInit_s(sizeof(SCtgMsgCtx), pCtx->fetchNum);
for (int32_t i = 0; i < pCtx->fetchNum; ++i) {
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, i);
STablesReq* pReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
SName* pName = taosArrayGet(pReq->pTables, pFetch->tbIdx);
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, i);
if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs;
SCtgTaskReq tReq;
tReq.pTask = pTask;
tReq.msgIdx = pFetch->fetchIdx;
switch (pFetch->fetchType) {
case FETCH_TSMA_SOURCE_TB_META: {
CTG_ERR_RET(ctgAsyncRefreshTbMeta(&tReq, pFetch->flag, pName, &pFetch->vgId));
} break;
case FETCH_TB_TSMA: {
CTG_ERR_RET(
ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, &tReq, TDMT_MND_GET_TABLE_TSMA));
} break;
default:
ASSERT(0);
break;
}
}
return TSDB_CODE_SUCCESS;
}
int32_t ctgLaunchGetTSMATask(SCtgTask* pTask) {
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgTbTSMACtx* pCtx = (SCtgTbTSMACtx*)pTask->taskCtx;
SRequestConnInfo* pConn = &pTask->pJob->conn;
SArray* pRes = NULL;
SCtgJob* pJob = pTask->pJob;
// currently, only support fetching one tsma
ASSERT(pCtx->pNames->size == 1);
STablesReq* pReq = taosArrayGet(pCtx->pNames, 0);
ASSERT(pReq->pTables->size == 1);
SName* pTsmaName = taosArrayGet(pReq->pTables, 0);
CTG_ERR_RET(ctgGetTSMAFromCache(pCtg, pCtx, pTsmaName));
if (pCtx->pResList->size == 0) {
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0);
if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs;
SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0};
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
CTG_ERR_RET(ctgGetTbTSMAFromMnode(pCtg, pConn, pTsmaName, NULL, &tReq, TDMT_MND_GET_TSMA));
} else {
SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0);
STableTSMAInfoRsp* pRsp = (STableTSMAInfoRsp*)pRes->pRes;
ASSERT(pRsp->pTsmas->size == 1);
const STSMACache* pTsma = taosArrayGetP(pRsp->pTsmas, 0);
TSWAP(pTask->res, pCtx->pResList);
// get tsma target stable meta if not existed in cache
int32_t exists = false;
CTG_ERR_RET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists));
if (!exists) {
SCtgTaskReq tReq = {.pTask = pTask, .msgIdx = 0};
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, 0);
if (!pMsgCtx->pBatchs) pMsgCtx->pBatchs = pJob->pBatchs;
CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, &tReq));
} else {
CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0));
}
return TSDB_CODE_SUCCESS;
}
return 0;
}
int32_t ctgHandleGetTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SMetaRes* pRes = taosArrayGet(pCtx->pResList, 0);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, 0);
SName* pName = taosArrayGet(pTbReq->pTables, 0);
SRequestConnInfo* pConn = &pTask->pJob->conn;
CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
switch (reqType) {
case TDMT_MND_TABLE_META: {
STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out;
if (!CTG_IS_META_NULL(pOut->metaType)) {
CTG_ERR_JRET(ctgUpdateTbMetaToCache(pCtg, pOut, CTG_FLAG_SYNC_OP));
}
} break;
case TDMT_MND_GET_TSMA: {
STableTSMAInfoRsp* pOut = pMsgCtx->out;
pRes->code = 0;
if (pOut->pTsmas->size > 0) {
ASSERT(pOut->pTsmas->size == 1);
pRes->pRes = pOut;
pMsgCtx->out = NULL;
TSWAP(pTask->res, pCtx->pResList);
STableTSMAInfo* pTsma = taosArrayGetP(pOut->pTsmas, 0);
int32_t exists = false;
CTG_ERR_JRET(ctgTbMetaExistInCache(pCtg, pTsma->targetDbFName, pTsma->targetTb, &exists));
if (!exists) {
TSWAP(pMsgCtx->lastOut, pMsgCtx->out);
CTG_RET(ctgGetTbMetaFromMnodeImpl(pCtg, pConn, pTsma->targetDbFName, pTsma->targetTb, NULL, tReq));
}
}
} break;
default:
ASSERT(0);
}
_return:
if (code) {
if (TSDB_CODE_MND_SMA_NOT_EXIST == code) {
code = TSDB_CODE_SUCCESS;
} else {
ctgTaskError("Get tsma for %d.%s.%s failed with err: %s", pName->acctId, pName->dbname, pName->tname,
tstrerror(code));
}
}
ctgHandleTaskEnd(pTask, code);
CTG_RET(code);
}
static int32_t ctgTsmaFetchStreamProgress(SCtgTaskReq* tReq, SHashObj* pVgHash, const STableTSMAInfoRsp* pTsmas) {
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
int32_t subFetchIdx = 0;
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
SRequestConnInfo* pConn = &pTask->pJob->conn;
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
const SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
SVgroupInfo* pVgInfo = NULL;
pFetch->vgNum = taosHashGetSize(pVgHash);
for (int32_t i = 0; i < taosArrayGetSize(pTsmas->pTsmas); ++i) {
STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas->pTsmas, i);
pVgInfo = taosHashIterate(pVgHash, NULL);
pTsmaInfo->reqTs = taosGetTimestampMs();
while (pVgInfo) {
// make StreamProgressReq, send it
SStreamProgressReq req = {.fetchIdx = pFetch->fetchIdx,
.streamId = pTsmaInfo->streamUid,
.subFetchIdx = subFetchIdx++,
.vgId = pVgInfo->vgId};
CTG_ERR_JRET(ctgGetStreamProgressFromVnode(pCtg, pConn, pTbName, pVgInfo, NULL, tReq, &req));
pFetch->subFetchNum++;
pVgInfo = taosHashIterate(pVgHash, pVgInfo);
}
}
_return:
CTG_RET(code);
}
int32_t ctgHandleGetTbTSMARsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) {
bool taskDone = false;
int32_t code = 0;
SCtgTask* pTask = tReq->pTask;
SCatalog* pCtg = pTask->pJob->pCtg;
SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx);
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
SArray* pTsmas = NULL;
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx);
SHashObj* pVgHash = NULL;
SCtgDBCache* pDbCache = NULL;
STableTSMAInfo* pTsma = NULL;
SRequestConnInfo* pConn = &pTask->pJob->conn;
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
SName* pTbName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
if (reqType != TDMT_VND_GET_STREAM_PROGRESS)
CTG_ERR_JRET(ctgProcessRspMsg(pMsgCtx->out, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
switch (reqType) {
case TDMT_MND_GET_TABLE_TSMA: {
STableTSMAInfoRsp* pOut = pMsgCtx->out;
pFetch->fetchType = FETCH_TSMA_STREAM_PROGRESS;
pRes->pRes = pOut;
pMsgCtx->out = NULL;
if (pOut->pTsmas && taosArrayGetSize(pOut->pTsmas) > 0) {
// fetch progress
ctgAcquireVgInfoFromCache(pCtg, pTbReq->dbFName, &pDbCache);
if (!pDbCache) {
// do not know which vnodes to fetch, fetch vnode list first
SBuildUseDBInput input = {0};
tstrncpy(input.db, pTbReq->dbFName, tListLen(input.db));
input.vgVersion = CTG_DEFAULT_INVALID_VERSION;
CTG_ERR_JRET(ctgGetDBVgInfoFromMnode(pCtg, pConn, &input, NULL, tReq));
} else {
// fetch progress from every vnode
CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pDbCache->vgCache.vgInfo->vgHash, pOut));
ctgReleaseVgInfoToCache(pCtg, pDbCache);
pDbCache = NULL;
}
} else {
// no tsmas
if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) {
TSWAP(pTask->res, pCtx->pResList);
taskDone = true;
}
}
} break;
case TDMT_VND_GET_STREAM_PROGRESS: {
SStreamProgressRsp rsp = {0};
CTG_ERR_JRET(ctgProcessRspMsg(&rsp, reqType, pMsg->pData, pMsg->len, rspCode, pMsgCtx->target));
// update progress into res
STableTSMAInfoRsp* pTsmasRsp = pRes->pRes;
SArray* pTsmas = pTsmasRsp->pTsmas;
SStreamProgressRsp* pRsp = &rsp;
int32_t tsmaIdx = pRsp->subFetchIdx / pFetch->vgNum;
STableTSMAInfo* pTsmaInfo = taosArrayGetP(pTsmas, tsmaIdx);
if (pTsmaInfo->rspTs == 0) pTsmaInfo->fillHistoryFinished = true;
pTsmaInfo->rspTs = taosGetTimestampMs();
pTsmaInfo->delayDuration = TMAX(pRsp->progressDelay, pTsmaInfo->delayDuration);
pTsmaInfo->fillHistoryFinished = pTsmaInfo->fillHistoryFinished && pRsp->fillHisFinished;
qDebug("received stream progress for tsma %s rsp history: %d vnode: %d, delay: %" PRId64, pTsmaInfo->name,
pRsp->fillHisFinished, pRsp->subFetchIdx, pRsp->progressDelay);
if (atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) == pFetch->subFetchNum) {
// subfetch all finished
for (int32_t i = 0; i < taosArrayGetSize(pTsmas); ++i) {
STableTSMAInfo* pInfo = taosArrayGetP(pTsmas, i);
CTG_ERR_JRET(tCloneTbTSMAInfo(pInfo, &pTsma));
CTG_ERR_JRET(ctgUpdateTbTSMAEnqueue(pCtg, &pTsma, 0, false));
}
if (atomic_sub_fetch_32(&pCtx->fetchNum, 1) == 0) {
TSWAP(pTask->res, pCtx->pResList);
taskDone = true;
}
}
} break;
case TDMT_MND_USE_DB: {
SUseDbOutput* pOut = (SUseDbOutput*)pMsgCtx->out;
switch (pFetch->fetchType) {
case FETCH_TSMA_SOURCE_TB_META: {
SVgroupInfo vgInfo = {0};
CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, &pConn->mgmtEps, pOut->dbVgroup, pTbName, &vgInfo));
pFetch->vgId = vgInfo.vgId;
CTG_ERR_JRET(ctgGetTbMetaFromVnode(pCtg, pConn, pTbName, &vgInfo, NULL, tReq));
} break;
case FETCH_TSMA_STREAM_PROGRESS: {
STableTSMAInfoRsp* pTsmas = pRes->pRes;
TSWAP(pOut->dbVgroup->vgHash, pVgHash);
CTG_ERR_JRET(ctgTsmaFetchStreamProgress(tReq, pVgHash, pTsmas));
} break;
default:
ASSERT(0);
}
} break;
case TDMT_VND_TABLE_META: {
// handle source tb meta
ASSERT(pFetch->fetchType == FETCH_TSMA_SOURCE_TB_META);
STableMetaOutput* pOut = (STableMetaOutput*)pMsgCtx->out;
pFetch->fetchType = FETCH_TB_TSMA;
pFetch->tsmaSourceTbName = *pTbName;
if (CTG_IS_META_NULL(pOut->metaType)) {
ctgTaskError("no tbmeta found when fetching tsma source tb meta: %s.%s", pTbName->dbname, pTbName->tname);
ctgRemoveTbMetaFromCache(pCtg, pTbName, false);
CTG_ERR_JRET(CTG_ERR_CODE_TABLE_NOT_EXIST);
}
if (META_TYPE_BOTH_TABLE == pOut->metaType) {
// rewrite tsma fetch table with it's super table name
sprintf(pFetch->tsmaSourceTbName.tname, "%s", pOut->tbName);
}
CTG_ERR_JRET(ctgGetTbTSMAFromMnode(pCtg, pConn, &pFetch->tsmaSourceTbName, NULL, tReq, TDMT_MND_GET_TABLE_TSMA));
} break;
default:
ASSERT(0);
}
_return:
if (pDbCache) {
ctgReleaseVgInfoToCache(pCtg, pDbCache);
}
if (pTsma) {
tFreeTableTSMAInfo(pTsma);
pTsma = NULL;
}
if (pVgHash) {
taosHashCleanup(pVgHash);
}
if (code) {
SMetaRes* pRes = taosArrayGet(pCtx->pResList, pFetch->resIdx);
pRes->code = code;
if (TSDB_CODE_MND_SMA_NOT_EXIST == code) {
code = TSDB_CODE_SUCCESS;
} else {
ctgTaskError("Get tsma for %d.%s.%s faield with err: %s", pTbName->acctId, pTbName->dbname, pTbName->tname,
tstrerror(code));
}
bool allSubFetchFinished = false;
if (pMsgCtx->reqType == TDMT_VND_GET_STREAM_PROGRESS) {
allSubFetchFinished = atomic_add_fetch_32(&pFetch->finishedSubFetchNum, 1) >= pFetch->subFetchNum;
}
if ((allSubFetchFinished || pFetch->subFetchNum == 0) && 0 == atomic_sub_fetch_32(&pCtx->fetchNum, 1)) {
TSWAP(pTask->res, pCtx->pResList);
taskDone = true;
}
}
if (pTask->res && taskDone) {
ctgHandleTaskEnd(pTask, code);
}
CTG_RET(code);
}
int32_t ctgDumpTbTSMARes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pTableTsmas = pTask->res;
return TSDB_CODE_SUCCESS;
}
int32_t ctgDumpTSMARes(SCtgTask* pTask) {
if (pTask->subTask) {
return TSDB_CODE_SUCCESS;
}
SCtgJob* pJob = pTask->pJob;
pJob->jobRes.pTsmas = pTask->res;
return TSDB_CODE_SUCCESS;
}
int32_t ctgRelaunchGetTbMetaTask(SCtgTask* pTask) {
ctgResetTbMetaTask(pTask);
@ -2690,6 +3122,8 @@ SCtgAsyncFps gCtgAsyncFps[] = {
{ctgInitGetTbHashsTask, ctgLaunchGetTbHashsTask, ctgHandleGetTbHashsRsp, ctgDumpTbHashsRes, NULL, NULL},
{ctgInitGetTbTagTask, ctgLaunchGetTbTagTask, ctgHandleGetTbTagRsp, ctgDumpTbTagRes, NULL, NULL},
{ctgInitGetViewsTask, ctgLaunchGetViewsTask, ctgHandleGetViewsRsp, ctgDumpViewsRes, NULL, NULL},
{ctgInitGetTbTSMATask, ctgLaunchGetTbTSMATask, ctgHandleGetTbTSMARsp, ctgDumpTbTSMARes, NULL, NULL},
{ctgInitGetTSMATask, ctgLaunchGetTSMATask, ctgHandleGetTSMARsp, ctgDumpTSMARes, NULL, NULL},
};
int32_t ctgMakeAsyncRes(SCtgJob* pJob) {
@ -2821,7 +3255,7 @@ int32_t ctgLaunchJob(SCtgJob* pJob) {
CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask));
pTask = taosArrayGet(pJob->pTasks, i);
pTask->status = CTG_TASK_LAUNCHED;
atomic_val_compare_exchange_32((int32_t*)&pTask->status, 0, CTG_TASK_LAUNCHED);
}
if (taskNum <= 0) {

515
source/libs/catalog/src/ctgCache.c
View File

@ -32,6 +32,8 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v
{CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex},
{CTG_OP_UPDATE_VIEW_META, "update viewMeta", ctgOpUpdateViewMeta},
{CTG_OP_DROP_VIEW_META, "drop viewMeta", ctgOpDropViewMeta},
{CTG_OP_UPDATE_TB_TSMA, "update tbTSMA", ctgOpUpdateTbTSMA},
{CTG_OP_DROP_TB_TSMA, "drop tbTSMA", ctgOpDropTbTSMA},
{CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}};
SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = {
@ -52,6 +54,7 @@ SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = {
{"TblTag ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TAG,
{"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_INDEX_INFO,
{"viewMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_VIEW,
{"TblTSMA ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_TSMA
{"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_USER,
{"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_UDF,
{"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER,
@ -209,6 +212,17 @@ void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *
}
}
void ctgReleaseTSMAToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTSMACache* pCache) {
if (pCache && dbCache) {
CTG_UNLOCK(CTG_READ, &pCache->tsmaLock);
taosHashRelease(dbCache->tsmaCache, pCache);
}
if (dbCache) {
ctgReleaseDBCache(pCtg, dbCache);
}
}
int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) {
SCtgDBCache *dbCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
@ -245,7 +259,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) {
int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) {
SCtgDBCache *dbCache = NULL;
SCtgTbCache *pCache = NULL;
ctgAcquireDBCache(pCtg, dbFName, &dbCache);
@ -489,7 +503,7 @@ _return:
return TSDB_CODE_SUCCESS;
}
int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) {
int32_t ctgTbMetaExistInCache(SCatalog *pCtg, const char *dbFName, const char *tbName, int32_t *exist) {
SCtgDBCache *dbCache = NULL;
SCtgTbCache *tbCache = NULL;
ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache);
@ -1381,6 +1395,13 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
newDBCache.tsmaCache = taosHashInit(gCtgMgmt.cfg.maxTSMACacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY),
true, HASH_ENTRY_LOCK);
if (!newDBCache.tsmaCache) {
ctgError("taosHashInit %d tsmaCache failed", gCtgMgmt.cfg.maxTSMACacheNum);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
code = taosHashPut(pCtg->dbCache, dbFName, strlen(dbFName), &newDBCache, sizeof(SCtgDBCache));
if (code) {
if (HASH_NODE_EXIST(code)) {
@ -1394,7 +1415,7 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) {
CTG_CACHE_NUM_INC(CTG_CI_DB, 1);
SDbCacheInfo dbCacheInfo = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0, .cfgVersion = -1};
SDbCacheInfo dbCacheInfo = {.dbId = newDBCache.dbId, .vgVersion = -1, .stateTs = 0, .cfgVersion = -1, .tsmaVersion = -1};
tstrncpy(dbCacheInfo.dbFName, dbFName, sizeof(dbCacheInfo.dbFName));
ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId);
@ -1424,6 +1445,7 @@ int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *d
atomic_store_8(&dbCache->deleted, 1);
ctgRemoveStbRent(pCtg, dbCache);
ctgRemoveViewRent(pCtg, dbCache);
ctgRemoveTSMARent(pCtg, dbCache);
ctgFreeDbCache(dbCache);
CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock);
@ -1780,7 +1802,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
bool newAdded = false;
SDbCacheInfo dbCacheInfo = {
.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .cfgVersion = -1, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs};
.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .cfgVersion = -1, .numOfTable = dbInfo->numOfTable, .stateTs = dbInfo->stateTs, .tsmaVersion = -1};
SCtgDBCache *dbCache = NULL;
CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache));
@ -1824,6 +1846,7 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) {
if (dbCache->cfgCache.cfgInfo) {
dbCacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion;
dbCacheInfo.tsmaVersion = dbCache->tsmaVersion;
}
vgCache->vgInfo = dbInfo;
@ -1891,6 +1914,7 @@ int32_t ctgOpUpdateDbCfg(SCtgCacheOperation *operation) {
} else {
cacheInfo.vgVersion = -1;
}
cacheInfo.tsmaVersion = dbCache->tsmaVersion;
ctgWLockDbCfgInfo(dbCache);
@ -2074,7 +2098,8 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) {
}
tblType = pTbCache->pMeta->tableType;
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex));
atomic_sub_fetch_64(&dbCache->dbCacheSize,
ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex));
ctgFreeTbCacheImpl(pTbCache, true);
if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) {
@ -2126,7 +2151,8 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) {
}
tblType = pTbCache->pMeta->tableType;
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) + ctgGetTbIndexCacheSize(pTbCache->pIndex));
atomic_sub_fetch_64(&dbCache->dbCacheSize, ctgGetTbMetaCacheSize(pTbCache->pMeta) +
ctgGetTbIndexCacheSize(pTbCache->pIndex));
ctgFreeTbCacheImpl(pTbCache, true);
if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) {
@ -2554,6 +2580,7 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
case CTG_OP_UPDATE_VG_EPSET:
case CTG_OP_DROP_TB_INDEX:
case CTG_OP_DROP_VIEW_META:
case CTG_OP_DROP_TB_TSMA:
case CTG_OP_CLEAR_CACHE: {
taosMemoryFreeClear(op->data);
break;
@ -2582,6 +2609,14 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) {
taosMemoryFreeClear(op->data);
break;
}
case CTG_OP_UPDATE_TB_TSMA: {
SCtgUpdateTbTSMAMsg *msg = op->data;
if (msg->pTsma) {
tFreeTableTSMAInfo(msg->pTsma);
taosMemoryFreeClear(msg->pTsma);
}
break;
}
default: {
qError("invalid cache op id:%d", op->opId);
break;
@ -2982,11 +3017,7 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq
CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta));
if (NULL == tblMeta) {
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
return TSDB_CODE_SUCCESS;
}
if (NULL != tblMeta) {
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTableName, dbFName);
@ -2995,6 +3026,11 @@ int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq
} else {
CTG_ERR_JRET(ctgDropTbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq));
}
} else {
ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname);
}
CTG_ERR_JRET(ctgDropTSMAForTbEnqueue(pCtg, pTableName, syncReq));
_return:
@ -3136,5 +3172,462 @@ int32_t ctgGetViewsFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgViewsC
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, int32_t dbIdx, int32_t* fetchIdx, int32_t baseResIdx,
SArray* pList) {
int32_t code = 0;
SCtgDBCache * dbCache = NULL;
SCtgTSMACache *pCache = NULL;
char dbFName[TSDB_DB_FNAME_LEN] = {0};
int32_t flag = CTG_FLAG_UNKNOWN_STB;
uint64_t lastSuid = 0;
STableMeta * pTableMeta = NULL;
SName * pName = taosArrayGet(pList, 0);
int32_t tbNum = taosArrayGetSize(pList);
SCtgTbCache * pTbCache = NULL;
if (IS_SYS_DBNAME(pName->dbname)) {
return TSDB_CODE_SUCCESS;
}
tNameGetFullDbName(pName, dbFName);
// get db cache
CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &dbCache));
if (!dbCache) {
ctgDebug("DB %s not in cache", dbFName);
for (int32_t i = 0; i < tbNum; ++i) {
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaData){0});
}
return TSDB_CODE_SUCCESS;
}
for (int32_t i = 0; i < tbNum; ++i) {
// get tb cache
pName = taosArrayGet(pList, i);
pTbCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname));
if (!pTbCache || !pTbCache->pMeta) {
ctgDebug("tb: %s.%s not in cache", dbFName, pName->tname);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
uint64_t suid = pTbCache->pMeta->suid;
int8_t tbType = pTbCache->pMeta->tableType;
taosHashRelease(dbCache->tbCache, pTbCache);
SName tsmaSourceTbName = *pName;
// if child table, get stable name
if (tbType == TSDB_CHILD_TABLE) {
char* stbName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(uint64_t));
if (stbName) {
snprintf(tsmaSourceTbName.tname, TMIN(TSDB_TABLE_NAME_LEN, strlen(stbName) + 1), "%s", stbName);
taosHashRelease(dbCache->stbCache, stbName);
} else {
ctgDebug("stb in db: %s, uid: %" PRId64 " not in cache", dbFName, suid);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TSMA_SOURCE_TB_META, NULL);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
}
// get tsma cache
pCache = taosHashAcquire(dbCache->tsmaCache, tsmaSourceTbName.tname, strlen(tsmaSourceTbName.tname));
if (!pCache || !pCache->pTsmas || pCache->pTsmas->size == 0) {
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
continue;
}
CTG_LOCK(CTG_READ, &pCache->tsmaLock);
if (hasOutOfDateTSMACache(pCache->pTsmas)) {
CTG_UNLOCK(CTG_READ, &pCache->tsmaLock);
taosHashRelease(dbCache->tsmaCache, pCache);
ctgDebug("tsma for tb: %s.%s not in cache", tsmaSourceTbName.tname, dbFName);
ctgAddTSMAFetch(&pCtx->pFetches, dbIdx, i, fetchIdx, baseResIdx + i, flag, FETCH_TB_TSMA, &tsmaSourceTbName);
taosArrayPush(pCtx->pResList, &(SMetaRes){0});
CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1);
continue;
}
CTG_CACHE_HIT_INC(CTG_CI_TBL_TSMA, 1);
STableTSMAInfoRsp *pRsp = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (!pRsp) {
ctgReleaseTSMAToCache(pCtg, dbCache, pCache);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
pRsp->pTsmas = taosArrayInit(pCache->pTsmas->size, POINTER_BYTES);
if (!pRsp->pTsmas) {
ctgReleaseTSMAToCache(pCtg, dbCache, pCache);
taosMemoryFreeClear(pRsp);
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
SMetaRes res = {0};
for (int32_t i = 0; i < pCache->pTsmas->size; ++i) {
STSMACache *pTsmaOut = NULL;
STSMACache *pTsmaCache = taosArrayGetP(pCache->pTsmas, i);
code = tCloneTbTSMAInfo(pTsmaCache, &pTsmaOut);
if (code) {
ctgReleaseTSMAToCache(pCtg, dbCache, pCache);
tFreeTableTSMAInfoRsp(pRsp);
taosMemoryFreeClear(pRsp);
CTG_ERR_RET(code);
}
taosArrayPush(pRsp->pTsmas, &pTsmaOut);
}
res.pRes = pRsp;
taosArrayPush(pCtx->pResList, &res);
CTG_UNLOCK(CTG_READ, &pCache->tsmaLock);
taosHashRelease(dbCache->tsmaCache, pCache);
}
ctgReleaseDBCache(pCtg, dbCache);
CTG_RET(code);
}
int32_t ctgGetTSMAFromCache(SCatalog* pCtg, SCtgTbTSMACtx* pCtx, SName* pTsmaName) {
char dbFName[TSDB_DB_FNAME_LEN] = {0};
SCtgDBCache *pDbCache = NULL;
int32_t code = TSDB_CODE_SUCCESS;
SMetaRes res = {0};
bool found = false;
STSMACache * pTsmaOut = NULL;
tNameGetFullDbName(pTsmaName, dbFName);
CTG_ERR_RET(ctgAcquireDBCache(pCtg, dbFName, &pDbCache));
if (!pDbCache) {
ctgDebug("DB %s not in cache", dbFName);
CTG_RET(code);
}
void *pIter = taosHashIterate(pDbCache->tsmaCache, NULL);
while (pIter && !found) {
SCtgTSMACache* pCtgCache = pIter;
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
int32_t size = pCtgCache ? (pCtgCache->pTsmas ? pCtgCache->pTsmas->size : 0) : 0;
for (int32_t i = 0; i < size; ++i) {
STSMACache* pCache = taosArrayGetP(pCtgCache->pTsmas, i);
if (memcmp(pCache->name, pTsmaName->tname, TSDB_TABLE_NAME_LEN) == 0) {
found = true;
CTG_CACHE_NHIT_INC(CTG_CI_TBL_TSMA, 1);
code = tCloneTbTSMAInfo(pCache, &pTsmaOut);
break;
}
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
pIter = taosHashIterate(pDbCache->tsmaCache, pIter);
}
taosHashCancelIterate(pDbCache->tsmaCache, pIter);
if (found && code == TSDB_CODE_SUCCESS) {
res.pRes = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (!res.pRes) {
tFreeAndClearTableTSMAInfo(pTsmaOut);
CTG_RET(TSDB_CODE_OUT_OF_MEMORY);
}
STableTSMAInfoRsp* pRsp = res.pRes;
pRsp->pTsmas = taosArrayInit(1, POINTER_BYTES);
if (!pRsp->pTsmas) {
tFreeAndClearTableTSMAInfo(pTsmaOut);
CTG_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pRsp->pTsmas, &pTsmaOut);
taosArrayPush(pCtx->pResList, &res);
}
ctgReleaseDBCache(pCtg, pDbCache);
CTG_RET(code);
}
int32_t ctgUpdateTbTSMAEnqueue(SCatalog *pCtg, STSMACache **pTsma, int32_t tsmaVersion, bool syncOp) {
int32_t code = 0;
SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
op->opId = CTG_OP_UPDATE_TB_TSMA;
op->syncOp = syncOp;
SCtgUpdateTbTSMAMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbTSMAMsg));
if (NULL == msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbTSMAMsg));
taosMemoryFree(op);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
msg->pTsma = *pTsma;
msg->dbTsmaVersion = tsmaVersion;
msg->dbId = (*pTsma)->dbId;
op->data = msg;
CTG_ERR_JRET(ctgEnqueue(pCtg, op));
*pTsma = NULL;
return TSDB_CODE_SUCCESS;
_return:
CTG_RET(code);
}
int32_t ctgDropTbTSMAEnqueue(SCatalog* pCtg, const STSMACache* pTsma, bool syncOp) {
int32_t code = 0;
SCtgCacheOperation* op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
if (!op) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
op->opId = CTG_OP_DROP_TB_TSMA;
op->syncOp = syncOp;
SCtgDropTbTSMAMsg* msg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg));
if (!msg) {
ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbTSMAMsg));
taosMemoryFree(op);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
msg->pCtg = pCtg;
msg->dbId = pTsma->dbId;
msg->tbId = pTsma->suid;
msg->tsmaId = pTsma->tsmaId;
tstrncpy(msg->dbFName, pTsma->dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(msg->tbName, pTsma->tb, TSDB_TABLE_NAME_LEN);
tstrncpy(msg->tsmaName, pTsma->name, TSDB_TABLE_NAME_LEN);
op->data = msg;
CTG_ERR_JRET(ctgEnqueue(pCtg, op));
return TSDB_CODE_SUCCESS;
_return:
CTG_RET(code);
}
static SCtgCacheOperation* createDropAllTbTsmaCtgCacheOp(SCatalog* pCtg, const STSMACache* pCache, bool syncOp) {
SCtgCacheOperation* pOp = taosMemoryCalloc(1, sizeof(SCtgCacheOperation));
if (!pOp) return NULL;
SCtgDropTbTSMAMsg* pMsg = taosMemoryCalloc(1, sizeof(SCtgDropTbTSMAMsg));
if (!pMsg) {
taosMemoryFree(pOp);
return NULL;
}
pOp->opId = CTG_OP_DROP_TB_TSMA;
pOp->syncOp = syncOp;
pMsg->pCtg = pCtg;
pMsg->dbId = pCache->dbId;
pMsg->tbId = pCache->suid;
pMsg->tsmaId = pCache->tsmaId;
pMsg->dropAllForTb = true;
tstrncpy(pMsg->tsmaName, pCache->name, TSDB_TABLE_NAME_LEN);
tstrncpy(pMsg->dbFName, pCache->dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(pMsg->tbName, pCache->tb, TSDB_TABLE_NAME_LEN);
pOp->data = pMsg;
return pOp;
}
int32_t ctgDropTSMAForTbEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) {
ctgDebug("drop tsma meta for tb: %s.%s", pName->dbname, pName->tname);
int32_t code = 0;
SCtgDBCache* pDbCache = NULL;
SCtgCacheOperation* pOp = NULL;
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pName, dbFName);
CTG_ERR_JRET(ctgGetDBCache(pCtg, dbFName, &pDbCache));
if (NULL == pDbCache || !pDbCache->tsmaCache) {
goto _return;
}
SCtgTSMACache *pCtgCache = taosHashGet(pDbCache->tsmaCache, pName->tname, strlen(pName->tname));
if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) {
goto _return;
}
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
STSMACache *pCache = taosArrayGetP(pCtgCache->pTsmas, 0);
pOp = createDropAllTbTsmaCtgCacheOp(pCtg, pCache, syncOp);
if (!pOp) {
code = TSDB_CODE_OUT_OF_MEMORY;
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
goto _return;
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
CTG_ERR_JRET(ctgEnqueue(pCtg, pOp));
return TSDB_CODE_SUCCESS;
_return:
if (pOp) {
taosMemoryFree(pOp->data);
taosMemoryFree(pOp);
}
CTG_RET(code);
}
int32_t ctgWriteTbTSMAToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName,
STSMACache **ppTsmaCache) {
if (NULL == dbCache->tsmaCache) {
ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId);
CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED);
}
STSMACache *pTsmaCache = *ppTsmaCache;
int32_t code = TSDB_CODE_SUCCESS;
SCtgTSMACache* pCache = taosHashGet(dbCache->tsmaCache, tbName, strlen(tbName));
if (!pCache) {
SCtgTSMACache cache = {0};
cache.pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES));
if (!cache.pTsmas) CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
taosArrayPush(cache.pTsmas, &pTsmaCache);
if (taosHashPut(dbCache->tsmaCache, tbName, strlen(tbName), &cache, sizeof(cache))) {
ctgError("taosHashPut new tsmacache for tb: %s.%s failed", dbFName, tbName);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
atomic_add_fetch_64(&dbCache->dbCacheSize, strlen(tbName) + sizeof(STSMACache) + ctgGetTbTSMACacheSize(pTsmaCache));
CTG_DB_NUM_INC(CTG_CI_TBL_TSMA);
ctgDebug("tb %s tsma updated to cache, name: %s", tbName, pTsmaCache->name);
CTG_ERR_JRET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache));
*ppTsmaCache = NULL;
goto _return;
}
CTG_LOCK(CTG_WRITE, &pCache->tsmaLock);
if (pCache->pTsmas) {
uint64_t cacheSize = 0;
for (int32_t i = 0; i < pCache->pTsmas->size; ++i) {
STableTSMAInfo* pInfo = taosArrayGetP(pCache->pTsmas, i);
if (pInfo->tsmaId == pTsmaCache->tsmaId) {
ctgDebug("tsma: %s removed from cache, history from %d to %d, reqTs from %" PRId64 " to %" PRId64
"rspTs from %" PRId64 " to %" PRId64 " delay from %" PRId64 " to %" PRId64,
pInfo->name, pInfo->fillHistoryFinished, pTsmaCache->fillHistoryFinished, pInfo->reqTs,
pTsmaCache->reqTs, pInfo->rspTs, pTsmaCache->rspTs, pInfo->delayDuration, pTsmaCache->delayDuration);
cacheSize = ctgGetTbTSMACacheSize(pInfo);
taosArrayRemove(pCache->pTsmas, i);
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
tFreeTableTSMAInfo(pInfo);
taosMemoryFreeClear(pInfo);
break;
}
}
} else {
pCache->pTsmas = taosArrayInit(4, sizeof(POINTER_BYTES));
if (!pCache->pTsmas) {
CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock);
CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY);
}
}
// push the new cache
taosArrayPush(pCache->pTsmas, &pTsmaCache);
*ppTsmaCache = NULL;
atomic_add_fetch_64(&dbCache->dbCacheSize, ctgGetTbTSMACacheSize(pTsmaCache));
CTG_ERR_RET(ctgUpdateRentTSMAVersion(pCtg, dbFName, pTsmaCache));
CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock);
ctgDebug("table %s tsma updated to cache, tsma: %s", tbName, pTsmaCache->name);
_return:
CTG_RET(code);
}
int32_t ctgOpDropTbTSMA(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgDropTbTSMAMsg * msg = operation->data;
SCatalog *pCtg = msg->pCtg;
SCtgDBCache *dbCache = NULL;
if (pCtg->stopUpdate) {
goto _return;
}
CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache));
if (NULL == dbCache || !dbCache->tsmaCache || (msg->dbId != dbCache->dbId && msg->dbId != 0)) {
goto _return;
}
SCtgTSMACache* pCtgCache = taosHashGet(dbCache->tsmaCache, msg->tbName, strlen(msg->tbName));
if (!pCtgCache || !pCtgCache->pTsmas || pCtgCache->pTsmas->size == 0) {
goto _return;
}
uint64_t cacheSize = 0;
STSMACache *pCache = NULL;
if (msg->dropAllForTb) {
CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock);
for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) {
pCache = taosArrayGetP(pCtgCache->pTsmas, i);
cacheSize += ctgGetTbTSMACacheSize(pCache);
ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare);
CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA);
}
taosArrayDestroyP(pCtgCache->pTsmas, tFreeAndClearTableTSMAInfo);
pCtgCache->pTsmas = NULL;
ctgDebug("all tsmas for table dropped: %s.%s", msg->dbFName, msg->tbName);
taosHashRemove(dbCache->tsmaCache, msg->tbName, TSDB_TABLE_NAME_LEN);
CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock);
} else {
CTG_LOCK(CTG_WRITE, &pCtgCache->tsmaLock);
pCache = taosArrayGetP(pCtgCache->pTsmas, 0);
if (msg->tbId != 0 && pCache->suid != msg->tbId) {
// table id mismatch, skip drops
CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock);
goto _return;
}
for (int32_t i = 0; i < pCtgCache->pTsmas->size; ++i) {
pCache = taosArrayGetP(pCtgCache->pTsmas, i);
if (pCache->tsmaId != msg->tsmaId) {
continue;
}
cacheSize = ctgGetTbTSMACacheSize(pCache);
ctgMetaRentRemove(&msg->pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSearchCompare, ctgTSMAVersionSearchCompare);
taosArrayRemove(pCtgCache->pTsmas, i);
tFreeAndClearTableTSMAInfo(pCache);
CTG_DB_NUM_DEC(CTG_CI_TBL_TSMA);
break;
}
CTG_UNLOCK(CTG_WRITE, &pCtgCache->tsmaLock);
}
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
_return:
taosMemoryFreeClear(msg);
CTG_RET(code);
}
int32_t ctgOpUpdateTbTSMA(SCtgCacheOperation *operation) {
int32_t code = 0;
SCtgUpdateTbTSMAMsg *msg = operation->data;
SCatalog * pCtg = msg->pCtg;
STableTSMAInfo * pTsmaInfo = msg->pTsma;
SCtgDBCache * dbCache = NULL;
if (pCtg->stopUpdate) {
goto _return;
}
CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pTsmaInfo->dbFName, pTsmaInfo->dbId, &dbCache));
CTG_ERR_JRET(ctgWriteTbTSMAToCache(pCtg, dbCache, pTsmaInfo->dbFName, pTsmaInfo->tb, &pTsmaInfo));
if (dbCache && msg->dbTsmaVersion > 0) {
dbCache->tsmaVersion = msg->dbTsmaVersion;
SDbCacheInfo cacheInfo = {0};
cacheInfo.dbId = dbCache->dbId;
if (dbCache->cfgCache.cfgInfo) {
cacheInfo.cfgVersion = dbCache->cfgCache.cfgInfo->cfgVersion;
tstrncpy(cacheInfo.dbFName, dbCache->cfgCache.cfgInfo->db, TSDB_DB_FNAME_LEN);
}
if (dbCache->vgCache.vgInfo) {
cacheInfo.vgVersion = dbCache->vgCache.vgInfo->vgVersion;
cacheInfo.numOfTable = dbCache->vgCache.vgInfo->numOfTable;
cacheInfo.stateTs = dbCache->vgCache.vgInfo->stateTs;
}
cacheInfo.tsmaVersion = dbCache->tsmaVersion;
CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &cacheInfo, cacheInfo.dbId, sizeof(SDbCacheInfo),
ctgDbCacheInfoSortCompare, ctgDbCacheInfoSearchCompare));
}
_return:
if (pTsmaInfo) {
tFreeTableTSMAInfo(pTsmaInfo);
taosMemoryFreeClear(pTsmaInfo);
}
taosMemoryFreeClear(msg);
CTG_RET(code);
}

167
source/libs/catalog/src/ctgRemote.c
View File

@ -337,6 +337,35 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize,
qDebug("Got view-meta from mnode, viewFName:%s", target);
break;
}
case TDMT_MND_GET_TSMA:
case TDMT_MND_GET_TABLE_TSMA: {
if (TSDB_CODE_SUCCESS != rspCode) {
if (TSDB_CODE_MND_SMA_NOT_EXIST != rspCode) {
qError("error rsp for get table tsma, error:%s, tbFName:%s", tstrerror(rspCode), target);
}
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get table tsma rsp failed, error:%s, tbFName:%s", tstrerror(code), target);
CTG_ERR_RET(code);
}
qDebug("Got table tsma from mnode, tbFName:%s", target);
break;
}
case TDMT_VND_GET_STREAM_PROGRESS: {
if (TSDB_CODE_SUCCESS != rspCode) {
CTG_ERR_RET(rspCode);
}
code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize);
if (code) {
qError("Process get stream progress rsp failed, err: %s, tbFName: %s", tstrerror(code), target);
CTG_ERR_RET(code);
}
break;
}
default:
if (TSDB_CODE_SUCCESS != rspCode) {
qError("Got error rsp, error:%s", tstrerror(rspCode));
@ -528,10 +557,20 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
pName = ctgGetFetchName(ctx->pNames, fetch);
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
pName = ctx->pName;
}
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -578,10 +617,20 @@ int32_t ctgAddBatch(SCatalog* pCtg, int32_t vgId, SRequestConnInfo* pConn, SCtgT
SCtgTbMetasCtx* ctx = (SCtgTbMetasCtx*)pTask->taskCtx;
SCtgFetch* fetch = taosArrayGet(ctx->pFetchs, tReq->msgIdx);
pName = ctgGetFetchName(ctx->pNames, fetch);
} else if (CTG_TASK_GET_TB_TSMA == pTask->type){
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
SCtgTbMetaCtx* ctx = (SCtgTbMetaCtx*)pTask->taskCtx;
pName = ctx->pName;
}
} else if (TDMT_VND_GET_STREAM_PROGRESS == msgType) {
SCtgTbTSMACtx* pCtx = pTask->taskCtx;
SCtgTSMAFetch* pFetch = taosArrayGet(pCtx->pFetches, tReq->msgIdx);
STablesReq* pTbReq = taosArrayGet(pCtx->pNames, pFetch->dbIdx);
pName = taosArrayGet(pTbReq->pTables, pFetch->tbIdx);
} else {
ctgError("invalid vnode msgType %d", msgType);
CTG_ERR_JRET(TSDB_CODE_APP_ERROR);
@ -1104,7 +1153,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, char* dbFName, char* tbName,
int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo* pConn, const char* dbFName, const char* tbName,
STableMetaOutput* out, SCtgTaskReq* tReq) {
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName};
@ -1463,4 +1512,120 @@ int32_t ctgGetViewInfoFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, SName*
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetTbTSMAFromMnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* name, STableTSMAInfoRsp* out,
SCtgTaskReq* tReq, int32_t reqType) {
char* msg = NULL;
int32_t msgLen = 0;
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(name, tbFName);
ctgDebug("try to get tb index from mnode, tbFName:%s", tbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)]((void*)tbFName, &msg, 0, &msgLen, mallocFp);
if (code) {
ctgError("Build get index msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName);
CTG_ERR_RET(code);
}
if (pTask) {
void* pOut = taosMemoryCalloc(1, sizeof(STableTSMAInfoRsp));
if (NULL == pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, (char*)tbFName));
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, 0, pConn, tReq, reqType, msg, msgLen));
#else
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pTaskId, &pTask->taskId);
CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask->pJob, pTaskId, -1, NULL, NULL, 0, reqType, msg, msgLen));
#endif
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
rpcFreeCont(rpcRsp.pCont);
return TSDB_CODE_SUCCESS;
}
int32_t ctgGetStreamProgressFromVnode(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTbName,
SVgroupInfo* vgroupInfo, SStreamProgressRsp* out, SCtgTaskReq* tReq,
void* bInput) {
char* msg = NULL;
int32_t msgLen = 0;
int32_t reqType = TDMT_VND_GET_STREAM_PROGRESS;
char tbFName[TSDB_TABLE_FNAME_LEN];
tNameExtractFullName(pTbName, tbFName);
SCtgTask* pTask = tReq ? tReq->pTask : NULL;
void* (*mallocFp)(int64_t) = pTask ? (MallocType)taosMemoryMalloc : (MallocType)rpcMallocCont;
SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse];
ctgDebug("try to get stream progress from vnode, vgId:%d, ep num:%d, ep %s:%d, target:%s", vgroupInfo->vgId,
vgroupInfo->epSet.numOfEps, pEp->fqdn, pEp->port, tbFName);
int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](bInput, &msg, 0, &msgLen, mallocFp);
if (code) {
ctgError("Build get stream progress failed, code:%s, tbFName:%s", tstrerror(code), tbFName);
CTG_ERR_RET(code);
}
if (pTask) {
SStreamProgressRsp* pOut = taosMemoryCalloc(1, sizeof(SStreamProgressRsp));
if (!pOut) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
CTG_ERR_RET(ctgUpdateMsgCtx(CTG_GET_TASK_MSGCTX(pTask, tReq->msgIdx), reqType, pOut, (char*)tbFName));
SRequestConnInfo vConn = {.pTrans = pConn->pTrans,
.requestId = pConn->requestId,
.requestObjRefId = pConn->requestObjRefId,
.mgmtEps = vgroupInfo->epSet};
#if CTG_BATCH_FETCH
CTG_RET(ctgAddBatch(pCtg, vgroupInfo->vgId, &vConn, tReq, reqType, msg, msgLen));
#else
char dbFName[TSDB_DB_FNAME_LEN];
tNameGetFullDbName(pTbName, dbFName);
SArray* pTaskId = taosArrayInit(1, sizeof(int32_t));
if (NULL == pTaskId) {
CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
taosArrayPush(pTaskId, &pTask->taskId);
CTG_RET(
ctgAsyncSendMsg(pCtg, &vConn, pTask->pJob, pTaskId, -1, NULL, dbFName, vgroupInfo->vgId, reqType, msg, msgLen));
#endif
}
SRpcMsg rpcMsg = {
.msgType = reqType,
.pCont = msg,
.contLen = msgLen,
};
SRpcMsg rpcRsp = {0};
rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp);
CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName));
rpcFreeCont(rpcRsp.pCont);
return TSDB_CODE_SUCCESS;
}

32
source/libs/catalog/src/ctgRent.c
View File

@ -259,6 +259,24 @@ void ctgRemoveViewRent(SCatalog *pCtg, SCtgDBCache *dbCache) {
}
}
void ctgRemoveTSMARent(SCatalog *pCtg, SCtgDBCache *dbCache) {
if (!dbCache->tsmaCache) return;
void* pIter = taosHashIterate(dbCache->tsmaCache, NULL);
while (pIter) {
SCtgTSMACache* pCtgCache = pIter;
CTG_LOCK(CTG_READ, &pCtgCache->tsmaLock);
int32_t size = (pCtgCache && pCtgCache->pTsmas) ? pCtgCache->pTsmas->size : 0;
for (int32_t i = 0; i < size; ++i) {
STSMACache* pCache = taosArrayGetP(pCtgCache->pTsmas, i);
if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->tsmaRent, pCache->tsmaId, ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare)) {
ctgDebug("tsma removed from rent, viewId: %" PRIx64 " name: %s.%s.%s", pCache->tsmaId, pCache->dbFName, pCache->tb, pCache->name);
}
}
CTG_UNLOCK(CTG_READ, &pCtgCache->tsmaLock);
pIter = taosHashIterate(dbCache->tsmaCache, pIter);
}
}
int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid,
SCtgTbCache *pCache) {
@ -300,3 +318,17 @@ int32_t ctgUpdateRentViewVersion(SCatalog *pCtg, char *dbFName, char *viewName,
return TSDB_CODE_SUCCESS;
}
int32_t ctgUpdateRentTSMAVersion(SCatalog *pCtg, char *dbFName, const STSMACache *pCache) {
const STableTSMAInfo *pTsmaInfo = pCache;
STSMAVersion tsmaRent = {.dbId = pTsmaInfo->dbId, .tsmaId = pTsmaInfo->tsmaId, .version = pTsmaInfo->version};
tstrncpy(tsmaRent.name, pTsmaInfo->name, TSDB_TABLE_NAME_LEN);
tstrncpy(tsmaRent.dbFName, dbFName, TSDB_DB_FNAME_LEN);
tstrncpy(tsmaRent.tbName, pTsmaInfo->tb, TSDB_TABLE_NAME_LEN);
CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->tsmaRent, &tsmaRent, tsmaRent.tsmaId, sizeof(STSMAVersion),
ctgTSMAVersionSortCompare, ctgTSMAVersionSearchCompare));
ctgDebug("db %s, 0x%" PRIx64 " tsma %s, 0x%" PRIx64 "version %d updated to tsmaRent", dbFName, tsmaRent.dbId,
pTsmaInfo->name, pTsmaInfo->tsmaId, pTsmaInfo->version);
return TSDB_CODE_SUCCESS;
}

161
source/libs/catalog/src/ctgUtil.c
View File

@ -190,6 +190,12 @@ void ctgFreeSMetaData(SMetaData* pData) {
taosArrayDestroy(pData->pView);
pData->pView = NULL;
taosArrayDestroy(pData->pTableTsmas);
pData->pTableTsmas = NULL;
taosArrayDestroy(pData->pTsmas);
pData->pTsmas = NULL;
taosMemoryFreeClear(pData->pSvrVer);
}
@ -297,6 +303,31 @@ void ctgFreeTbCache(SCtgDBCache* dbCache) {
dbCache->tbCache = NULL;
}
void ctgFreeTSMACacheImpl(SCtgTSMACache* pCache, bool lock) {
if (lock) {
CTG_LOCK(CTG_WRITE, &pCache->tsmaLock);
}
if (pCache->pTsmas) {
taosArrayDestroyP(pCache->pTsmas, tFreeAndClearTableTSMAInfo);
pCache->pTsmas = NULL;
}
if (lock) {
CTG_UNLOCK(CTG_WRITE, &pCache->tsmaLock);
}
}
void ctgFreeTSMACache(SCtgDBCache* dbCache) {
if (!dbCache) return;
SCtgTSMACache* pCache = taosHashIterate(dbCache->tsmaCache, NULL);
while (pCache) {
ctgFreeTSMACacheImpl(pCache, false);
pCache = taosHashIterate(dbCache->tsmaCache, pCache);
}
taosHashCleanup(dbCache->tsmaCache);
dbCache->tsmaCache = NULL;
}
void ctgFreeVgInfoCache(SCtgDBCache* dbCache) { freeVgInfo(dbCache->vgCache.vgInfo); }
void ctgFreeCfgInfoCache(SCtgDBCache* dbCache) { freeDbCfgInfo(dbCache->cfgCache.cfgInfo); }
@ -310,6 +341,7 @@ void ctgFreeDbCache(SCtgDBCache* dbCache) {
ctgFreeStbMetaCache(dbCache);
ctgFreeTbCache(dbCache);
ctgFreeViewCache(dbCache);
ctgFreeTSMACache(dbCache);
}
void ctgFreeInstDbCache(SHashObj* pDbCache) {
@ -353,6 +385,7 @@ void ctgFreeHandleImpl(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeMetaRent(&pCtg->tsmaRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
@ -382,6 +415,7 @@ void ctgFreeHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeMetaRent(&pCtg->tsmaRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
@ -410,7 +444,8 @@ void ctgClearHandleMeta(SCatalog* pCtg, int64_t *pClearedSize, int64_t *pCleardN
}
taosHashRemove(dbCache->tbCache, key, len);
cacheSize = len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex);
cacheSize =
len + sizeof(SCtgTbCache) + ctgGetTbMetaCacheSize(pCache->pMeta) + ctgGetTbIndexCacheSize(pCache->pIndex);
atomic_sub_fetch_64(&dbCache->dbCacheSize, cacheSize);
*pClearedSize += cacheSize;
(*pCleardNum)++;
@ -468,6 +503,7 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgFreeMetaRent(&pCtg->dbRent);
ctgFreeMetaRent(&pCtg->stbRent);
ctgFreeMetaRent(&pCtg->viewRent);
ctgFreeMetaRent(&pCtg->tsmaRent);
ctgFreeInstDbCache(pCtg->dbCache);
ctgFreeInstUserCache(pCtg->userCache);
@ -475,6 +511,7 @@ void ctgClearHandle(SCatalog* pCtg) {
ctgMetaRentInit(&pCtg->dbRent, gCtgMgmt.cfg.dbRentSec, CTG_RENT_DB, sizeof(SDbCacheInfo));
ctgMetaRentInit(&pCtg->stbRent, gCtgMgmt.cfg.stbRentSec, CTG_RENT_STABLE, sizeof(SSTableVersion));
ctgMetaRentInit(&pCtg->viewRent, gCtgMgmt.cfg.viewRentSec, CTG_RENT_VIEW, sizeof(SViewVersion));
ctgMetaRentInit(&pCtg->tsmaRent, gCtgMgmt.cfg.tsmaRentSec, CTG_RENT_TSMA, sizeof(STSMAVersion));
pCtg->dbCache = taosHashInit(gCtgMgmt.cfg.maxDBCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false,
HASH_ENTRY_LOCK);
@ -592,6 +629,20 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) {
}
break;
}
case TDMT_MND_GET_TSMA:
case TDMT_MND_GET_TABLE_TSMA: {
if (pCtx->out) {
tFreeTableTSMAInfoRsp(pCtx->out);
taosMemoryFreeClear(pCtx->out);
}
break;
}
case TDMT_VND_GET_STREAM_PROGRESS: {
if (pCtx->out) {
taosMemoryFreeClear(pCtx->out);
}
break;
}
default:
qError("invalid reqType %d", pCtx->reqType);
break;
@ -663,6 +714,20 @@ void ctgFreeViewMetaRes(void* res) {
}
}
void ctgFreeTbTSMARes(void* res) {
if (!res) {
return;
}
SMetaRes* pRes = res;
if (pRes->pRes) {
STableTSMAInfoRsp * pTsmaRsp = pRes->pRes;
tFreeTableTSMAInfoRsp(pTsmaRsp);
taosMemoryFree(pTsmaRsp);
pRes->pRes = NULL;
}
}
void ctgFreeJsonTagVal(void* val) {
if (NULL == val) {
return;
@ -760,6 +825,16 @@ void ctgFreeTaskRes(CTG_TASK_TYPE type, void** pRes) {
*pRes = NULL; // no need to free it
break;
}
case CTG_TASK_GET_TSMA:
case CTG_TASK_GET_TB_TSMA: {
SArray* pArr = (SArray*)*pRes;
int32_t num = taosArrayGetSize(pArr);
for (int32_t i = 0; i < num; ++i) {
ctgFreeTbTSMARes(taosArrayGet(pArr, i));
}
*pRes = NULL;
break;
}
default:
qError("invalid task type %d", type);
break;
@ -923,6 +998,15 @@ void ctgFreeTaskCtx(SCtgTask* pTask) {
taosMemoryFreeClear(pTask->taskCtx);
break;
}
case CTG_TASK_GET_TSMA:
case CTG_TASK_GET_TB_TSMA: {
SCtgTbTSMACtx* pTsmaCtx = pTask->taskCtx;
taosArrayDestroyEx(pTsmaCtx->pResList, ctgFreeTbTSMARes);
taosArrayDestroy(pTsmaCtx->pFetches);
taosArrayDestroyEx(pTask->msgCtxs, (FDelete)ctgFreeMsgCtx);
taosMemoryFreeClear(pTask->taskCtx);
break;
}
default:
qError("invalid task type %d", pTask->type);
break;
@ -1328,6 +1412,15 @@ int32_t ctgViewVersionSearchCompare(const void* key1, const void* key2) {
}
}
int32_t ctgTSMAVersionSearchCompare(const void* key1, const void* key2) {
if (*(uint64_t*)key1 < ((STSMAVersion*)key2)->tsmaId) {
return -1;
} else if (*(uint64_t*)key1 > ((STSMAVersion*)key2)->tsmaId) {
return 1;
} else {
return 0;
}
}
int32_t ctgStbVersionSortCompare(const void* key1, const void* key2) {
if (((SSTableVersion*)key1)->suid < ((SSTableVersion*)key2)->suid) {
@ -1359,6 +1452,15 @@ int32_t ctgViewVersionSortCompare(const void* key1, const void* key2) {
}
}
int32_t ctgTSMAVersionSortCompare(const void* key1, const void* key2) {
if (((STSMAVersion*)key1)->tsmaId < ((STSMAVersion*)key2)->tsmaId) {
return -1;
} else if (((STSMAVersion*)key1)->tsmaId > ((STSMAVersion*)key2)->tsmaId) {
return 1;
} else {
return 0;
}
}
int32_t ctgMakeVgArray(SDBVgInfo* dbInfo) {
if (NULL == dbInfo) {
@ -1657,6 +1759,10 @@ static void ctgFreeViewMeta(void* p) {
taosMemoryFree(pMeta);
}
void ctgFreeTbTSMAInfo(void* p) {
tFreeTableTSMAInfoRsp(((SMetaRes*)p)->pRes);
taosMemoryFree(((SMetaRes*)p)->pRes);
}
int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) {
int32_t code = 0;
@ -1990,6 +2096,8 @@ void ctgDestroySMetaData(SMetaData* pData) {
taosArrayDestroyEx(pData->pTableCfg, ctgFreeTableCfg);
taosArrayDestroyEx(pData->pDnodeList, ctgFreeDnodeList);
taosArrayDestroyEx(pData->pView, ctgFreeViewMeta);
taosArrayDestroyEx(pData->pTableTsmas, ctgFreeTbTSMAInfo);
taosArrayDestroyEx(pData->pTsmas, ctgFreeTbTSMAInfo);
taosMemoryFreeClear(pData->pSvrVer);
}
@ -2287,5 +2395,56 @@ int32_t dupViewMetaFromRsp(SViewMetaRsp* pRsp, SViewMeta* pViewMeta) {
return TSDB_CODE_SUCCESS;
}
uint64_t ctgGetTbTSMACacheSize(STableTSMAInfo* pTsmaInfo) {
if (!pTsmaInfo) return 0;
uint64_t size = sizeof(STableTSMAInfo);
if (pTsmaInfo->pFuncs) size += sizeof(STableTSMAFuncInfo) * pTsmaInfo->pFuncs->size;
if (pTsmaInfo->pTags) size += sizeof(SSchema) * pTsmaInfo->pTags->size;
if (pTsmaInfo->pUsedCols) size += sizeof(SSchema) * pTsmaInfo->pUsedCols->size;
return size;
}
bool hasOutOfDateTSMACache(SArray* pTsmas) {
if (!pTsmas || pTsmas->size == 0) {
return false;
}
for (int32_t i = 0; i < pTsmas->size; ++i) {
STSMACache* pTsmaInfo = taosArrayGetP(pTsmas, i);
if (isCtgTSMACacheOutOfDate(pTsmaInfo)) return true;
}
return false;
}
bool isCtgTSMACacheOutOfDate(STSMACache* pTsmaCache) {
int64_t now = taosGetTimestampMs();
bool ret = !pTsmaCache->fillHistoryFinished ||
(tsMaxTsmaCalcDelay * 1000 - pTsmaCache->delayDuration) < (now - pTsmaCache->reqTs);
if (ret) {
qDebug("tsma %s.%s in cache has been out of date, history finished: %d, remain valid after: %" PRId64
" passed: %" PRId64,
pTsmaCache->dbFName, pTsmaCache->name, pTsmaCache->fillHistoryFinished,
tsMaxTsmaCalcDelay * 1000 - pTsmaCache->delayDuration, now - pTsmaCache->reqTs);
}
return ret;
}
int32_t ctgAddTSMAFetch(SArray** pFetchs, int32_t dbIdx, int32_t tbIdx, int32_t* fetchIdx, int32_t resIdx, int32_t flag,
CTG_TSMA_FETCH_TYPE fetchType, const SName* sourceTbName) {
if (NULL == (*pFetchs)) {
*pFetchs = taosArrayInit(CTG_DEFAULT_FETCH_NUM, sizeof(SCtgTSMAFetch));
}
SCtgTSMAFetch fetch = {0};
fetch.dbIdx = dbIdx;
fetch.tbIdx = tbIdx;
fetch.fetchIdx = (*fetchIdx)++;
fetch.resIdx = resIdx;
fetch.flag = flag;
fetch.fetchType = fetchType;
if (sourceTbName) fetch.tsmaSourceTbName = *sourceTbName;
taosArrayPush(*pFetchs, &fetch);
return TSDB_CODE_SUCCESS;
}

15
source/libs/command/inc/commandInt.h
View File

@ -53,9 +53,12 @@ extern "C" {
#define EXPLAIN_FILTER_FORMAT "Filter: "
#define EXPLAIN_MERGEBLOCKS_FORMAT "Merge ResBlocks: %s"
#define EXPLAIN_FILL_VALUE_FORMAT "Fill Values: "
#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Cond: "
#define EXPLAIN_PRIM_CONDITIONS_FORMAT "Join Prim Cond: "
#define EXPLAIN_ON_CONDITIONS_FORMAT "Join Full Cond: "
#define EXPLAIN_COL_ON_CONDITIONS_FORMAT "Join Col Cond: "
#define EXPLAIN_TIMERANGE_FORMAT "Time Range: [%" PRId64 ", %" PRId64 "]"
#define EXPLAIN_OUTPUT_FORMAT "Output: "
#define EXPLAIN_JOIN_PARAM_FORMAT "Join Param: "
#define EXPLAIN_TIME_WINDOWS_FORMAT "Time Window: interval=%" PRId64 "%c offset=%" PRId64 "%c sliding=%" PRId64 "%c"
#define EXPLAIN_WINDOW_FORMAT "Window: gap=%" PRId64
#define EXPLAIN_RATIO_TIME_FORMAT "Ratio: %f"
@ -71,8 +74,11 @@ extern "C" {
#define EXPLAIN_DYN_QRY_CTRL_FORMAT "Dynamic Query Control for %s"
#define EXPLAIN_COUNT_FORMAT "Count"
#define EXPLAIN_COUNT_INFO_FORMAT "Window Count Info"
#define EXPLAIN_JOIN_EQ_LEFT_FORMAT "Left Equal Cond: "
#define EXPLAIN_JOIN_EQ_RIGHT_FORMAT "Right Equal Cond: "
#define EXPLAIN_COUNT_NUM_FORMAT "Window Count=%" PRId64
#define EXPLAIN_COUNT_SLIDING_FORMAT "Window Sliding=%" PRId64
#define EXPLAIN_TABLE_TIMERANGE_FORMAT "%s Table Time Range: [%" PRId64 ", %" PRId64 "]"
#define EXPLAIN_PLANNING_TIME_FORMAT "Planning Time: %.3f ms"
#define EXPLAIN_EXEC_TIME_FORMAT "Execution Time: %.3f ms"
@ -93,7 +99,6 @@ extern "C" {
#define EXPLAIN_SCAN_MODE_FORMAT "mode=%s"
#define EXPLAIN_SCAN_DATA_LOAD_FORMAT "data_load=%s"
#define EXPLAIN_GROUPS_FORMAT "groups=%d"
#define EXPLAIN_WIDTH_FORMAT "width=%d"
#define EXPLAIN_INTERVAL_VALUE_FORMAT "interval=%" PRId64 "%c"
#define EXPLAIN_FUNCTIONS_FORMAT "functions=%d"
#define EXPLAIN_EXECINFO_FORMAT "cost=%.3f..%.3f rows=%" PRIu64
@ -112,6 +117,12 @@ extern "C" {
#define EXPLAIN_SRC_SCAN_FORMAT "src_scan=%d,%d"
#define EXPLAIN_PLAN_BLOCKING "blocking=%d"
#define EXPLAIN_MERGE_MODE_FORMAT "mode=%s"
#define EXPLAIN_ASOF_OP_FORMAT "asof_op='%s'"
#define EXPLAIN_WIN_OFFSET_FORMAT "window_offset=(%s, %s)"
#define EXPLAIN_JLIMIT_FORMAT "jlimit=%" PRId64
#define EXPLAIN_SEQ_WIN_GRP_FORMAT "seq_win_grp=%d"
#define EXPLAIN_GRP_JOIN_FORMAT "group_join=%d"
#define EXPLAIN_JOIN_ALGO "algo=%s"
#define COMMAND_RESET_LOG "resetLog"
#define COMMAND_SCHEDULE_POLICY "schedulePolicy"

1
source/libs/command/src/command.c
View File

@ -757,6 +757,7 @@ static int32_t setCreateViewResultIntoDataBlock(SSDataBlock* pBlock, SShowCreate
}
SViewMeta* pMeta = pStmt->pViewMeta;
ASSERT(pMeta);
snprintf(varDataVal(buf2), SHOW_CREATE_VIEW_RESULT_FIELD2_LEN - VARSTR_HEADER_SIZE, "CREATE VIEW `%s`.`%s` AS %s", pStmt->dbName, pStmt->viewName, pMeta->querySql);
int32_t len = strlen(varDataVal(buf2));
varDataLen(buf2) = (len > 65535) ? 65535 : len;

150
source/libs/command/src/explain.c
View File

@ -22,6 +22,14 @@
#include "systable.h"
#include "functionMgt.h"
char *gJoinTypeStr[JOIN_TYPE_MAX_VALUE][JOIN_STYPE_MAX_VALUE] = {
/* NONE OUTER SEMI ANTI ASOF WINDOW */
/*INNER*/ {"Inner Join", NULL, NULL, NULL, NULL, NULL},
/*LEFT*/ {"Left Join", "Left Join", "Left Semi Join", "Left Anti Join", "Left ASOF Join", "Left Window Join"},
/*RIGHT*/ {"Right Join", "Right Join", "Right Semi Join", "Right Anti Join", "Right ASOF Join", "Right Window Join"},
/*FULL*/ {"Full Join", "Full Join", NULL, NULL, NULL, NULL},
};
int32_t qExplainGenerateResNode(SPhysiNode *pNode, SExplainGroup *group, SExplainResNode **pRes);
int32_t qExplainAppendGroupResRows(void *pCtx, int32_t groupId, int32_t level, bool singleChannel);
@ -36,6 +44,33 @@ char *qExplainGetDynQryCtrlType(EDynQueryType type) {
return "unknown task";
}
char* qExplainGetAsofOpStr(int32_t opType) {
switch (opType) {
case OP_TYPE_GREATER_THAN:
return ">";
case OP_TYPE_GREATER_EQUAL:
return ">=";
case OP_TYPE_LOWER_THAN:
return "<";
case OP_TYPE_LOWER_EQUAL:
return "<=";
case OP_TYPE_EQUAL:
return "=";
default:
return "UNKNOWN";
}
}
char* qExplainGetTimerangeTargetStr(int32_t target) {
static char* targetName[] = {"", "Left", "Right", "Left/Right"};
if (target <= 0 || target > 3) {
return "Unknown";
}
return targetName[target];
}
void qExplainFreeResNode(SExplainResNode *resNode) {
if (NULL == resNode) {
return;
@ -375,6 +410,14 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
if (pTagScanNode->scan.node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pTagScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
@ -593,7 +636,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode *pJoinNode = (SSortMergeJoinPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType));
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, gJoinTypeStr[pJoinNode->joinType][pJoinNode->subType]);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@ -605,6 +648,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pJoinNode->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_JOIN_ALGO, "Merge");
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -620,6 +664,36 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (IS_ASOF_JOIN(pJoinNode->subType) || IS_WINDOW_JOIN(pJoinNode->subType)) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_JOIN_PARAM_FORMAT);
if (IS_ASOF_JOIN(pJoinNode->subType)) {
EXPLAIN_ROW_APPEND(EXPLAIN_ASOF_OP_FORMAT, qExplainGetAsofOpStr(pJoinNode->asofOpType));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (NULL != pJoinNode->pWindowOffset) {
SWindowOffsetNode* pWinOffset = (SWindowOffsetNode*)pJoinNode->pWindowOffset;
SValueNode* pStart = (SValueNode*)pWinOffset->pStartOffset;
SValueNode* pEnd = (SValueNode*)pWinOffset->pEndOffset;
EXPLAIN_ROW_APPEND(EXPLAIN_WIN_OFFSET_FORMAT, pStart->literal, pEnd->literal);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (NULL != pJoinNode->pJLimit) {
SLimitNode* pJLimit = (SLimitNode*)pJoinNode->pJLimit;
EXPLAIN_ROW_APPEND(EXPLAIN_JLIMIT_FORMAT, pJLimit->limit);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
if (IS_WINDOW_JOIN(pJoinNode->subType)) {
EXPLAIN_ROW_APPEND(EXPLAIN_SEQ_WIN_GRP_FORMAT, pJoinNode->seqWinGroup);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
}
EXPLAIN_ROW_APPEND(EXPLAIN_GRP_JOIN_FORMAT, pJoinNode->grpJoin);
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
@ -628,17 +702,49 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
if (pJoinNode->pOtherOnCond) {
EXPLAIN_ROW_APPEND(" AND ");
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pOtherOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
if (NULL != pJoinNode->pPrimKeyCond) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_PRIM_CONDITIONS_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pPrimKeyCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pEqLeft && pJoinNode->pEqLeft->length > 0) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_JOIN_EQ_LEFT_FORMAT);
SNode* pCol = NULL;
FOREACH(pCol, pJoinNode->pEqLeft) {
EXPLAIN_ROW_APPEND("`%s`.`%s` ", ((SColumnNode*)pCol)->tableAlias, ((SColumnNode*)pCol)->colName);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pEqRight && pJoinNode->pEqRight->length > 0) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_JOIN_EQ_RIGHT_FORMAT);
SNode* pCol = NULL;
FOREACH(pCol, pJoinNode->pEqRight) {
EXPLAIN_ROW_APPEND("`%s`.`%s` ", ((SColumnNode*)pCol)->tableAlias, ((SColumnNode*)pCol)->colName);
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pFullOnCond) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
if (NULL != pJoinNode->pColOnCond) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_COL_ON_CONDITIONS_FORMAT);
QRY_ERR_RET(
nodesNodeToSQL(pJoinNode->pColOnCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
@ -1575,7 +1681,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN:{
SHashJoinPhysiNode *pJoinNode = (SHashJoinPhysiNode *)pNode;
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, EXPLAIN_JOIN_STRING(pJoinNode->joinType));
EXPLAIN_ROW_NEW(level, EXPLAIN_JOIN_FORMAT, gJoinTypeStr[pJoinNode->joinType][pJoinNode->subType]);
EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT);
if (pResNode->pExecInfo) {
QRY_ERR_RET(qExplainBufAppendExecInfo(pResNode->pExecInfo, tbuf, &tlen));
@ -1587,6 +1693,7 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_INPUT_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pJoinNode->node.inputTsOrder));
EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT);
EXPLAIN_ROW_APPEND(EXPLAIN_JOIN_ALGO, "Hash");
EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level));
@ -1602,19 +1709,11 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->node.pConditions || pJoinNode->pFilterConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT);
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->node.pConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
if (pJoinNode->pFilterConditions) {
if (pJoinNode->node.pConditions) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFilterConditions, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
@ -1642,8 +1741,21 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i
nodesNodeToSQL(pJoinNode->pTagEqCond, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
conditionsGot = true;
}
if (pJoinNode->pFullOnCond) {
if (conditionsGot) {
EXPLAIN_ROW_APPEND(" AND ");
}
QRY_ERR_RET(nodesNodeToSQL(pJoinNode->pFullOnCond, tbuf + VARSTR_HEADER_SIZE,
TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen));
}
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
if (pJoinNode->timeRangeTarget) {
EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TABLE_TIMERANGE_FORMAT, qExplainGetTimerangeTargetStr(pJoinNode->timeRangeTarget), pJoinNode->timeRange.skey, pJoinNode->timeRange.ekey);
EXPLAIN_ROW_END();
QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1));
}
}
break;
}

4
source/libs/executor/inc/executorInt.h
View File

@ -836,6 +836,7 @@ int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* de
extern void doDestroyExchangeOperatorInfo(void* param);
int32_t doFilterImpl(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo, SColumnInfoData** pResCol);
int32_t doFilter(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SColMatchInfo* pColMatchInfo);
int32_t addTagPseudoColumnData(SReadHandle* pHandle, const SExprInfo* pExpr, int32_t numOfExpr, SSDataBlock* pBlock,
int32_t rows, SExecTaskInfo* pTask, STableMetaCacheInfo* pCache);
@ -974,6 +975,9 @@ void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBlock, SArr
int32_t getNextQualifiedWindow(SInterval* pInterval, STimeWindow* pNext, SDataBlockInfo* pDataBlockInfo,
TSKEY* primaryKeys, int32_t prevPosition, int32_t order);
void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status);
#ifdef __cplusplus
}

1
source/libs/executor/inc/groupcache.h
View File

@ -65,6 +65,7 @@ typedef struct SGcDownstreamCtx {
SHashObj* pSessions;
SHashObj* pWaitSessions;
SGcFileCacheCtx fileCtx;
bool fetchDone;
} SGcDownstreamCtx;
typedef struct SGcVgroupCtx {

100
source/libs/executor/inc/hashjoin.h
View File

@ -20,6 +20,13 @@ extern "C" {
#endif
#define HASH_JOIN_DEFAULT_PAGE_SIZE 10485760
#define HJOIN_DEFAULT_BLK_ROWS_NUM 4096
#define HJOIN_BLK_SIZE_LIMIT 10485760
#define HJOIN_ROW_BITMAP_SIZE (2 * 1048576)
#define HJOIN_BLK_THRESHOLD_RATIO 0.9
typedef int32_t (*hJoinImplFp)(SOperatorInfo*);
#pragma pack(push, 1)
typedef struct SBufRowInfo {
@ -29,18 +36,26 @@ typedef struct SBufRowInfo {
} SBufRowInfo;
#pragma pack(pop)
typedef enum EHJoinPhase {
E_JOIN_PHASE_PRE = 1,
E_JOIN_PHASE_CUR,
E_JOIN_PHASE_POST
} EHJoinPhase;
typedef struct SHJoinCtx {
bool rowRemains;
bool midRemains;
int64_t limit;
SBufRowInfo* pBuildRow;
SSDataBlock* pProbeData;
int32_t probeIdx;
EHJoinPhase probePhase;
int32_t probePreIdx;
int32_t probeStartIdx;
int32_t probeEndIdx;
int32_t probePostIdx;
bool readMatch;
} SHJoinCtx;
typedef struct SRowLocation {
SSDataBlock* pDataBlock;
int32_t pos;
} SRowLocation;
typedef struct SHJoinColInfo {
int32_t srcSlot;
int32_t dstSlot;
@ -50,6 +65,7 @@ typedef struct SHJoinColInfo {
int32_t bytes;
char* data;
char* bitMap;
SColumnInfoData* colData;
} SHJoinColInfo;
typedef struct SBufPageInfo {
@ -63,11 +79,31 @@ typedef struct SGroupData {
SBufRowInfo* rows;
} SGroupData;
typedef struct SHJoinTableInfo {
typedef struct SHJoinColMap {
int32_t srcSlot;
int32_t dstSlot;
bool vardata;
int32_t bytes;
} SHJoinColMap;
// for now timetruncate only
typedef struct SHJoinPrimExprCtx {
int64_t truncateUnit;
int64_t timezoneUnit;
int32_t targetSlotId;
} SHJoinPrimExprCtx;
typedef struct SHJoinTableCtx {
int32_t downStreamIdx;
SOperatorInfo* downStream;
int32_t blkId;
SQueryStat inputStat;
bool hasTimeRange;
SHJoinColMap* primCol;
SNode* primExpr;
SHJoinPrimExprCtx primCtx;
int32_t keyNum;
SHJoinColInfo* keyCols;
@ -81,7 +117,7 @@ typedef struct SHJoinTableInfo {
int32_t valBufSize;
SArray* valVarCols;
bool valColExist;
} SHJoinTableInfo;
} SHJoinTableCtx;
typedef struct SHJoinExecInfo {
int64_t buildBlkNum;
@ -94,21 +130,57 @@ typedef struct SHJoinExecInfo {
typedef struct SHJoinOperatorInfo {
int32_t joinType;
SHJoinTableInfo tbs[2];
SHJoinTableInfo* pBuild;
SHJoinTableInfo* pProbe;
SSDataBlock* pRes;
EJoinType joinType;
EJoinSubType subType;
SHJoinTableCtx tbs[2];
SHJoinTableCtx* pBuild;
SHJoinTableCtx* pProbe;
SFilterInfo* pPreFilter;
SFilterInfo* pFinFilter;
SSDataBlock* finBlk;
SSDataBlock* midBlk;
STimeWindow tblTimeRange;
int32_t pResColNum;
int8_t* pResColMap;
SArray* pRowBufs;
SNode* pCond;
SSHashObj* pKeyHash;
bool keyHashBuilt;
SHJoinCtx ctx;
SHJoinExecInfo execInfo;
int32_t blkThreshold;
hJoinImplFp joinFp;
} SHJoinOperatorInfo;
#define HJ_ERR_RET(c) \
do { \
int32_t _code = (c); \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define HJ_ERR_JRET(c) \
do { \
code = (c); \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
int32_t hInnerJoinDo(struct SOperatorInfo* pOperator);
int32_t hLeftJoinDo(struct SOperatorInfo* pOperator);
void hJoinSetDone(struct SOperatorInfo* pOperator);
void hJoinAppendResToBlock(struct SOperatorInfo* pOperator, SSDataBlock* pRes, bool* allFetched);
bool hJoinCopyKeyColsDataToBuf(SHJoinTableCtx* pTable, int32_t rowIdx, size_t *pBufLen);
int32_t hJoinCopyMergeMidBlk(SHJoinCtx* pCtx, SSDataBlock** ppMid, SSDataBlock** ppFin);
int32_t hJoinHandleMidRemains(SHJoinOperatorInfo* pJoin, SHJoinCtx* pCtx);
bool hJoinBlkReachThreshold(SHJoinOperatorInfo* pInfo, int64_t blkRows);
int32_t hJoinCopyNMatchRowsToBlock(SHJoinOperatorInfo* pJoin, SSDataBlock* pRes, int32_t startIdx, int32_t rows);
#ifdef __cplusplus
}
#endif

482
source/libs/executor/inc/mergejoin.h Executable file
View File

@ -0,0 +1,482 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef TDENGINE_MERGEJOIN_H
#define TDENGINE_MERGEJOIN_H
#ifdef __cplusplus
extern "C" {
#endif
#if 0
#define MJOIN_DEFAULT_BLK_ROWS_NUM 3 //4096
#define MJOIN_HJOIN_CART_THRESHOLD 10
#define MJOIN_BLK_SIZE_LIMIT 0 //10485760
#define MJOIN_ROW_BITMAP_SIZE (2 * 1048576)
#else
#define MJOIN_DEFAULT_BLK_ROWS_NUM 4096
#define MJOIN_HJOIN_CART_THRESHOLD 16
#define MJOIN_BLK_SIZE_LIMIT 10485760
#define MJOIN_ROW_BITMAP_SIZE (2 * 1048576)
#endif
#define MJOIN_SEMI_ANTI_BLK_ROWS_NUM 64
#define MJOIN_BLK_THRESHOLD_RATIO 0.9
struct SMJoinOperatorInfo;
typedef enum EJoinTableType {
E_JOIN_TB_BUILD = 1,
E_JOIN_TB_PROBE
} EJoinTableType;
#define MJOIN_TBTYPE(_type) (E_JOIN_TB_BUILD == (_type) ? "BUILD" : "PROBE")
#define IS_FULL_OUTER_JOIN(_jtype, _stype) ((_jtype) == JOIN_TYPE_FULL && (_stype) == JOIN_STYPE_OUTER)
typedef struct SMJoinRowPos {
SSDataBlock* pBlk;
int32_t pos;
} SMJoinRowPos;
typedef struct SMJoinColMap {
int32_t srcSlot;
int32_t dstSlot;
bool vardata;
int32_t bytes;
} SMJoinColMap;
typedef struct SMJoinColInfo {
int32_t srcSlot;
int32_t dstSlot;
bool jsonData;
bool vardata;
int32_t* offset;
int32_t bytes;
char* data;
char* bitMap;
SColumnInfoData* colData;
} SMJoinColInfo;
typedef struct SMJoinHashGrpRows {
int32_t rowBitmapOffset;
int32_t rowMatchNum;
bool allRowsMatch;
bool allRowsNMatch;
SArray* pRows;
} SMJoinHashGrpRows;
typedef struct SMJoinNMatchCtx {
void* pGrp;
int32_t iter;
int32_t bitIdx;
int32_t rowIdx;
int32_t grpIdx;
} SMJoinNMatchCtx;
// for now timetruncate only
typedef struct SMJoinPrimExprCtx {
int64_t truncateUnit;
int64_t timezoneUnit;
int32_t targetSlotId;
} SMJoinPrimExprCtx;
typedef struct SMJoinTableCtx {
EJoinTableType type;
int32_t downStreamIdx;
SOperatorInfo* downStream;
bool dsInitDone;
bool dsFetchDone;
SNode* primExpr;
SMJoinPrimExprCtx primCtx;
int32_t blkId;
SQueryStat inputStat;
uint64_t lastInGid;
SSDataBlock* remainInBlk;
SMJoinColMap* primCol;
char* primData;
int32_t finNum;
SMJoinColMap* finCols;
int32_t keyNum;
int32_t keyNullSize;
SMJoinColInfo* keyCols;
char* keyBuf;
char* keyData;
bool newBlk;
SSDataBlock* blk;
int32_t blkRowIdx;
// merge join
int64_t grpTotalRows;
int32_t grpIdx;
bool noKeepEqGrpRows;
bool multiEqGrpRows;
int64_t eqRowLimit;
int64_t eqRowNum;
SArray* eqGrps;
SArray* createdBlks;
// hash join
int32_t grpArrayIdx;
SArray* pGrpArrays;
bool multiRowsGrp;
int32_t grpRowIdx;
SArray* pHashCurGrp;
SMJoinHashGrpRows* pHashGrpRows;
SSHashObj* pGrpHash;
int64_t rowBitmapSize;
int64_t rowBitmapOffset;
char* pRowBitmap;
SMJoinNMatchCtx nMatchCtx;
} SMJoinTableCtx;
typedef struct SMJoinMatchInfo {
int32_t rowBitmapOffset;
int32_t rowMatchNum;
bool allRowsNMatch;
bool allRowsMatch;
} SMJoinMatchInfo;
typedef struct SMJoinGrpRows {
SSDataBlock* blk;
int32_t beginIdx;
int32_t endIdx;
int32_t readIdx;
int32_t rowBitmapOffset;
int32_t rowMatchNum;
bool allRowsNMatch;
bool allRowsMatch;
bool readMatch;
bool clonedBlk;
} SMJoinGrpRows;
#define MJOIN_COMMON_CTX \
struct SMJoinOperatorInfo* pJoin; \
bool ascTs; \
bool grpRemains; \
SSDataBlock* finBlk; \
bool lastEqGrp; \
bool lastProbeGrp; \
bool seqWinGrp; \
bool groupJoin; \
int32_t blkThreshold; \
int64_t limit; \
int64_t jLimit
typedef struct SMJoinCommonCtx {
MJOIN_COMMON_CTX;
} SMJoinCommonCtx;
typedef int32_t (*joinCartFp)(void*);
typedef struct SMJoinMergeCtx {
// KEEP IT FIRST
struct SMJoinOperatorInfo* pJoin;
bool ascTs;
bool grpRemains;
SSDataBlock* finBlk;
bool lastEqGrp;
bool lastProbeGrp;
bool seqWinGrp;
bool groupJoin;
int32_t blkThreshold;
int64_t limit;
int64_t jLimit;
// KEEP IT FIRST
bool hashCan;
bool midRemains;
bool nmatchRemains;
SSDataBlock* midBlk;
int64_t lastEqTs;
SMJoinGrpRows probeNEqGrp;
SMJoinGrpRows buildNEqGrp;
bool hashJoin;
joinCartFp hashCartFp;
joinCartFp mergeCartFp;
} SMJoinMergeCtx;
typedef enum {
E_CACHE_NONE = 0,
E_CACHE_OUTBLK,
E_CACHE_INBLK
} SMJoinCacheMode;
typedef struct SMJoinWinCache {
int32_t pageLimit;
int32_t outRowIdx;
int32_t colNum;
int32_t rowNum;
int8_t grpIdx;
SArray* grps;
SArray* grpsQueue;
SSDataBlock* outBlk;
} SMJoinWinCache;
typedef int32_t (*joinMoveWin)(void*);
typedef struct SMJoinWindowCtx {
// KEEP IT FIRST
struct SMJoinOperatorInfo* pJoin;
bool ascTs;
bool grpRemains;
SSDataBlock* finBlk;
bool lastEqGrp;
bool lastProbeGrp;
bool seqWinGrp;
bool groupJoin;
int32_t blkThreshold;
int64_t limit;
int64_t jLimit;
// KEEP IT FIRST
int32_t asofOpType;
int64_t winBeginOffset;
int64_t winEndOffset;
bool lowerRowsAcq;
bool eqRowsAcq;
bool greaterRowsAcq;
bool forwardRowsAcq;
int64_t winBeginTs;
int64_t winEndTs;
joinMoveWin moveWinBeginFp;
joinMoveWin moveWinEndFp;
bool eqPostDone;
int64_t lastTs;
SMJoinGrpRows probeGrp;
SMJoinGrpRows buildGrp;
SMJoinWinCache cache;
} SMJoinWindowCtx;
typedef struct SMJoinFlowFlags {
bool mergeJoin;
bool windowJoin;
bool preFilter;
bool retrieveAfterBuildDone;
} SMJoinFlowFlags;
typedef struct SMJoinCtx {
SMJoinFlowFlags* pFlags;
bool mergeCtxInUse;
union {
SMJoinMergeCtx mergeCtx;
SMJoinWindowCtx windowCtx;
};
} SMJoinCtx;
typedef struct SMJoinExecInfo {
int64_t buildBlkNum;
int64_t buildBlkRows;
int64_t probeBlkNum;
int64_t probeBlkRows;
int64_t resRows;
int64_t expectRows;
} SMJoinExecInfo;
typedef SSDataBlock* (*joinImplFp)(SOperatorInfo*);
typedef SSDataBlock* (*joinRetrieveFp)(struct SMJoinOperatorInfo*, SMJoinTableCtx*);
typedef void (*joinResetFp)(struct SMJoinOperatorInfo*);
typedef struct SMJoinOperatorInfo {
SOperatorInfo* pOperator;
int32_t joinType;
int32_t subType;
int32_t inputTsOrder;
int32_t errCode;
int64_t outGrpId;
uint64_t outBlkId;
SMJoinTableCtx tbs[2];
SMJoinTableCtx* build;
SMJoinTableCtx* probe;
SFilterInfo* pFPreFilter;
SFilterInfo* pPreFilter;
SFilterInfo* pFinFilter;
joinImplFp joinFp;
joinRetrieveFp retrieveFp;
joinResetFp grpResetFp;
SMJoinCtx ctx;
SMJoinExecInfo execInfo;
} SMJoinOperatorInfo;
#define MJOIN_DS_REQ_INIT(_pOp) ((_pOp)->pOperatorGetParam && ((SSortMergeJoinOperatorParam*)(_pOp)->pOperatorGetParam->value)->initDownstream)
#define MJOIN_DS_NEED_INIT(_pOp, _tbctx) (MJOIN_DS_REQ_INIT(_pOp) && (!(_tbctx)->dsInitDone))
#define MJOIN_TB_LOW_BLK(_tbctx) ((_tbctx)->blkNum <= 0 || ((_tbctx)->blkNum == 1 && (_tbctx)->pHeadBlk->cloned))
#define REACH_HJOIN_THRESHOLD(_prb, _bld) ((_bld)->grpTotalRows >= MJOIN_HJOIN_CART_THRESHOLD)
#define SET_SAME_TS_GRP_HJOIN(_pair, _octx) ((_pair)->hashJoin = (_octx)->hashCan && REACH_HJOIN_THRESHOLD(_pair))
#define PROBE_TS_NMATCH(_asc, _pts, _bts) (((_asc) && (_pts) < (_bts)) || (!(_asc) && (_pts) > (_bts)))
#define PROBE_TS_NREACH(_asc, _pts, _bts) (((_asc) && (_pts) > (_bts)) || (!(_asc) && (_pts) < (_bts)))
#define MJOIN_BUILD_BLK_OOR(_asc, _pts, _pidx, _bts, _bnum) (((_asc) && (*((int64_t*)(_pts) + (_pidx)) > *((int64_t*)(_bts) + (_bnum) - 1))) || ((!(_asc)) && (*((int64_t*)(_pts) + (_pidx)) < *((int64_t*)(_bts) + (_bnum) - 1))))
#define GRP_REMAIN_ROWS(_grp) ((_grp)->endIdx - (_grp)->readIdx + 1)
#define GRP_DONE(_grp) ((_grp)->readIdx > (_grp)->endIdx)
#define MJOIN_PROBE_TB_ROWS_DONE(_tb) ((_tb)->blkRowIdx >= (_tb)->blk->info.rows)
#define FJOIN_PROBE_TB_ROWS_DONE(_tb) ((NULL == (_tb)->blk) || ((_tb)->blkRowIdx >= (_tb)->blk->info.rows))
#define MJOIN_BUILD_TB_ROWS_DONE(_tb) ((NULL == (_tb)->blk) || ((_tb)->blkRowIdx >= (_tb)->blk->info.rows))
#define MJOIN_TB_GRP_ROWS_DONE(_tb, _grpJoin) ((_tb)->dsFetchDone || ((_grpJoin) && NULL == (_tb)->blk && NULL != (_tb)->remainInBlk))
#define BLK_IS_FULL(_blk) ((_blk)->info.rows == (_blk)->info.capacity)
#define MJOIN_ROW_BITMAP_SET(_b, _base, _idx) (!colDataIsNull_f((_b + _base), _idx))
#define MJOIN_SET_ROW_BITMAP(_b, _base, _idx) colDataClearNull_f((_b + _base), _idx)
#define ASOF_EQ_ROW_INCLUDED(_op) (OP_TYPE_GREATER_EQUAL == (_op) || OP_TYPE_LOWER_EQUAL == (_op) || OP_TYPE_EQUAL == (_op))
#define ASOF_LOWER_ROW_INCLUDED(_op) (OP_TYPE_GREATER_EQUAL == (_op) || OP_TYPE_GREATER_THAN == (_op))
#define ASOF_GREATER_ROW_INCLUDED(_op) (OP_TYPE_LOWER_EQUAL == (_op) || OP_TYPE_LOWER_THAN == (_op))
#define WIN_ONLY_EQ_ROW_INCLUDED(_soff, _eoff) (0 == ((SValueNode*)(_soff))->datum.i && 0 == ((SValueNode*)(_eoff))->datum.i)
#define MJOIN_PUSH_BLK_TO_CACHE(_cache, _blk) \
do { \
ASSERT(taosArrayGetSize((_cache)->grps) <= 1); \
SMJoinGrpRows* pGrp = (SMJoinGrpRows*)taosArrayReserve((_cache)->grps, 1); \
(_cache)->rowNum += (_blk)->info.rows; \
pGrp->blk = (_blk); \
pGrp->beginIdx = 0; \
pGrp->endIdx = (_blk)->info.rows - 1; \
} while (0)
#define MJOIN_RESTORE_TB_BLK(_cache, _tb) \
do { \
SMJoinGrpRows* pGrp = taosArrayGet((_cache)->grps, 0); \
if (NULL != pGrp) { \
(_tb)->blk = pGrp->blk; \
(_tb)->blkRowIdx = pGrp->beginIdx; \
} else { \
(_tb)->blk = NULL; \
(_tb)->blkRowIdx = 0; \
} \
} while (0)
#define MJOIN_SAVE_TB_BLK(_cache, _tb) \
do { \
SMJoinGrpRows* pGrp = taosArrayGet((_cache)->grps, 0); \
if (NULL != pGrp) { \
ASSERT(pGrp->blk == (_tb)->blk); \
pGrp->beginIdx = (_tb)->blkRowIdx; \
pGrp->readIdx = pGrp->beginIdx; \
} \
} while (0)
#define MJOIN_POP_TB_BLK(_cache) \
do { \
SMJoinGrpRows* pGrp = taosArrayGet((_cache)->grps, 0); \
if (NULL != pGrp) { \
if (pGrp->blk == (_cache)->outBlk) { \
blockDataCleanup(pGrp->blk); \
} \
taosArrayPopFrontBatch((_cache)->grps, 1); \
} \
} while (0)
#define MJOIN_GET_TB_COL_TS(_col, _ts, _tb) \
do { \
if (NULL != (_tb)->blk && (_tb)->blkRowIdx < (_tb)->blk->info.rows) { \
(_col) = taosArrayGet((_tb)->blk->pDataBlock, (_tb)->primCtx.targetSlotId); \
(_ts) = *((int64_t*)(_col)->pData + (_tb)->blkRowIdx); \
} else { \
(_ts) = INT64_MAX; \
} \
} while (0)
#define MJOIN_GET_TB_CUR_TS(_col, _ts, _tb) \
do { \
(_ts) = *((int64_t*)(_col)->pData + (_tb)->blkRowIdx); \
} while (0)
#define MJ_ERR_RET(c) \
do { \
int32_t _code = (c); \
if (_code != TSDB_CODE_SUCCESS) { \
terrno = _code; \
return _code; \
} \
} while (0)
#define MJ_ERR_JRET(c) \
do { \
code = (c); \
if (code != TSDB_CODE_SUCCESS) { \
terrno = code; \
goto _return; \
} \
} while (0)
void mJoinDestroyMergeCtx(SMJoinOperatorInfo* pJoin);
void mJoinDestroyWindowCtx(SMJoinOperatorInfo* pJoin);
int32_t mJoinInitWindowCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJoinNode);
int32_t mJoinInitMergeCtx(SMJoinOperatorInfo* pJoin, SSortMergeJoinPhysiNode* pJoinNode);
void mWinJoinResetWindowCache(SMJoinWindowCtx* pCtx, SMJoinWinCache* pCache);
SSDataBlock* mInnerJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mLeftJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mFullJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mSemiJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mAntiJoinDo(struct SOperatorInfo* pOperator);
SSDataBlock* mWinJoinDo(struct SOperatorInfo* pOperator);
void mJoinResetGroupTableCtx(SMJoinTableCtx* pCtx);
void mJoinResetTableCtx(SMJoinTableCtx* pCtx);
void mLeftJoinGroupReset(SMJoinOperatorInfo* pJoin);
void mWinJoinGroupReset(SMJoinOperatorInfo* pJoin);
bool mJoinRetrieveBlk(SMJoinOperatorInfo* pJoin, int32_t* pIdx, SSDataBlock** ppBlk, SMJoinTableCtx* pTb);
void mJoinSetDone(SOperatorInfo* pOperator);
bool mJoinIsDone(SOperatorInfo* pOperator);
bool mJoinCopyKeyColsDataToBuf(SMJoinTableCtx* pTable, int32_t rowIdx, size_t *pBufLen);
int32_t mJoinBuildEqGroups(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, bool restart);
int32_t mJoinRetrieveEqGrpRows(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable, int64_t timestamp);
int32_t mJoinCreateFullBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinCreateBuildTbHash(SMJoinOperatorInfo* pJoin, SMJoinTableCtx* pTable);
int32_t mJoinSetKeyColsData(SSDataBlock* pBlock, SMJoinTableCtx* pTable);
int32_t mJoinProcessEqualGrp(SMJoinMergeCtx* pCtx, int64_t timestamp, bool lastBuildGrp);
bool mJoinHashGrpCart(SSDataBlock* pBlk, SMJoinGrpRows* probeGrp, bool append, SMJoinTableCtx* probe, SMJoinTableCtx* build);
int32_t mJoinMergeGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pFirst, SMJoinGrpRows* pSecond);
int32_t mJoinHandleMidRemains(SMJoinMergeCtx* pCtx);
int32_t mJoinNonEqGrpCart(SMJoinOperatorInfo* pJoin, SSDataBlock* pRes, bool append, SMJoinGrpRows* pGrp, bool probeGrp);
int32_t mJoinNonEqCart(SMJoinCommonCtx* pCtx, SMJoinGrpRows* pGrp, bool probeGrp, bool singleProbeRow);
int32_t mJoinCopyMergeMidBlk(SMJoinMergeCtx* pCtx, SSDataBlock** ppMid, SSDataBlock** ppFin);
int32_t mJoinFilterAndMarkRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SMJoinTableCtx* build, int32_t startGrpIdx, int32_t startRowIdx);
int32_t mJoinFilterAndMarkHashRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo, SMJoinTableCtx* build, int32_t startRowIdx);
int32_t mJoinGetRowBitmapOffset(SMJoinTableCtx* pTable, int32_t rowNum, int32_t *rowBitmapOffset);
int32_t mJoinProcessLowerGrp(SMJoinMergeCtx* pCtx, SMJoinTableCtx* pTb, SColumnInfoData* pCol, int64_t* probeTs, int64_t* buildTs);
int32_t mJoinProcessGreaterGrp(SMJoinMergeCtx* pCtx, SMJoinTableCtx* pTb, SColumnInfoData* pCol, int64_t* probeTs, int64_t* buildTs);
int32_t mJoinFilterAndKeepSingleRow(SSDataBlock* pBlock, SFilterInfo* pFilterInfo);
int32_t mJoinFilterAndNoKeepRows(SSDataBlock* pBlock, SFilterInfo* pFilterInfo);
int32_t mJoinBuildEqGrp(SMJoinTableCtx* pTable, int64_t timestamp, bool* wholeBlk, SMJoinGrpRows* pGrp);
#ifdef __cplusplus
}
#endif
#endif // TDENGINE_MERGEJOIN_H

14
source/libs/executor/src/aggregateoperator.c
View File

@ -47,6 +47,7 @@ typedef struct SAggOperatorInfo {
bool groupKeyOptimized;
bool hasValidBlock;
SSDataBlock* pNewGroupBlock;
bool hasCountFunc;
} SAggOperatorInfo;
static void destroyAggOperatorInfo(void* param);
@ -111,6 +112,7 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SAggPhysiN
pInfo->groupId = UINT64_MAX;
pInfo->binfo.inputTsOrder = pAggNode->node.inputTsOrder;
pInfo->binfo.outputTsOrder = pAggNode->node.outputTsOrder;
pInfo->hasCountFunc = pAggNode->hasCountLikeFunc;
setOperatorInfo(pOperator, "TableAggregate", QUERY_NODE_PHYSICAL_PLAN_HASH_AGG,
!pAggNode->node.forceCreateNonBlockingOptr, OP_NOT_OPENED, pInfo, pTaskInfo);
@ -317,18 +319,8 @@ static int32_t createDataBlockForEmptyInput(SOperatorInfo* pOperator, SSDataBloc
}
SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx;
bool hasCountFunc = false;
for (int32_t i = 0; i < pOperator->exprSupp.numOfExprs; ++i) {
const char* pName = pCtx[i].pExpr->pExpr->_function.functionName;
if ((strcmp(pName, "count") == 0) || (strcmp(pName, "hyperloglog") == 0) ||
(strcmp(pName, "_hyperloglog_partial") == 0) || (strcmp(pName, "_hyperloglog_merge") == 0)) {
hasCountFunc = true;
break;
}
}
if (!hasCountFunc) {
if (!pAggInfo->hasCountFunc) {
return TSDB_CODE_SUCCESS;
}

8
source/libs/executor/src/dynqueryctrloperator.c
View File

@ -48,7 +48,7 @@ static void destroyStbJoinTableList(SStbJoinTableList* pListHead) {
}
static void destroyStbJoinDynCtrlInfo(SStbJoinDynCtrlInfo* pStbJoin) {
qError("dynQueryCtrl exec info, prevBlk:%" PRId64 ", prevRows:%" PRId64 ", postBlk:%" PRId64 ", postRows:%" PRId64 ", leftCacheNum:%" PRId64 ", rightCacheNum:%" PRId64,
qDebug("dynQueryCtrl exec info, prevBlk:%" PRId64 ", prevRows:%" PRId64 ", postBlk:%" PRId64 ", postRows:%" PRId64 ", leftCacheNum:%" PRId64 ", rightCacheNum:%" PRId64,
pStbJoin->execInfo.prevBlkNum, pStbJoin->execInfo.prevBlkRows, pStbJoin->execInfo.postBlkNum,
pStbJoin->execInfo.postBlkRows, pStbJoin->execInfo.leftCacheNum, pStbJoin->execInfo.rightCacheNum);
@ -432,7 +432,7 @@ static int32_t buildSeqStbJoinOperatorParam(SDynQueryCtrlOperatorInfo* pInfo, SS
int64_t* rightUid = pPrev->pListHead->pRightUid + rowIdx;
int32_t code = TSDB_CODE_SUCCESS;
qError("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
qDebug("start %" PRId64 ":%" PRId64 "th stbJoin, left:%d,%" PRIu64 " - right:%d,%" PRIu64,
rowIdx, pPrev->tableNum, *leftVg, *leftUid, *rightVg, *rightUid);
updatePostJoinCurrTableInfo(&pInfo->stbJoin);
@ -524,7 +524,7 @@ static int32_t notifySeqJoinTableCacheEnd(SOperatorInfo* pOperator, SStbJoinPost
int32_t vgId = leftTable ? pPost->leftVgId : pPost->rightVgId;
int64_t uid = leftTable ? pPost->leftCurrUid : pPost->rightCurrUid;
qError("notify table %" PRIu64 " in vgId %d downstreamId %d cache end", uid, vgId, downstreamId);
qDebug("notify table %" PRIu64 " in vgId %d downstreamId %d cache end", uid, vgId, downstreamId);
int32_t code = buildGroupCacheNotifyOperatorParam(&pGcParam, downstreamId, vgId, uid);
if (TSDB_CODE_SUCCESS != code) {
@ -758,7 +758,7 @@ static void postProcessStbJoinTableHash(SOperatorInfo* pOperator) {
}
pStbJoin->execInfo.leftCacheNum = tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache);
qError("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache));
qDebug("more than 1 ref build table num %" PRId64, (int64_t)tSimpleHashGetSize(pStbJoin->ctx.prev.leftCache));
// debug only
iter = 0;

1
source/libs/executor/src/executorInt.c
View File

@ -77,7 +77,6 @@ static void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExpr, SSDataBlock*
static void initCtxOutputBuffer(SqlFunctionCtx* pCtx, int32_t size);
static void doApplyScalarCalculation(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t order, int32_t scanFlag);
static void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const SColumnInfoData* p, int32_t status);
static int32_t doSetInputDataBlock(SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t order, int32_t scanFlag,
bool createDummyCol);
static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf,

36
source/libs/executor/src/groupcacheoperator.c
View File

@ -348,7 +348,7 @@ static int32_t addBlkToDirtyBufList(SGroupCacheOperatorInfo* pGCache, SGcDownstr
taosWLockLatch(&pCache->dirtyLock);
pCache->blkCacheSize += pBufInfo->basic.bufSize;
qError("group cache total dirty block num:%d size:%" PRId64 , taosHashGetSize(pCache->pDirtyBlk), pCache->blkCacheSize);
qDebug("group cache total dirty block num:%d size:%" PRId64 , taosHashGetSize(pCache->pDirtyBlk), pCache->blkCacheSize);
if (NULL == pCache->pDirtyHead) {
pCache->pDirtyHead = pBufInfo;
@ -664,7 +664,7 @@ static FORCE_INLINE int32_t getBlkFromDownstreamOperator(struct SOperatorInfo* p
}
if (pBlock) {
qError("%s blk retrieved from group %" PRIu64, GET_TASKID(pOperator->pTaskInfo), pBlock->info.id.groupId);
qDebug("%s blk retrieved from group %" PRIu64, GET_TASKID(pOperator->pTaskInfo), pBlock->info.id.groupId);
pGCache->execInfo.pDownstreamBlkNum[downstreamIdx]++;
if (NULL == pGCache->pDownstreams[downstreamIdx].pBaseBlock) {
@ -803,7 +803,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
}
initNewGroupData(pCtx, *ppGrp, pParam->downstreamIdx, vgId, pGCache->batchFetch, pGcParam->needCache);
qError("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
qDebug("new group %" PRIu64 " initialized, downstreamIdx:%d, vgId:%d, needCache:%d", uid, pParam->downstreamIdx, vgId, pGcParam->needCache);
if (pParam->pChildren) {
SGcNewGroupInfo newGroup;
@ -821,6 +821,7 @@ static int32_t addNewGroupData(struct SOperatorInfo* pOperator, SOperatorParam*
taosArrayDestroy(pParam->pChildren);
pParam->pChildren = NULL;
pCtx->fetchDone = false;
}
return TSDB_CODE_SUCCESS;
@ -832,7 +833,7 @@ static int32_t addBlkToGroupCache(bool batchFetch, SGroupCacheData* pGroup, SGcB
*pIdx = taosArrayGetSize(pGroup->blkList.pList) - 1;
taosWUnLockLatch(&pGroup->blkList.lock);
qError("block added to group cache, total block num:%" PRId64, *pIdx + 1);
qDebug("block added to group cache, total block num:%" PRId64, *pIdx + 1);
return TSDB_CODE_SUCCESS;
}
@ -870,7 +871,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
}
if (pGroup->needCache) {
qError("add block to group cache");
qDebug("add block to group cache");
SGcBlkBufInfo newBlkBuf;
code = addBlkToBufCache(pOperator, pBlock, pCtx, pGroup, &newBlkBuf);
@ -883,7 +884,7 @@ static int32_t handleGroupCacheRetrievedBlk(struct SOperatorInfo* pOperator, SSD
return code;
}
} else {
qError("no need to add block to group cache");
qDebug("no need to add block to group cache");
pGroup->pBlock = pBlock;
}
@ -910,6 +911,7 @@ static int32_t handleDownstreamFetchDone(struct SOperatorInfo* pOperator, SGcSes
while (NULL != (pGroup = taosHashIterate(pGrpHash, pGroup))) {
handleGroupFetchDone(pGroup);
}
pCtx->fetchDone = true;
} else {
int32_t uidNum = 0;
SGcVgroupCtx* pVgCtx = NULL;
@ -979,6 +981,7 @@ static int32_t getBlkFromSessionCacheImpl(struct SOperatorInfo* pOperator, int64
SGroupCacheOperatorInfo* pGCache = pOperator->info;
*got = true;
if (NULL != pSession->pGroupData) {
if (pSession->pGroupData->needCache) {
SGcBlkList* pBlkList = &pSession->pGroupData->blkList;
taosRLockLatch(&pBlkList->lock);
@ -1002,10 +1005,17 @@ static int32_t getBlkFromSessionCacheImpl(struct SOperatorInfo* pOperator, int64
} else if (pSession->pGroupData->pBlock) {
*ppRes = pSession->pGroupData->pBlock;
pSession->pGroupData->pBlock = NULL;
return TSDB_CODE_SUCCESS;
}
if (atomic_load_8((int8_t*)&pSession->pGroupData->fetchDone)) {
*ppRes = NULL;
qDebug("sessionId: %" PRIu64 " fetch done", sessionId);
return code;
}
} else {
*ppRes = NULL;
qDebug("sessionId: %" PRIu64 " fetch done since downstream fetch done", sessionId);
return code;
}
@ -1136,13 +1146,17 @@ static int32_t initGroupCacheSession(struct SOperatorInfo* pOperator, SOperatorP
SHashObj* pGrpHash = pGCache->globalGrp ? pGCache->pGrpHash : pCtx->pGrpHash;
SGroupCacheData* pGroup = taosHashGet(pGrpHash, &pGcParam->tbUid, sizeof(pGcParam->tbUid));
if (NULL == pGroup) {
if (NULL == pGroup && (NULL != pParam->pChildren || !pCtx->fetchDone)) {
code = addNewGroupData(pOperator, pParam, &pGroup, pGCache->batchFetch ? GROUP_CACHE_DEFAULT_VGID : pGcParam->vgId, pGcParam->tbUid);
if (TSDB_CODE_SUCCESS != code) {
return code;
}
}
if (NULL == pGroup) {
return TSDB_CODE_SUCCESS;
}
initGroupCacheSessionCtx(&ctx, pGcParam, pGroup);
code = taosHashPut(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId), &ctx, sizeof(ctx));
@ -1166,6 +1180,10 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
if (TSDB_CODE_SUCCESS != code) {
return code;
}
if (NULL == pSession) {
qDebug("session %" PRId64 " in downstream %d total got 0 rows since downtream fetch done", pGcParam->sessionId, pCtx->id);
return TSDB_CODE_SUCCESS;
}
} else if (pSession->pGroupData->needCache) {
SSDataBlock** ppBlock = taosHashGet(pGCache->blkCache.pReadBlk, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
if (ppBlock) {
@ -1176,11 +1194,11 @@ static int32_t getBlkFromGroupCache(struct SOperatorInfo* pOperator, SSDataBlock
code = getBlkFromSessionCache(pOperator, pGcParam->sessionId, pSession, ppRes);
if (NULL == *ppRes) {
qError("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
qDebug("session %" PRId64 " in downstream %d total got %" PRId64 " rows", pGcParam->sessionId, pCtx->id, pSession->resRows);
taosHashRemove(pCtx->pSessions, &pGcParam->sessionId, sizeof(pGcParam->sessionId));
} else {
pSession->resRows += (*ppRes)->info.rows;
qError("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
qDebug("session %" PRId64 " in downstream %d got %" PRId64 " rows in one block", pGcParam->sessionId, pCtx->id, (*ppRes)->info.rows);
}
return code;

349
source/libs/executor/src/hashjoin.c Executable file
View File

@ -0,0 +1,349 @@
/*
* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
*
* This program is free software: you can use, redistribute, and/or modify
* it under the terms of the GNU Affero General Public License, version 3
* or later ("AGPL"), as published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "executorInt.h"
#include "filter.h"
#include "function.h"
#include "operator.h"
#include "os.h"
#include "querynodes.h"
#include "querytask.h"
#include "tcompare.h"
#include "tdatablock.h"
#include "thash.h"
#include "tmsg.h"
#include "ttypes.h"
#include "hashjoin.h"
int32_t hInnerJoinDo(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinTableCtx* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
SSDataBlock* pRes = pJoin->finBlk;
size_t bufLen = 0;
int32_t code = 0;
bool allFetched = false;
if (pJoin->ctx.pBuildRow) {
hJoinAppendResToBlock(pOperator, pRes, &allFetched);
if (pRes->info.rows >= pRes->info.capacity) {
if (allFetched) {
++pCtx->probeStartIdx;
}
return code;
} else {
++pCtx->probeStartIdx;
}
}
for (; pCtx->probeStartIdx <= pCtx->probeEndIdx; ++pCtx->probeStartIdx) {
if (hJoinCopyKeyColsDataToBuf(pProbe, pCtx->probeStartIdx, &bufLen)) {
continue;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (pGroup) {
pCtx->pBuildRow = pGroup->rows;
hJoinAppendResToBlock(pOperator, pRes, &allFetched);
if (pRes->info.rows >= pRes->info.capacity) {
if (allFetched) {
++pCtx->probeStartIdx;
}
return code;
}
}
}
pCtx->rowRemains = false;
return code;
}
int32_t hLeftJoinHandleSeqRowRemains(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
bool allFetched = false;
SHJoinCtx* pCtx = &pJoin->ctx;
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
if (pCtx->midRemains) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
}
if (allFetched && !pCtx->readMatch) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
}
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
++pCtx->probeStartIdx;
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinHandleSeqProbeRows(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
SHJoinTableCtx* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
size_t bufLen = 0;
bool allFetched = false;
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
return TSDB_CODE_SUCCESS;
}
for (; pCtx->probeStartIdx <= pCtx->probeEndIdx; ++pCtx->probeStartIdx) {
if (hJoinCopyKeyColsDataToBuf(pProbe, pCtx->probeStartIdx, &bufLen)) {
continue;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (NULL == pGroup) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
++pCtx->probeStartIdx;
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
continue;
}
pCtx->readMatch = false;
pCtx->pBuildRow = pGroup->rows;
allFetched = false;
while (!allFetched) {
hJoinAppendResToBlock(pOperator, pJoin->midBlk, &allFetched);
if (pJoin->midBlk->info.rows > 0) {
doFilter(pJoin->midBlk, pJoin->pPreFilter, NULL);
if (pJoin->midBlk->info.rows > 0) {
pCtx->readMatch = true;
HJ_ERR_RET(hJoinCopyMergeMidBlk(pCtx, &pJoin->midBlk, &pJoin->finBlk));
if (pCtx->midRemains) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
}
if (allFetched && !pCtx->readMatch) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
}
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
}
pCtx->probePhase = E_JOIN_PHASE_POST;
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinHandleRowRemains(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
bool allFetched = false;
SHJoinCtx* pCtx = &pJoin->ctx;
hJoinAppendResToBlock(pOperator, pJoin->finBlk, &allFetched);
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
} else {
++pCtx->probeStartIdx;
}
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinHandleProbeRows(struct SOperatorInfo* pOperator, SHJoinOperatorInfo* pJoin, bool* loopCont) {
SHJoinTableCtx* pProbe = pJoin->pProbe;
SHJoinCtx* pCtx = &pJoin->ctx;
size_t bufLen = 0;
bool allFetched = false;
for (; pCtx->probeStartIdx <= pCtx->probeEndIdx; ++pCtx->probeStartIdx) {
if (hJoinCopyKeyColsDataToBuf(pProbe, pCtx->probeStartIdx, &bufLen)) {
continue;
}
SGroupData* pGroup = tSimpleHashGet(pJoin->pKeyHash, pProbe->keyData, bufLen);
/*
size_t keySize = 0;
int32_t* pKey = tSimpleHashGetKey(pGroup, &keySize);
ASSERT(keySize == bufLen && 0 == memcmp(pKey, pProbe->keyData, bufLen));
int64_t rows = getSingleKeyRowsNum(pGroup->rows);
pJoin->execInfo.expectRows += rows;
qTrace("hash_key:%d, rows:%" PRId64, *pKey, rows);
*/
if (NULL == pGroup) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pCtx->probeStartIdx, 1));
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
++pCtx->probeStartIdx;
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
continue;
}
pCtx->pBuildRow = pGroup->rows;
hJoinAppendResToBlock(pOperator, pJoin->finBlk, &allFetched);
if (hJoinBlkReachThreshold(pJoin, pJoin->finBlk->info.rows)) {
if (allFetched) {
++pCtx->probeStartIdx;
}
*loopCont = false;
return TSDB_CODE_SUCCESS;
}
}
pCtx->probePhase = E_JOIN_PHASE_POST;
*loopCont = true;
return TSDB_CODE_SUCCESS;
}
int32_t hLeftJoinDo(struct SOperatorInfo* pOperator) {
SHJoinOperatorInfo* pJoin = pOperator->info;
SHJoinCtx* pCtx = &pJoin->ctx;
while (pCtx->rowRemains) {
switch (pCtx->probePhase) {
case E_JOIN_PHASE_PRE: {
int32_t rows = pCtx->probeStartIdx - pCtx->probePreIdx;
int32_t rowsLeft = pJoin->finBlk->info.capacity - pJoin->finBlk->info.rows;
if (rows <= rowsLeft) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, 0, rows));
pCtx->probePhase = E_JOIN_PHASE_CUR;
} else {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, 0, rowsLeft));
pJoin->ctx.probePreIdx += rowsLeft;
return TSDB_CODE_SUCCESS;
}
break;
}
case E_JOIN_PHASE_CUR: {
bool loopCont = false;
if (NULL == pJoin->ctx.pBuildRow) {
HJ_ERR_RET(pJoin->pPreFilter ? hLeftJoinHandleSeqProbeRows(pOperator, pJoin, &loopCont) : hLeftJoinHandleProbeRows(pOperator, pJoin, &loopCont));
} else {
HJ_ERR_RET(pJoin->pPreFilter ? hLeftJoinHandleSeqRowRemains(pOperator, pJoin, &loopCont) : hLeftJoinHandleRowRemains(pOperator, pJoin, &loopCont));
}
if (!loopCont) {
return TSDB_CODE_SUCCESS;
}
break;
}
case E_JOIN_PHASE_POST: {
if (pCtx->probeEndIdx < (pCtx->pProbeData->info.rows - 1) && pCtx->probePostIdx <= (pCtx->pProbeData->info.rows - 1)) {
int32_t rowsLeft = pJoin->finBlk->info.capacity - pJoin->finBlk->info.rows;
int32_t rows = pCtx->pProbeData->info.rows - pCtx->probePostIdx;
if (rows <= rowsLeft) {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pJoin->ctx.probePostIdx, rows));
pCtx->rowRemains = false;
} else {
HJ_ERR_RET(hJoinCopyNMatchRowsToBlock(pJoin, pJoin->finBlk, pJoin->ctx.probePostIdx, rowsLeft));
pCtx->probePostIdx += rowsLeft;
return TSDB_CODE_SUCCESS;
}
} else {
pJoin->ctx.rowRemains = false;
}
break;
}
default:
return TSDB_CODE_QRY_EXECUTOR_INTERNAL_ERROR;
}
}
return TSDB_CODE_SUCCESS;
}

673
source/libs/executor/src/hashjoinoperator.c
View File

File diff suppressed because it is too large Load Diff

3458
source/libs/executor/src/mergejoin.c Executable file
View File

File diff suppressed because it is too large Load Diff

2390
source/libs/executor/src/mergejoinoperator.c
View File

File diff suppressed because it is too large Load Diff

1
source/libs/executor/src/mergeoperator.c
View File

@ -282,6 +282,7 @@ SSDataBlock* doNonSortMerge(SOperatorInfo* pOperator) {
idx = NON_SORT_NEXT_SRC(pNonSortMerge, pNonSortMerge->lastSourceIdx);
continue;
}
pNonSortMerge->lastSourceIdx = idx - 1;
break;
}

4
source/libs/executor/src/projectoperator.c
View File

@ -231,7 +231,7 @@ static int32_t doIngroupLimitOffset(SLimitInfo* pLimitInfo, uint64_t groupId, SS
// set current group id
pLimitInfo->currentGroupId = groupId;
bool limitReached = applyLimitOffset(pLimitInfo, pBlock, pOperator->pTaskInfo);
if (pBlock->info.rows == 0) {
if (pBlock->info.rows == 0 && 0 != pLimitInfo->limit.limit) {
return PROJECT_RETRIEVE_CONTINUE;
} else {
if (limitReached && (pLimitInfo->slimit.limit >= 0 && pLimitInfo->slimit.limit <= pLimitInfo->numOfOutputGroups)) {
@ -352,7 +352,7 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) {
// continue merge data, ignore the group id
blockDataMerge(pFinalRes, pRes);
if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold) {
if (pFinalRes->info.rows + pRes->info.rows <= pOperator->resultInfo.threshold && (pOperator->status != OP_EXEC_DONE)) {
continue;
}
}

18
source/libs/executor/src/scanoperator.c
View File

@ -928,7 +928,7 @@ static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) {
return TSDB_CODE_INVALID_PARA;
}
qError("vgId:%d add total %d dynamic tables to scan, tableSeq:%d, exist num:%" PRId64 ", operator status:%d",
qDebug("vgId:%d add total %d dynamic tables to scan, tableSeq:%d, exist num:%" PRId64 ", operator status:%d",
pTaskInfo->id.vgId, num, pParam->tableSeq, (int64_t)taosArrayGetSize(pListInfo->pTableList), pOperator->status);
if (pParam->tableSeq) {
@ -962,7 +962,7 @@ static int32_t createTableListInfoFromParam(SOperatorInfo* pOperator) {
}
tableIdx++;
qError("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
qDebug("add dynamic table scan uid:%" PRIu64 ", %s", info.uid, GET_TASKID(pTaskInfo));
}
return code;
@ -4056,7 +4056,7 @@ static int32_t stopSubTablesTableMergeScan(STableMergeScanInfo* pInfo) {
taosMemoryFree(pSubTblsInfo);
pInfo->pSubTablesMergeInfo = NULL;
taosMemoryTrim(0);
//taosMemoryTrim(0);
}
return TSDB_CODE_SUCCESS;
}
@ -4074,6 +4074,8 @@ SSDataBlock* doTableMergeScanParaSubTables(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code);
}
int64_t st = taosGetTimestampUs();
size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
@ -4102,7 +4104,7 @@ SSDataBlock* doTableMergeScanParaSubTables(SOperatorInfo* pOperator) {
pBlock->info.id.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->bGroupProcessed = true;
return pBlock;
break;
} else {
// Data of this group are all dumped, let's try the next group
stopSubTablesTableMergeScan(pInfo);
@ -4118,6 +4120,8 @@ SSDataBlock* doTableMergeScanParaSubTables(SOperatorInfo* pOperator) {
}
}
pOperator->cost.totalCost += (taosGetTimestampUs() - st) / 1000.0;;
return pBlock;
}
@ -4480,6 +4484,8 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, code);
}
int64_t st = taosGetTimestampUs();
size_t tableListSize = tableListGetSize(pInfo->base.pTableListInfo);
if (!pInfo->hasGroupId) {
pInfo->hasGroupId = true;
@ -4509,7 +4515,7 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
pBlock->info.id.groupId = pInfo->groupId;
pOperator->resultInfo.totalRows += pBlock->info.rows;
pInfo->bGroupProcessed = true;
return pBlock;
break;
} else {
if (pInfo->bNewFilesetEvent) {
stopDurationForGroupTableMergeScan(pOperator);
@ -4533,6 +4539,8 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) {
}
}
pOperator->cost.totalCost += (taosGetTimestampUs() - st) / 1000.0;;
return pBlock;
}

5
source/libs/executor/src/sysscanoperator.c
View File

@ -1408,6 +1408,11 @@ static SSDataBlock* sysTableBuildUserTables(SOperatorInfo* pOperator) {
T_LONG_JMP(pTaskInfo->env, terrno);
}
if (isTsmaResSTb(mr.me.name)) {
pAPI->metaReaderFn.clearReader(&mr);
continue;
}
// number of columns
pColInfoData = taosArrayGet(p->pDataBlock, 3);
colDataSetVal(pColInfoData, numOfRows, (char*)&mr.me.stbEntry.schemaRow.nCols, false);

48
source/libs/executor/test/CMakeLists.txt
View File

@ -1,35 +1,35 @@
MESSAGE(STATUS "build parser unit test")
IF(NOT TD_DARWIN)
# GoogleTest requires at least C++11
# IF(NOT TD_DARWIN)
# # GoogleTest requires at least C++11
# SET(CMAKE_CXX_STANDARD 11)
# AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
#
# ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
# TARGET_LINK_LIBRARIES(
# executorTest
# PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
# )
#
# TARGET_INCLUDE_DIRECTORIES(
# executorTest
# PUBLIC "${TD_SOURCE_DIR}/include/libs/executor/"
# PRIVATE "${TD_SOURCE_DIR}/source/libs/executor/inc"
# )
# ENDIF ()
SET(CMAKE_CXX_STANDARD 11)
AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
ADD_EXECUTABLE(executorTest ${SOURCE_LIST})
ADD_EXECUTABLE(joinTests joinTests.cpp)
TARGET_LINK_LIBRARIES(
executorTest
PRIVATE os util common transport gtest taos_static qcom executor function planner scalar nodes vnode
joinTests
PRIVATE os util common executor gtest_main qcom function planner scalar nodes vnode
)
TARGET_INCLUDE_DIRECTORIES(
executorTest
PUBLIC "${TD_SOURCE_DIR}/include/libs/executor/"
PRIVATE "${TD_SOURCE_DIR}/source/libs/executor/inc"
joinTests
PUBLIC "${TD_SOURCE_DIR}/include/common"
PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
ENDIF ()
# SET(CMAKE_CXX_STANDARD 11)
# AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR} SOURCE_LIST)
# ADD_EXECUTABLE(tSimpleHashTest tSimpleHashTests.cpp)
# TARGET_LINK_LIBRARIES(
# tSimpleHashTest
# PRIVATE os util common executor gtest_main
# )
# TARGET_INCLUDE_DIRECTORIES(
# tSimpleHashTest
# PUBLIC "${TD_SOURCE_DIR}/include/common"
# PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
# )

3663
source/libs/executor/test/joinTests.cpp Executable file
View File

File diff suppressed because it is too large Load Diff

1
source/libs/function/inc/builtins.h
View File

@ -47,6 +47,7 @@ typedef struct SBuiltinFuncDefinition {
const char* pPartialFunc;
const char* pMiddleFunc;
const char* pMergeFunc;
const char* pStateFunc;
FCreateMergeFuncParameters createMergeParaFuc;
FEstimateReturnRows estimateReturnRowsFunc;
} SBuiltinFuncDefinition;

5
source/libs/function/inc/functionMgtInt.h
View File

@ -53,7 +53,10 @@ extern "C" {
#define FUNC_MGT_GEOMETRY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(24)
#define FUNC_MGT_FORBID_SYSTABLE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(25)
#define FUNC_MGT_SKIP_SCAN_CHECK_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(26)
#define FUNC_MGT_PRIMARY_KEY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(27)
#define FUNC_MGT_IGNORE_NULL_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(27)
#define FUNC_MGT_PRIMARY_KEY_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(28)
#define FUNC_MGT_TSMA_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(29)
#define FUNC_MGT_COUNT_LIKE_FUNC FUNC_MGT_FUNC_CLASSIFICATION_MASK(30) // funcs that should also return 0 when no rows found
#define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)

343
source/libs/function/src/builtins.c
View File

@ -46,7 +46,7 @@ static int32_t invaildFuncParaValueErrMsg(char* pErrBuf, int32_t len, const char
#define TIME_UNIT_TOO_SMALL 2
static int32_t validateTimeUnitParam(uint8_t dbPrec, const SValueNode* pVal) {
if (!pVal->isDuration) {
if (!IS_DURATION_VAL(pVal->flag)) {
return TIME_UNIT_INVALID;
}
@ -225,7 +225,6 @@ static int32_t addTimezoneParam(SNodeList* pList) {
}
pVal->literal = strndup(buf, len);
pVal->isDuration = false;
pVal->translate = true;
pVal->node.resType.type = TSDB_DATA_TYPE_BINARY;
pVal->node.resType.bytes = len + VARSTR_HEADER_SIZE;
@ -245,7 +244,6 @@ static int32_t addDbPrecisonParam(SNodeList** pList, uint8_t precision) {
}
pVal->literal = NULL;
pVal->isDuration = false;
pVal->translate = true;
pVal->notReserved = true;
pVal->node.resType.type = TSDB_DATA_TYPE_TINYINT;
@ -462,6 +460,24 @@ static int32_t translateAvgMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return TSDB_CODE_SUCCESS;
}
static int32_t translateAvgState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType))
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateAvgStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
pFunc->node.resType = (SDataType){.bytes = getAvgInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateStddevPartial(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@ -491,6 +507,35 @@ static int32_t translateStddevMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t
return TSDB_CODE_SUCCESS;
}
static int32_t translateStddevState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getStddevInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateStddevStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (TSDB_DATA_TYPE_BINARY != paraType) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getStddevInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateWduration(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
// pseudo column do not need to check parameters
pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT,
@ -547,6 +592,9 @@ static int32_t translatePercentile(SFunctionNode* pFunc, char* pErrBuf, int32_t
for (int32_t i = 1; i < numOfParams; ++i) {
SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, i);
if (QUERY_NODE_VALUE != nodeType(pValue)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pValue->notReserved = true;
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, i))->type;
@ -834,6 +882,31 @@ static int32_t translateSpreadMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t
return translateSpreadImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateSpreadState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (!IS_NUMERIC_TYPE(paraType) && !IS_TIMESTAMP_TYPE(paraType)) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateSpreadStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t paraType = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (paraType != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = getSpreadInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateElapsed(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList);
if (1 != numOfParams && 2 != numOfParams) {
@ -1317,6 +1390,24 @@ static int32_t translateHLLMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t le
return translateHLLImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateHLLState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
return translateHLLPartial(pFunc, pErrBuf, len);
}
static int32_t translateHLLStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
if (getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType =
(SDataType){.bytes = getHistogramInfoSize() + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static bool validateStateOper(const SValueNode* pVal) {
if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) {
return false;
@ -1804,6 +1895,28 @@ static int32_t translateFirstLastMerge(SFunctionNode* pFunc, char* pErrBuf, int3
return translateFirstLastImpl(pFunc, pErrBuf, len, false);
}
static int32_t translateFirstLastState(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
int32_t paraBytes = getSDataTypeFromNode(pPara)->bytes;
int32_t pkBytes = (pFunc->hasPk) ? pFunc->pkBytes : 0;
pFunc->node.resType =
(SDataType){.bytes = getFirstLastInfoSize(paraBytes, pkBytes) + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateFirstLastStateMerge(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0);
int32_t paraBytes = getSDataTypeFromNode(pPara)->bytes;
uint8_t paraType = getSDataTypeFromNode(pPara)->type;
if (paraType != TSDB_DATA_TYPE_BINARY) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = paraBytes, .type = TSDB_DATA_TYPE_BINARY};
return TSDB_CODE_SUCCESS;
}
static int32_t translateUniqueMode(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isUnique) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
@ -2394,12 +2507,26 @@ static int32_t translateTableCountPseudoColumn(SFunctionNode* pFunc, char* pErrB
return TSDB_CODE_SUCCESS;
}
static int32_t translateMd5(SFunctionNode* pFunc, char* pErrBuf, int32_t len) {
if (1 != LIST_LENGTH(pFunc->pParameterList)) {
return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName);
}
uint8_t para1Type = getSDataTypeFromNode(nodesListGetNode(pFunc->pParameterList, 0))->type;
if (para1Type != TSDB_DATA_TYPE_VARCHAR) {
return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName);
}
pFunc->node.resType = (SDataType){.bytes = MD5_OUTPUT_LEN, .type = TSDB_DATA_TYPE_VARCHAR};
return TSDB_CODE_SUCCESS;
}
// clang-format off
const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "count",
.type = FUNCTION_TYPE_COUNT,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_COUNT_LIKE_FUNC,
.translateFunc = translateCount,
.dataRequiredFunc = countDataRequired,
.getEnvFunc = getCountFuncEnv,
@ -2412,12 +2539,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = combineFunction,
.pPartialFunc = "count",
.pStateFunc = "count",
.pMergeFunc = "sum"
},
{
.name = "sum",
.type = FUNCTION_TYPE_SUM,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSum,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSumFuncEnv,
@ -2430,12 +2558,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = sumCombine,
.pPartialFunc = "sum",
.pStateFunc = "sum",
.pMergeFunc = "sum"
},
{
.name = "min",
.type = FUNCTION_TYPE_MIN,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInOutNum,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getMinmaxFuncEnv,
@ -2445,12 +2574,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = minmaxFunctionFinalize,
.combineFunc = minCombine,
.pPartialFunc = "min",
.pStateFunc = "min",
.pMergeFunc = "min"
},
{
.name = "max",
.type = FUNCTION_TYPE_MAX,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInOutNum,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getMinmaxFuncEnv,
@ -2460,12 +2590,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.finalizeFunc = minmaxFunctionFinalize,
.combineFunc = maxCombine,
.pPartialFunc = "max",
.pStateFunc = "max",
.pMergeFunc = "max"
},
{
.name = "stddev",
.type = FUNCTION_TYPE_STDDEV,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInNumOutDou,
.getEnvFunc = getStddevFuncEnv,
.initFunc = stddevFunctionSetup,
@ -2477,6 +2608,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = stddevCombine,
.pPartialFunc = "_stddev_partial",
.pStateFunc = "_stddev_state",
.pMergeFunc = "_stddev_merge"
},
{
@ -2506,6 +2638,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = stddevInvertFunction,
#endif
.combineFunc = stddevCombine,
.pPartialFunc = "_stddev_state_merge",
.pMergeFunc = "_stddev_merge",
},
{
.name = "leastsquares",
@ -2525,7 +2659,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "avg",
.type = FUNCTION_TYPE_AVG,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateInNumOutDou,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getAvgFuncEnv,
@ -2539,12 +2673,13 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.combineFunc = avgCombine,
.pPartialFunc = "_avg_partial",
.pMiddleFunc = "_avg_middle",
.pMergeFunc = "_avg_merge"
.pMergeFunc = "_avg_merge",
.pStateFunc = "_avg_state",
},
{
.name = "_avg_partial",
.type = FUNCTION_TYPE_AVG_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateAvgPartial,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getAvgFuncEnv,
@ -2559,7 +2694,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_avg_merge",
.type = FUNCTION_TYPE_AVG_MERGE,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateAvgMerge,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
@ -2569,6 +2704,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = avgInvertFunction,
#endif
.combineFunc = avgCombine,
.pPartialFunc = "_avg_state_merge",
.pMergeFunc = "_avg_merge",
},
{
.name = "percentile",
@ -2636,7 +2773,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "top",
.type = FUNCTION_TYPE_TOP,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC |
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC,
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateTopBot,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = topBotFunctionSetup,
@ -2652,7 +2789,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "bottom",
.type = FUNCTION_TYPE_BOTTOM,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_ROWS_FUNC | FUNC_MGT_KEEP_ORDER_FUNC |
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC,
FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_FILL_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateTopBot,
.getEnvFunc = getTopBotFuncEnv,
.initFunc = topBotFunctionSetup,
@ -2667,7 +2804,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "spread",
.type = FUNCTION_TYPE_SPREAD,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSpread,
.dataRequiredFunc = statisDataRequired,
.getEnvFunc = getSpreadFuncEnv,
@ -2680,6 +2817,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = spreadCombine,
.pPartialFunc = "_spread_partial",
.pStateFunc = "_spread_state",
.pMergeFunc = "_spread_merge"
},
{
@ -2711,6 +2849,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = NULL,
#endif
.combineFunc = spreadCombine,
.pPartialFunc = "_spread_state_merge",
.pMergeFunc = "_spread_merge",
},
{
.name = "elapsed",
@ -2850,7 +2990,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "_cache_last",
.type = FUNCTION_TYPE_CACHE_LAST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_FORBID_STREAM_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC,
.translateFunc = translateFirstLast,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
@ -2884,7 +3024,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "first",
.type = FUNCTION_TYPE_FIRST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateFirstLast,
.dynDataRequiredFunc = firstDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2893,6 +3033,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = firstLastScalarFunction,
.finalizeFunc = firstLastFinalize,
.pPartialFunc = "_first_partial",
.pStateFunc = "_first_state",
.pMergeFunc = "_first_merge",
.combineFunc = firstCombine,
},
@ -2900,7 +3041,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_first_partial",
.type = FUNCTION_TYPE_FIRST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastPartial,
.dynDataRequiredFunc = firstDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2913,19 +3054,21 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_first_merge",
.type = FUNCTION_TYPE_FIRST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = firstFunctionMerge,
.finalizeFunc = firstLastFinalize,
.combineFunc = firstCombine,
.pPartialFunc = "_first_state_merge",
.pMergeFunc = "_first_merge",
},
{
.name = "last",
.type = FUNCTION_TYPE_LAST,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateFirstLast,
.dynDataRequiredFunc = lastDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2934,6 +3077,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.sprocessFunc = firstLastScalarFunction,
.finalizeFunc = firstLastFinalize,
.pPartialFunc = "_last_partial",
.pStateFunc = "_last_state",
.pMergeFunc = "_last_merge",
.combineFunc = lastCombine,
},
@ -2941,7 +3085,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_last_partial",
.type = FUNCTION_TYPE_LAST_PARTIAL,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastPartial,
.dynDataRequiredFunc = lastDynDataReq,
.getEnvFunc = getFirstLastFuncEnv,
@ -2954,13 +3098,15 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.name = "_last_merge",
.type = FUNCTION_TYPE_LAST_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_IGNORE_NULL_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastFunctionMerge,
.finalizeFunc = firstLastFinalize,
.combineFunc = lastCombine,
.pPartialFunc = "_last_state_merge",
.pMergeFunc = "_last_merge",
},
{
.name = "twa",
@ -3023,7 +3169,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
{
.name = "hyperloglog",
.type = FUNCTION_TYPE_HYPERLOGLOG,
.classification = FUNC_MGT_AGG_FUNC,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_COUNT_LIKE_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateHLL,
.getEnvFunc = getHLLFuncEnv,
.initFunc = functionSetup,
@ -3035,6 +3181,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
#endif
.combineFunc = hllCombine,
.pPartialFunc = "_hyperloglog_partial",
.pStateFunc = "_hyperloglog_state",
.pMergeFunc = "_hyperloglog_merge"
},
{
@ -3064,6 +3211,8 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.invertFunc = NULL,
#endif
.combineFunc = hllCombine,
.pPartialFunc = "_hyperloglog_state_merge",
.pMergeFunc = "_hyperloglog_merge",
},
{
.name = "diff",
@ -3823,7 +3972,153 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
.initFunc = NULL,
.sprocessFunc = qPseudoTagFunction,
.finalizeFunc = NULL
}
},
{
.name = "_stddev_state",
.type = FUNCTION_TYPE_STDDEV_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateStddevState,
.getEnvFunc = getStddevFuncEnv,
.initFunc = stddevFunctionSetup,
.processFunc = stddevFunction,
.finalizeFunc = stddevPartialFinalize,
.pPartialFunc = "_stddev_partial",
.pMergeFunc = "_stddev_state_merge",
},
{
.name = "_stddev_state_merge",
.type = FUNCTION_TYPE_STDDEV_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateStddevStateMerge,
.getEnvFunc = getStddevFuncEnv,
.initFunc = stddevFunctionSetup,
.processFunc = stddevFunctionMerge,
.finalizeFunc = stddevPartialFinalize,
},
{
.name = "_avg_state",
.type = FUNCTION_TYPE_AVG_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateAvgState,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
.processFunc = avgFunction,
.finalizeFunc = avgPartialFinalize,
.pPartialFunc = "_avg_partial",
.pMergeFunc = "_avg_state_merge"
},
{
.name = "_avg_state_merge",
.type = FUNCTION_TYPE_AVG_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateAvgStateMerge,
.getEnvFunc = getAvgFuncEnv,
.initFunc = avgFunctionSetup,
.processFunc = avgFunctionMerge,
.finalizeFunc = avgPartialFinalize,
},
{
.name = "_spread_state",
.type = FUNCTION_TYPE_SPREAD_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSpreadState,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunction,
.finalizeFunc = spreadPartialFinalize,
.pPartialFunc = "_spread_partial",
.pMergeFunc = "_spread_state_merge"
},
{
.name = "_spread_state_merge",
.type = FUNCTION_TYPE_SPREAD_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateSpreadStateMerge,
.getEnvFunc = getSpreadFuncEnv,
.initFunc = spreadFunctionSetup,
.processFunc = spreadFunctionMerge,
.finalizeFunc = spreadPartialFinalize,
},
{
.name = "_first_state",
.type = FUNCTION_TYPE_FIRST_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastState,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = firstFunction,
.finalizeFunc = firstLastPartialFinalize,
.pPartialFunc = "_first_partial",
.pMergeFunc = "_first_state_merge"
},
{
.name = "_first_state_merge",
.type = FUNCTION_TYPE_FIRST_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastStateMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = firstFunctionMerge,
.finalizeFunc = firstLastPartialFinalize,
},
{
.name = "_last_state",
.type = FUNCTION_TYPE_LAST_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastState,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastFunction,
.finalizeFunc = firstLastPartialFinalize,
.pPartialFunc = "_last_partial",
.pMergeFunc = "_last_state_merge"
},
{
.name = "_last_state_merge",
.type = FUNCTION_TYPE_LAST_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SELECT_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_IMPLICIT_TS_FUNC |
FUNC_MGT_KEEP_ORDER_FUNC | FUNC_MGT_FORBID_SYSTABLE_FUNC | FUNC_MGT_TSMA_FUNC | FUNC_MGT_PRIMARY_KEY_FUNC,
.translateFunc = translateFirstLastStateMerge,
.getEnvFunc = getFirstLastFuncEnv,
.initFunc = functionSetup,
.processFunc = lastFunctionMerge,
.finalizeFunc = firstLastPartialFinalize,
},
{
.name = "_hyperloglog_state",
.type = FUNCTION_TYPE_HYPERLOGLOG_STATE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_COUNT_LIKE_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateHLLState,
.getEnvFunc = getHLLFuncEnv,
.initFunc = functionSetup,
.processFunc = hllFunction,
.finalizeFunc = hllPartialFinalize,
.pPartialFunc = "_hyperloglog_partial",
.pMergeFunc = "_hyperloglog_state_merge",
},
{
.name = "_hyperloglog_state_merge",
.type = FUNCTION_TYPE_HYPERLOGLOG_STATE_MERGE,
.classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_COUNT_LIKE_FUNC | FUNC_MGT_TSMA_FUNC,
.translateFunc = translateHLLStateMerge,
.getEnvFunc = getHLLFuncEnv,
.initFunc = functionSetup,
.processFunc = hllFunctionMerge,
.finalizeFunc = hllPartialFinalize,
},
{
.name = "md5",
.type = FUNCTION_TYPE_MD5,
.classification = FUNC_MGT_SCALAR_FUNC,
.translateFunc = translateMd5,
.getEnvFunc = NULL,
.initFunc = NULL,
.sprocessFunc = md5Function,
.finalizeFunc = NULL
},
};
// clang-format on

37
source/libs/function/src/builtinsimpl.c
View File

@ -867,6 +867,12 @@ int32_t sumFunction(SqlFunctionCtx* pCtx) {
}
_sum_over:
if (numOfElem == 0) {
if (tsCountAlwaysReturnValue && pCtx->pExpr->pExpr->_function.pFunctNode->hasOriginalFunc &&
fmIsCountLikeFunc(pCtx->pExpr->pExpr->_function.pFunctNode->originalFuncId)) {
numOfElem = 1;
}
}
// data in the check operation are all null, not output
SET_VAL(GET_RES_INFO(pCtx), numOfElem, 1);
return TSDB_CODE_SUCCESS;
@ -1448,6 +1454,12 @@ static void stddevTransferInfo(SStddevRes* pInput, SStddevRes* pOutput) {
int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -1455,6 +1467,7 @@ int32_t stddevFunctionMerge(SqlFunctionCtx* pCtx) {
SStddevRes* pInfo = GET_ROWCELL_INTERBUF(GET_RES_INFO(pCtx));
for (int32_t i = pInput->startRowIndex; i < pInput->startRowIndex + pInput->numOfRows; ++i) {
if (colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SStddevRes* pInputInfo = (SStddevRes*)varDataVal(data);
stddevTransferInfo(pInputInfo, pInfo);
@ -2846,6 +2859,12 @@ static int32_t firstLastTransferInfo(SqlFunctionCtx* pCtx, SFirstLastRes* pInput
static int32_t firstLastFunctionMergeImpl(SqlFunctionCtx* pCtx, bool isFirstQuery) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -3273,6 +3292,11 @@ int32_t diffFunction(SqlFunctionCtx* pCtx) {
if (pDiffInfo->includeNull) {
colDataSetNull_f_s(pOutput, pos);
// handle selectivity
if (pCtx->subsidiaries.num > 0) {
appendSelectivityCols(pCtx, row.block, row.rowIndex, pos);
}
numOfElems += 1;
}
continue;
@ -3847,6 +3871,12 @@ static void spreadTransferInfo(SSpreadInfo* pInput, SSpreadInfo* pOutput) {
int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -3855,6 +3885,7 @@ int32_t spreadFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
if(colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SSpreadInfo* pInputInfo = (SSpreadInfo*)varDataVal(data);
if (pInputInfo->hasResult) {
@ -4583,6 +4614,11 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_SUCCESS;
}
@ -4592,6 +4628,7 @@ int32_t hllFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
if (colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SHLLInfo* pInputInfo = (SHLLInfo*)varDataVal(data);
hllTransferInfo(pInputInfo, pInfo);

7
source/libs/function/src/detail/tavgfunction.c
View File

@ -705,6 +705,12 @@ static void avgTransferInfo(SAvgRes* pInput, SAvgRes* pOutput) {
int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
SInputColumnInfoData* pInput = &pCtx->input;
SColumnInfoData* pCol = pInput->pData[0];
if (IS_NULL_TYPE(pCol->info.type)) {
SET_VAL(GET_RES_INFO(pCtx), 0, 1);
return TSDB_CODE_SUCCESS;
}
if (pCol->info.type != TSDB_DATA_TYPE_BINARY) {
return TSDB_CODE_FUNC_FUNTION_PARA_TYPE;
}
@ -714,6 +720,7 @@ int32_t avgFunctionMerge(SqlFunctionCtx* pCtx) {
int32_t start = pInput->startRowIndex;
for (int32_t i = start; i < start + pInput->numOfRows; ++i) {
if(colDataIsNull_s(pCol, i)) continue;
char* data = colDataGetData(pCol, i);
SAvgRes* pInputInfo = (SAvgRes*)varDataVal(data);
avgTransferInfo(pInputInfo, pInfo);

129
source/libs/function/src/functionMgt.c
View File

@ -274,6 +274,8 @@ bool fmIsBlockDistFunc(int32_t funcId) {
return FUNCTION_TYPE_BLOCK_DIST == funcMgtBuiltins[funcId].type;
}
bool fmIsIgnoreNullFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_IGNORE_NULL_FUNC); }
void fmFuncMgtDestroy() {
void* m = gFunMgtService.pFuncNameHashTable;
if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) {
@ -411,6 +413,8 @@ static int32_t createPartialFunction(const SFunctionNode* pSrcFunc, SFunctionNod
nodesDestroyList(pParameterList);
return TSDB_CODE_OUT_OF_MEMORY;
}
(*pPartialFunc)->hasOriginalFunc = true;
(*pPartialFunc)->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
char name[TSDB_FUNC_NAME_LEN + TSDB_NAME_DELIMITER_LEN + TSDB_POINTER_PRINT_BYTES + 1] = {0};
int32_t len = snprintf(name, sizeof(name) - 1, "%s.%p", (*pPartialFunc)->functionName, pSrcFunc);
taosCreateMD5Hash(name, len);
@ -473,6 +477,8 @@ static int32_t createMergeFunction(const SFunctionNode* pSrcFunc, const SFunctio
}
}
if (TSDB_CODE_SUCCESS == code) {
pFunc->hasOriginalFunc = true;
pFunc->originalFuncId = pSrcFunc->hasOriginalFunc ? pSrcFunc->originalFuncId : pSrcFunc->funcId;
// overwrite function restype set by translate function
if (fmIsSameInOutType(pSrcFunc->funcId)) {
pFunc->node.resType = pSrcFunc->node.resType;
@ -496,7 +502,7 @@ int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc
}
int32_t code = createPartialFunction(pFunc, pPartialFunc);
if (TSDB_CODE_SUCCESS == code) {
if (TSDB_CODE_SUCCESS == code && pMidFunc) {
code = createMidFunction(pFunc, *pPartialFunc, pMidFunc);
}
if (TSDB_CODE_SUCCESS == code) {
@ -505,7 +511,7 @@ int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc
if (TSDB_CODE_SUCCESS != code) {
nodesDestroyNode((SNode*)*pPartialFunc);
nodesDestroyNode((SNode*)*pMidFunc);
if (pMidFunc) nodesDestroyNode((SNode*)*pMidFunc);
nodesDestroyNode((SNode*)*pMergeFunc);
}
@ -518,3 +524,122 @@ char* fmGetFuncName(int32_t funcId) {
}
return taosStrdup(funcMgtBuiltins[funcId].name);
}
/// @param [out] pStateFunc, not changed if error occured or no need to create state func
/// @retval 0 for succ, otherwise err occured
static int32_t fmCreateStateFunc(const SFunctionNode* pFunc, SFunctionNode** pStateFunc) {
if (funcMgtBuiltins[pFunc->funcId].pStateFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pStateFunc, pParams);
if (!*pStateFunc) {
nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR;
}
strcpy((*pStateFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateFunc)->node.userAlias, pFunc->node.userAlias);
}
return TSDB_CODE_SUCCESS;
}
bool fmIsTSMASupportedFunc(func_id_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_TSMA_FUNC) &&
!isSpecificClassifyFunc(funcId, FUNC_MGT_FORBID_STREAM_FUNC);
}
int32_t fmCreateStateFuncs(SNodeList* pFuncs) {
int32_t code;
SNode* pNode;
char buf[128] = {0};
FOREACH(pNode, pFuncs) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
code = fmGetFuncInfo(pFunc, buf, 128);
if (code) break;
if (fmIsTSMASupportedFunc(pFunc->funcId)) {
SFunctionNode* pNewFunc = NULL;
code = fmCreateStateFunc(pFunc, &pNewFunc);
if (code) {
// error
break;
} else if (!pNewFunc) {
// no need state func
continue;
} else {
REPLACE_NODE(pNewFunc);
nodesDestroyNode(pNode);
}
}
}
return code;
}
static int32_t fmCreateStateMergeFunc(SFunctionNode* pFunc, SFunctionNode** pStateMergeFunc) {
if (funcMgtBuiltins[pFunc->funcId].pMergeFunc) {
SNodeList* pParams = nodesCloneList(pFunc->pParameterList);
if (!pParams) return TSDB_CODE_OUT_OF_MEMORY;
*pStateMergeFunc = createFunction(funcMgtBuiltins[pFunc->funcId].pMergeFunc, pParams);
if (!*pStateMergeFunc) {
nodesDestroyList(pParams);
return TSDB_CODE_FUNC_FUNTION_ERROR;
}
strcpy((*pStateMergeFunc)->node.aliasName, pFunc->node.aliasName);
strcpy((*pStateMergeFunc)->node.userAlias, pFunc->node.userAlias);
}
return TSDB_CODE_SUCCESS;
}
int32_t fmCreateStateMergeFuncs(SNodeList* pFuncs) {
int32_t code;
SNode* pNode;
char buf[128] = {0};
FOREACH(pNode, pFuncs) {
SFunctionNode* pFunc = (SFunctionNode*)pNode;
if (fmIsTSMASupportedFunc(pFunc->funcId)) {
SFunctionNode* pNewFunc = NULL;
code = fmCreateStateMergeFunc(pFunc, &pNewFunc);
if (code) {
// error
break;
} else if (!pNewFunc) {
// no state merge func
continue;
} else {
REPLACE_NODE(pNewFunc);
nodesDestroyNode(pNode);
}
}
}
return code;
}
int32_t fmGetFuncId(const char* name) {
if (NULL != gFunMgtService.pFuncNameHashTable) {
void* pVal = taosHashGet(gFunMgtService.pFuncNameHashTable, name, strlen(name));
if (NULL != pVal) {
return *(int32_t*)pVal;
}
return -1;
}
for (int32_t i = 0; i < funcMgtBuiltinsNum; ++i) {
if (0 == strcmp(funcMgtBuiltins[i].name, name)) {
return i;
}
}
return -1;
}
bool fmIsMyStateFunc(int32_t funcId, int32_t stateFuncId) {
const SBuiltinFuncDefinition* pFunc = &funcMgtBuiltins[funcId];
const SBuiltinFuncDefinition* pStateFunc = &funcMgtBuiltins[stateFuncId];
if (!pFunc->pStateFunc) {
return false;
}
if (strcmp(pFunc->pStateFunc, pStateFunc->name) == 0) return true;
int32_t stateMergeFuncId = fmGetFuncId(pFunc->pStateFunc);
const SBuiltinFuncDefinition* pStateMergeFunc = &funcMgtBuiltins[stateMergeFuncId];
return strcmp(pStateFunc->name, pStateMergeFunc->pMergeFunc) == 0;
}
bool fmIsCountLikeFunc(int32_t funcId) {
return isSpecificClassifyFunc(funcId, FUNC_MGT_COUNT_LIKE_FUNC);
}

39
source/libs/nodes/src/nodesCloneFuncs.c
View File

@ -114,6 +114,7 @@ static int32_t columnNodeCopy(const SColumnNode* pSrc, SColumnNode* pDst) {
COPY_SCALAR_FIELD(projIdx);
COPY_SCALAR_FIELD(colType);
COPY_SCALAR_FIELD(hasIndex);
COPY_SCALAR_FIELD(isPrimTs);
COPY_CHAR_ARRAY_FIELD(dbName);
COPY_CHAR_ARRAY_FIELD(tableName);
COPY_CHAR_ARRAY_FIELD(tableAlias);
@ -138,7 +139,7 @@ static int32_t columnDefNodeCopy(const SColumnDefNode* pSrc, SColumnDefNode* pDs
static int32_t valueNodeCopy(const SValueNode* pSrc, SValueNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, exprNodeCopy);
COPY_CHAR_POINT_FIELD(literal);
COPY_SCALAR_FIELD(isDuration);
COPY_SCALAR_FIELD(flag);
COPY_SCALAR_FIELD(translate);
COPY_SCALAR_FIELD(notReserved);
COPY_SCALAR_FIELD(isNull);
@ -223,6 +224,8 @@ static int32_t functionNodeCopy(const SFunctionNode* pSrc, SFunctionNode* pDst)
COPY_SCALAR_FIELD(udfBufSize);
COPY_SCALAR_FIELD(hasPk);
COPY_SCALAR_FIELD(pkBytes);
COPY_SCALAR_FIELD(hasOriginalFunc);
COPY_SCALAR_FIELD(originalFuncId);
return TSDB_CODE_SUCCESS;
}
@ -297,6 +300,7 @@ static int32_t tempTableNodeCopy(const STempTableNode* pSrc, STempTableNode* pDs
static int32_t joinTableNodeCopy(const SJoinTableNode* pSrc, SJoinTableNode* pDst) {
COPY_BASE_OBJECT_FIELD(table, tableNodeCopy);
COPY_SCALAR_FIELD(joinType);
COPY_SCALAR_FIELD(subType);
COPY_SCALAR_FIELD(hasSubQuery);
COPY_SCALAR_FIELD(isLowLevelJoin);
CLONE_NODE_FIELD(pLeft);
@ -413,6 +417,14 @@ static int32_t hintNodeCopy(const SHintNode* pSrc, SHintNode* pDst) {
return copyHintValue(pSrc, pDst);
}
static int32_t windowOffsetCopy(const SWindowOffsetNode* pSrc, SWindowOffsetNode* pDst) {
COPY_SCALAR_FIELD(type);
CLONE_NODE_FIELD(pStartOffset);
CLONE_NODE_FIELD(pEndOffset);
return TSDB_CODE_SUCCESS;
}
static int32_t logicNodeCopy(const SLogicNode* pSrc, SLogicNode* pDst) {
CLONE_NODE_LIST_FIELD(pTargets);
CLONE_NODE_FIELD(pConditions);
@ -473,20 +485,37 @@ static int32_t logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) {
CLONE_OBJECT_FIELD(pFuncTypes, functParamClone);
COPY_SCALAR_FIELD(paraTablesSort);
COPY_SCALAR_FIELD(smallDataTsSort);
COPY_SCALAR_FIELD(needSplit);
return TSDB_CODE_SUCCESS;
}
static int32_t logicJoinCopy(const SJoinLogicNode* pSrc, SJoinLogicNode* pDst) {
COPY_BASE_OBJECT_FIELD(node, logicNodeCopy);
COPY_SCALAR_FIELD(joinType);
COPY_SCALAR_FIELD(subType);
COPY_SCALAR_FIELD(joinAlgo);
CLONE_NODE_FIELD(pWindowOffset);
CLONE_NODE_FIELD(pJLimit);
CLONE_NODE_FIELD(addPrimEqCond);
CLONE_NODE_FIELD(pPrimKeyEqCond);
CLONE_NODE_FIELD(pColEqCond);
CLONE_NODE_FIELD(pColOnCond);
CLONE_NODE_FIELD(pTagEqCond);
CLONE_NODE_FIELD(pTagOnCond);
CLONE_NODE_FIELD(pOtherOnCond);
CLONE_NODE_FIELD(pFullOnCond);
CLONE_NODE_LIST_FIELD(pLeftEqNodes);
CLONE_NODE_LIST_FIELD(pRightEqNodes);
COPY_SCALAR_FIELD(allEqTags);
COPY_SCALAR_FIELD(isSingleTableJoin);
COPY_SCALAR_FIELD(hasSubQuery);
COPY_SCALAR_FIELD(isLowLevelJoin);
COPY_SCALAR_FIELD(seqWinGroup);
COPY_SCALAR_FIELD(grpJoin);
COPY_SCALAR_FIELD(hashJoinHint);
CLONE_NODE_FIELD(pLeftOnCond);
CLONE_NODE_FIELD(pRightOnCond);
COPY_SCALAR_FIELD(timeRangeTarget);
COPY_OBJECT_FIELD(timeRange, sizeof(STimeWindow));
return TSDB_CODE_SUCCESS;
}
@ -546,6 +575,8 @@ static int32_t logicMergeCopy(const SMergeLogicNode* pSrc, SMergeLogicNode* pDst
CLONE_NODE_LIST_FIELD(pInputs);
COPY_SCALAR_FIELD(numOfChannels);
COPY_SCALAR_FIELD(srcGroupId);
COPY_SCALAR_FIELD(srcEndGroupId);
COPY_SCALAR_FIELD(numOfSubplans);
COPY_SCALAR_FIELD(colsMerge);
COPY_SCALAR_FIELD(needSort);
COPY_SCALAR_FIELD(groupSort);
@ -811,6 +842,7 @@ static int32_t selectStmtCopy(const SSelectStmt* pSrc, SSelectStmt* pDst) {
COPY_SCALAR_FIELD(precision);
COPY_SCALAR_FIELD(isEmptyResult);
COPY_SCALAR_FIELD(timeLineResMode);
COPY_SCALAR_FIELD(timeLineCurMode);
COPY_SCALAR_FIELD(hasAggFuncs);
COPY_SCALAR_FIELD(hasRepeatScanFuncs);
CLONE_NODE_LIST_FIELD(pHint);
@ -924,6 +956,9 @@ SNode* nodesCloneNode(const SNode* pNode) {
case QUERY_NODE_HINT:
code = hintNodeCopy((const SHintNode*)pNode, (SHintNode*)pDst);
break;
case QUERY_NODE_WINDOW_OFFSET:
code = windowOffsetCopy((const SWindowOffsetNode*)pNode, (SWindowOffsetNode*)pDst);
break;
case QUERY_NODE_SET_OPERATOR:
code = setOperatorCopy((const SSetOperator*)pNode, (SSetOperator*)pDst);
break;

379
source/libs/nodes/src/nodesCodeFuncs.c
View File

@ -91,6 +91,8 @@ const char* nodesNodeName(ENodeType type) {
return "CaseWhen";
case QUERY_NODE_EVENT_WINDOW:
return "EventWindow";
case QUERY_NODE_WINDOW_OFFSET:
return "WindowOffset";
case QUERY_NODE_COUNT_WINDOW:
return "CountWindow";
case QUERY_NODE_SET_OPERATOR:
@ -1621,7 +1623,7 @@ static int32_t logicJoinNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddObject(pJson, jkJoinLogicPlanTagEqCondition, nodeToJson, pNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pOtherOnCond);
code = tjsonAddObject(pJson, jkJoinLogicPlanOnConditions, nodeToJson, pNode->pFullOnCond);
}
return code;
}
@ -1646,7 +1648,7 @@ static int32_t jsonToLogicJoinNode(const SJson* pJson, void* pObj) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanTagEqCondition, &pNode->pTagEqCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pOtherOnCond);
code = jsonToNodeObject(pJson, jkJoinLogicPlanOnConditions, &pNode->pFullOnCond);
}
return code;
@ -2166,15 +2168,34 @@ static int32_t jsonToPhysiProjectNode(const SJson* pJson, void* pObj) {
}
static const char* jkJoinPhysiPlanJoinType = "JoinType";
static const char* jkJoinPhysiPlanSubType = "SubType";
static const char* jkJoinPhysiPlanWinOffset = "WindowOffset";
static const char* jkJoinPhysiPlanJoinLimit = "JoinLimit";
static const char* jkJoinPhysiPlanAsofOp = "AsofOp";
static const char* jkJoinPhysiPlanLeftPrimExpr = "LeftPrimExpr";
static const char* jkJoinPhysiPlanRightPrimExpr = "RightPrimExpr";
static const char* jkJoinPhysiPlanLeftPrimSlotId = "LeftPrimSlotId";
static const char* jkJoinPhysiPlanRightPrimSlotId = "RightPrimSlotId";
static const char* jkJoinPhysiPlanLeftEqCols = "LeftEqCols";
static const char* jkJoinPhysiPlanRightEqCols = "RightEqCols";
static const char* jkJoinPhysiPlanInputTsOrder = "InputTsOrder";
static const char* jkJoinPhysiPlanOnLeftCols = "OnLeftColumns";
static const char* jkJoinPhysiPlanOnRightCols = "OnRightColumns";
static const char* jkJoinPhysiPlanPrimKeyCondition = "PrimKeyCondition";
static const char* jkJoinPhysiPlanOnConditions = "OnConditions";
static const char* jkJoinPhysiPlanTargets = "Targets";
static const char* jkJoinPhysiPlanColEqualOnConditions = "ColumnEqualOnConditions";
static const char* jkJoinPhysiPlanInputRowNum = "InputRowNum";
static const char* jkJoinPhysiPlanInputRowSize = "InputRowSize";
static const char* jkJoinPhysiPlanColOnConditions = "ColumnOnConditions";
static const char* jkJoinPhysiPlanLeftInputRowNum = "LeftInputRowNum";
static const char* jkJoinPhysiPlanRightInputRowNum = "RightInputRowNum";
static const char* jkJoinPhysiPlanLeftInputRowSize = "LeftInputRowSize";
static const char* jkJoinPhysiPlanRightInputRowSize = "RightInputRowSize";
static const char* jkJoinPhysiPlanSeqWinGroup = "SeqWinGroup";
static const char* jkJoinPhysiPlanGroupJoin = "GroupJoin";
static const char* jkJoinPhysiPlanLeftOnCond = "LeftOnCond";
static const char* jkJoinPhysiPlanRightOnCond = "RightOnCond";
static const char* jkJoinPhysiPlanTimeRangeSKey = "TimeRangeSKey";
static const char* jkJoinPhysiPlanTimeRangeEKey = "TimeRangeEKey";
static const char* jkJoinPhysiPlanTimeRangeTarget = "TimeRangeTarget";
static int32_t physiMergeJoinNodeToJson(const void* pObj, SJson* pJson) {
const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj;
@ -2184,17 +2205,63 @@ static int32_t physiMergeJoinNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanPrimKeyCondition, nodeToJson, pNode->pPrimKeyCond);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanSubType, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pOtherOnCond);
code = tjsonAddObject(pJson, jkJoinPhysiPlanWinOffset, nodeToJson, pNode->pWindowOffset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanJoinLimit, nodeToJson, pNode->pJLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanAsofOp, pNode->asofOpType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanLeftPrimExpr, nodeToJson, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanRightPrimExpr, nodeToJson, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanLeftEqCols, pNode->pEqLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanRightEqCols, pNode->pEqRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanColEqualOnConditions, nodeToJson, pNode->pColEqCond);
code = tjsonAddObject(pJson, jkJoinPhysiPlanColOnConditions, nodeToJson, pNode->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkJoinPhysiPlanSeqWinGroup, pNode->seqWinGroup);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkJoinPhysiPlanGroupJoin, pNode->grpJoin);
}
return code;
}
@ -2206,17 +2273,63 @@ static int32_t jsonToPhysiMergeJoinNode(const SJson* pJson, void* pObj) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pOtherOnCond);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanSubType, pNode->subType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanPrimKeyCondition, &pNode->pPrimKeyCond);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanWinOffset, &pNode->pWindowOffset);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanJoinLimit, &pNode->pJLimit);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanAsofOp, pNode->asofOpType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanLeftPrimExpr, &pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanRightPrimExpr, &pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanLeftEqCols, &pNode->pEqLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanRightEqCols, &pNode->pEqRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColEqualOnConditions, &pNode->pColEqCond);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanColOnConditions, &pNode->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkJoinPhysiPlanSeqWinGroup, &pNode->seqWinGroup);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkJoinPhysiPlanGroupJoin, &pNode->grpJoin);
}
return code;
}
@ -2227,6 +2340,9 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanSubType, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanOnLeftCols, pNode->pOnLeft);
}
@ -2234,23 +2350,51 @@ static int32_t physiHashJoinNodeToJson(const void* pObj, SJson* pJson) {
code = nodeListToJson(pJson, jkJoinPhysiPlanOnRightCols, pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFilterConditions);
code = tjsonAddObject(pJson, jkJoinPhysiPlanLeftPrimExpr, nodeToJson, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanRightPrimExpr, nodeToJson, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanOnConditions, nodeToJson, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = nodeListToJson(pJson, jkJoinPhysiPlanTargets, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[0].inputRowNum);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[0].inputRowSize);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[1].inputRowNum);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[1].inputRowSize);
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanLeftOnCond, nodeToJson, pNode->pLeftOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinPhysiPlanRightOnCond, nodeToJson, pNode->pRightOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanTimeRangeTarget, pNode->timeRangeTarget);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanTimeRangeSKey, pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinPhysiPlanTimeRangeEKey, pNode->timeRange.ekey);
}
return code;
}
@ -2261,6 +2405,9 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanSubType, pNode->subType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanOnLeftCols, &pNode->pOnLeft);
}
@ -2268,23 +2415,51 @@ static int32_t jsonToPhysiHashJoinNode(const SJson* pJson, void* pObj) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanOnRightCols, &pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFilterConditions);
code = jsonToNodeObject(pJson, jkJoinPhysiPlanLeftPrimExpr, &pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanRightPrimExpr, &pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftPrimSlotId, pNode->leftPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightPrimSlotId, pNode->rightPrimSlotId, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanOnConditions, &pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkJoinPhysiPlanTargets, &pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[0].inputRowNum, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowNum, pNode->inputStat[0].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[0].inputRowSize, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanLeftInputRowSize, pNode->inputStat[0].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowNum, pNode->inputStat[1].inputRowNum, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowNum, pNode->inputStat[1].inputRowNum, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanInputRowSize, pNode->inputStat[1].inputRowSize, code);
tjsonGetNumberValue(pJson, jkJoinPhysiPlanRightInputRowSize, pNode->inputStat[1].inputRowSize, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanLeftOnCond, &pNode->pLeftOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinPhysiPlanRightOnCond, &pNode->pRightOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinPhysiPlanTimeRangeTarget, pNode->timeRangeTarget, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkJoinPhysiPlanTimeRangeSKey, &pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBigIntValue(pJson, jkJoinPhysiPlanTimeRangeEKey, &pNode->timeRange.ekey);
}
return code;
}
@ -2293,6 +2468,7 @@ static const char* jkAggPhysiPlanGroupKeys = "GroupKeys";
static const char* jkAggPhysiPlanAggFuncs = "AggFuncs";
static const char* jkAggPhysiPlanMergeDataBlock = "MergeDataBlock";
static const char* jkAggPhysiPlanGroupKeyOptimized = "GroupKeyOptimized";
static const char* jkAggPhysiPlanHasCountLikeFunc = "HasCountFunc";
static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj;
@ -2313,6 +2489,9 @@ static int32_t physiAggNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanGroupKeyOptimized, pNode->groupKeyOptimized);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkAggPhysiPlanHasCountLikeFunc, pNode->hasCountLikeFunc);
}
return code;
}
@ -2336,6 +2515,9 @@ static int32_t jsonToPhysiAggNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanGroupKeyOptimized, &pNode->groupKeyOptimized);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkAggPhysiPlanHasCountLikeFunc, &pNode->hasCountLikeFunc);
}
return code;
}
@ -3728,7 +3910,7 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
static const char* jkValueLiteralSize = "LiteralSize";
static const char* jkValueLiteral = "Literal";
static const char* jkValueDuration = "Duration";
static const char* jkValueFlag = "Flag";
static const char* jkValueTranslate = "Translate";
static const char* jkValueNotReserved = "NotReserved";
static const char* jkValueIsNull = "IsNull";
@ -3812,7 +3994,7 @@ static int32_t valueNodeToJson(const void* pObj, SJson* pJson) {
code = tjsonAddStringToObject(pJson, jkValueLiteral, pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkValueDuration, pNode->isDuration);
code = tjsonAddBoolToObject(pJson, jkValueFlag, pNode->flag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkValueTranslate, pNode->translate);
@ -3966,7 +4148,7 @@ static int32_t jsonToValueNode(const SJson* pJson, void* pObj) {
code = tjsonDupStringValue(pJson, jkValueLiteral, &pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkValueDuration, &pNode->isDuration);
code = tjsonGetIntValue(pJson, jkValueFlag, &pNode->flag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkValueTranslate, &pNode->translate);
@ -4063,6 +4245,8 @@ static const char* jkFunctionParameter = "Parameters";
static const char* jkFunctionUdfBufSize = "UdfBufSize";
static const char* jkFunctionHasPk = "HasPk";
static const char* jkFunctionPkBytes = "PkBytes";
static const char* jkFunctionIsMergeFunc = "IsMergeFunc";
static const char* jkFunctionMergeFuncOf = "MergeFuncOf";
static int32_t functionNodeToJson(const void* pObj, SJson* pJson) {
const SFunctionNode* pNode = (const SFunctionNode*)pObj;
@ -4089,6 +4273,13 @@ static int32_t functionNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkFunctionPkBytes, pNode->pkBytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddBoolToObject(pJson, jkFunctionIsMergeFunc, pNode->hasOriginalFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkFunctionMergeFuncOf, pNode->originalFuncId);
}
return code;
}
@ -4117,6 +4308,13 @@ static int32_t jsonToFunctionNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkFunctionPkBytes, &pNode->pkBytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetBoolValue(pJson, jkFunctionIsMergeFunc, &pNode->hasOriginalFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetIntValue(pJson, jkFunctionMergeFuncOf, &pNode->originalFuncId);
}
return code;
}
@ -4311,6 +4509,7 @@ static int32_t jsonToTempTableNode(const SJson* pJson, void* pObj) {
}
static const char* jkJoinTableJoinType = "JoinType";
static const char* jkJoinTableSubType = "SubType";
static const char* jkJoinTableLeft = "Left";
static const char* jkJoinTableRight = "Right";
static const char* jkJoinTableOnCond = "OnCond";
@ -4322,6 +4521,9 @@ static int32_t joinTableNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinTableJoinType, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkJoinTableSubType, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkJoinTableLeft, nodeToJson, pNode->pLeft);
}
@ -4342,6 +4544,9 @@ static int32_t jsonToJoinTableNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinTableJoinType, pNode->joinType, code);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkJoinTableSubType, pNode->subType, code);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkJoinTableLeft, &pNode->pLeft);
}
@ -4830,6 +5035,29 @@ static int32_t jsonToDownstreamSourceNode(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkWindowOffsetStartOffset = "StartOffset";
static const char* jkWindowOffsetEndOffset = "EndOffset";
static int32_t windowOffsetNodeToJson(const void* pObj, SJson* pJson) {
const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj;
int32_t code = tjsonAddObject(pJson, jkWindowOffsetStartOffset, nodeToJson, pNode->pStartOffset);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkWindowOffsetEndOffset, nodeToJson, pNode->pEndOffset);
}
return code;
}
static int32_t jsonToWindowOffsetNode(const SJson* pJson, void* pObj) {
SWindowOffsetNode* pNode = (SWindowOffsetNode*)pObj;
int32_t code = jsonToNodeObject(pJson, jkWindowOffsetStartOffset, &pNode->pStartOffset);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkWindowOffsetEndOffset, &pNode->pEndOffset);
}
return code;
}
static const char* jkDatabaseOptionsBuffer = "Buffer";
static const char* jkDatabaseOptionsCacheModel = "CacheModel";
static const char* jkDatabaseOptionsCompressionLevel = "CompressionLevel";
@ -7106,6 +7334,105 @@ static int32_t jsonToInsertStmt(const SJson* pJson, void* pObj) {
return code;
}
static const char* jkTSMAOptionFuncs = "Funcs";
static const char* jkTSMAOptionCols = "Cols";
static const char* jkTSMAOptionInterval = "Interval";
static const char* jkTSMAOptionTsPrecision = "Precision";
static int32_t tsmaOptionToJson(const void* pObj, SJson* pJson) {
const STSMAOptions* pNode = (const STSMAOptions*)pObj;
int32_t code = nodeListToJson(pJson, jkTSMAOptionFuncs, pNode->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkTSMAOptionInterval, nodeToJson, pNode->pInterval);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddIntegerToObject(pJson, jkTSMAOptionTsPrecision, pNode->tsPrecision);
}
return code;
}
static int32_t jsonToTSMAOption(const SJson* pJson, void* pObj) {
STSMAOptions* pNode = (STSMAOptions*)pObj;
int32_t code = jsonToNodeList(pJson, jkTSMAOptionFuncs, &pNode->pFuncs);
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkTSMAOptionInterval, &pNode->pInterval);
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetUTinyIntValue(pJson, jkTSMAOptionTsPrecision, &pNode->tsPrecision);
}
return code;
}
static const char* jkCreateTSMAStmtIgnoreExists = "IgnoreExists";
static const char* jkCreateTSMAStmtTsmaName = "TSMAName";
static const char* jkCreateTSMAStmtDbName = "DbName";
static const char* jkCreateTSMAStmtTableName = "TableName";
static const char* jkCreateTSMAStmtpOptions = "Options";
static int32_t createTSMAStmtToJson(const void* pObj, SJson* pJson) {
const SCreateTSMAStmt* pNode = (const SCreateTSMAStmt*)pObj;
int32_t code = tjsonAddBoolToObject(pJson, jkCreateTSMAStmtIgnoreExists, pNode->ignoreExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTSMAStmtTsmaName, pNode->tsmaName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkCreateTSMAStmtTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkCreateTSMAStmtpOptions, nodeToJson, pNode->pOptions);
}
return code;
}
static int32_t jsonToCreateTSMAStmt(const SJson* pJson, void* pObj) {
SCreateTSMAStmt* pNode = (SCreateTSMAStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkCreateTSMAStmtIgnoreExists, &pNode->ignoreExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTSMAStmtTsmaName, pNode->tsmaName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkCreateTSMAStmtTableName, pNode->tableName);
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkCreateTSMAStmtpOptions, (SNode**)&pNode->pOptions);
}
return code;
}
static const char* jkDropTSMAStmtIgnoreNotExists = "IgnoreNotExists";
static const char* jkDropTSMAStmtDbName = "DbName";
static const char* jkDropTSMAStmtTsmaName = "TSMAName";
static int32_t dropTSMAStmtToJson(const void* pObj, SJson* pJson) {
const SDropTSMAStmt* pNode = (const SDropTSMAStmt*)pObj;
int32_t code = tjsonAddBoolToObject(pJson, jkDropTSMAStmtIgnoreNotExists, pNode->ignoreNotExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkDropTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddStringToObject(pJson, jkDropTSMAStmtTsmaName, pNode->tsmaName);
}
return code;
}
static int32_t jsonToDropTSMAStmt(const SJson* pJson, void* pObj) {
SDropTSMAStmt* pNode = (SDropTSMAStmt*)pObj;
int32_t code = tjsonGetBoolValue(pJson, jkDropTSMAStmtIgnoreNotExists, &pNode->ignoreNotExists);
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkDropTSMAStmtDbName, pNode->dbName);
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkDropTSMAStmtTsmaName, pNode->tsmaName);
}
return code;
}
static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
switch (nodeType(pObj)) {
case QUERY_NODE_COLUMN:
@ -7170,6 +7497,8 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) {
return caseWhenNodeToJson(pObj, pJson);
case QUERY_NODE_EVENT_WINDOW:
return eventWindowNodeToJson(pObj, pJson);
case QUERY_NODE_WINDOW_OFFSET:
return windowOffsetNodeToJson(pObj, pJson);
case QUERY_NODE_COUNT_WINDOW:
return countWindowNodeToJson(pObj, pJson);
case QUERY_NODE_SET_OPERATOR:
@ -7511,6 +7840,8 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) {
return jsonToCaseWhenNode(pJson, pObj);
case QUERY_NODE_EVENT_WINDOW:
return jsonToEventWindowNode(pJson, pObj);
case QUERY_NODE_WINDOW_OFFSET:
return jsonToWindowOffsetNode(pJson, pObj);
case QUERY_NODE_COUNT_WINDOW:
return jsonToCountWindowNode(pJson, pObj);
case QUERY_NODE_SET_OPERATOR:

284
source/libs/nodes/src/nodesMsgFuncs.c
View File

@ -808,7 +808,7 @@ static int32_t msgToColumnNode(STlvDecoder* pDecoder, void* pObj) {
enum {
VALUE_CODE_EXPR_BASE = 1,
VALUE_CODE_LITERAL,
VALUE_CODE_IS_DURATION,
VALUE_CODE_FLAG,
VALUE_CODE_TRANSLATE,
VALUE_CODE_NOT_RESERVED,
VALUE_CODE_IS_NULL,
@ -869,7 +869,7 @@ static int32_t valueNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = tlvEncodeCStr(pEncoder, VALUE_CODE_LITERAL, pNode->literal);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, VALUE_CODE_IS_DURATION, pNode->isDuration);
code = tlvEncodeI32(pEncoder, VALUE_CODE_FLAG, pNode->flag);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, VALUE_CODE_TRANSLATE, pNode->translate);
@ -992,8 +992,8 @@ static int32_t msgToValueNode(STlvDecoder* pDecoder, void* pObj) {
case VALUE_CODE_LITERAL:
code = tlvDecodeCStrP(pTlv, &pNode->literal);
break;
case VALUE_CODE_IS_DURATION:
code = tlvDecodeBool(pTlv, &pNode->isDuration);
case VALUE_CODE_FLAG:
code = tlvDecodeI32(pTlv, &pNode->flag);
break;
case VALUE_CODE_TRANSLATE:
code = tlvDecodeBool(pTlv, &pNode->translate);
@ -1109,7 +1109,9 @@ enum {
FUNCTION_CODE_PARAMETERS,
FUNCTION_CODE_UDF_BUF_SIZE,
FUNCTION_NODE_HAS_PK,
FUNCTION_NODE_PK_BYTES
FUNCTION_NODE_PK_BYTES,
FUNCTION_CODE_IS_MERGE_FUNC,
FUNCTION_CODE_MERGE_FUNC_OF,
};
static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -1137,6 +1139,13 @@ static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, FUNCTION_NODE_PK_BYTES, pNode->pkBytes);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, FUNCTION_CODE_IS_MERGE_FUNC, pNode->hasOriginalFunc);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, FUNCTION_CODE_MERGE_FUNC_OF, pNode->originalFuncId);
}
return code;
}
@ -1171,6 +1180,12 @@ static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) {
case FUNCTION_NODE_PK_BYTES:
code = tlvDecodeI32(pTlv, &pNode->pkBytes);
break;
case FUNCTION_CODE_IS_MERGE_FUNC:
code = tlvDecodeBool(pTlv, &pNode->hasOriginalFunc);
break;
case FUNCTION_CODE_MERGE_FUNC_OF:
code = tlvDecodeI32(pTlv, &pNode->originalFuncId);
break;
default:
break;
}
@ -1879,6 +1894,41 @@ static int32_t msgToCaseWhenNode(STlvDecoder* pDecoder, void* pObj) {
return code;
}
enum { WINDOW_OFFSET_CODE_START_OFFSET = 1, WINDOW_OFFSET_CODE_END_OFFSET };
static int32_t windowOffsetNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
const SWindowOffsetNode* pNode = (const SWindowOffsetNode*)pObj;
int32_t code = tlvEncodeObj(pEncoder, WINDOW_OFFSET_CODE_START_OFFSET, nodeToMsg, pNode->pStartOffset);
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, WINDOW_OFFSET_CODE_END_OFFSET, nodeToMsg, pNode->pEndOffset);
}
return code;
}
static int32_t msgToWindowOffsetNode(STlvDecoder* pDecoder, void* pObj) {
SWindowOffsetNode* pNode = (SWindowOffsetNode*)pObj;
int32_t code = TSDB_CODE_SUCCESS;
STlv* pTlv = NULL;
tlvForEach(pDecoder, pTlv, code) {
switch (pTlv->type) {
case WINDOW_OFFSET_CODE_START_OFFSET:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartOffset);
break;
case WINDOW_OFFSET_CODE_END_OFFSET:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndOffset);
break;
default:
break;
}
}
return code;
}
enum {
PHY_NODE_CODE_OUTPUT_DESC = 1,
PHY_NODE_CODE_CONDITIONS,
@ -2465,11 +2515,25 @@ static int32_t msgToPhysiProjectNode(STlvDecoder* pDecoder, void* pObj) {
enum {
PHY_SORT_MERGE_JOIN_CODE_BASE_NODE = 1,
PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE,
PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION,
PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_SUB_TYPE,
PHY_SORT_MERGE_JOIN_CODE_WINDOW_OFFSET,
PHY_SORT_MERGE_JOIN_CODE_JOIN_LIMIT,
PHY_SORT_MERGE_JOIN_CODE_ASOF_OP,
PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_EXPR,
PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_EXPR,
PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_SLOT_ID,
PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_SLOT_ID,
PHY_SORT_MERGE_JOIN_CODE_LEFT_EQ_COLS,
PHY_SORT_MERGE_JOIN_CODE_RIGHT_EQ_COLS,
PHY_SORT_MERGE_JOIN_CODE_FULL_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_TARGETS,
PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER,
PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS
PHY_SORT_MERGE_JOIN_CODE_COL_ON_CONDITIONS,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM0,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE0,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM1,
PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE1,
PHY_SORT_MERGE_JOIN_CODE_SEQ_WIN_GROUP,
PHY_SORT_MERGE_JOIN_CODE_GROUP_JOIN
};
static int32_t physiMergeJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2480,17 +2544,63 @@ static int32_t physiMergeJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder)
code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION, nodeToMsg, pNode->pPrimKeyCond);
code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_SUB_TYPE, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pOtherOnCond);
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_WINDOW_OFFSET, nodeToMsg, pNode->pWindowOffset);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_LIMIT, nodeToMsg, pNode->pJLimit);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ASOF_OP, pNode->asofOpType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_EXPR, nodeToMsg, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_EXPR, nodeToMsg, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_SLOT_ID, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_SLOT_ID, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_LEFT_EQ_COLS, nodeListToMsg, pNode->pEqLeft);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_RIGHT_EQ_COLS, nodeListToMsg, pNode->pEqRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_COL_ON_CONDITIONS, nodeToMsg, pNode->pColOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_FULL_ON_CONDITIONS, nodeToMsg, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS, nodeToMsg, pNode->pColEqCond);
code = tlvEncodeI64(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM0, pNode->inputStat[0].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM1, pNode->inputStat[1].inputRowNum);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE0, pNode->inputStat[0].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE1, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_SORT_MERGE_JOIN_CODE_SEQ_WIN_GROUP, pNode->seqWinGroup);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_SORT_MERGE_JOIN_CODE_GROUP_JOIN, pNode->grpJoin);
}
return code;
}
@ -2507,17 +2617,62 @@ static int32_t msgToPhysiMergeJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType));
break;
case PHY_SORT_MERGE_JOIN_CODE_PRIM_KEY_CONDITION:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pPrimKeyCond);
case PHY_SORT_MERGE_JOIN_CODE_SUB_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->subType, sizeof(pNode->subType));
break;
case PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOtherOnCond);
case PHY_SORT_MERGE_JOIN_CODE_WINDOW_OFFSET:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pWindowOffset);
break;
case PHY_SORT_MERGE_JOIN_CODE_JOIN_LIMIT:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pJLimit);
break;
case PHY_SORT_MERGE_JOIN_CODE_ASOF_OP:
code = tlvDecodeI32(pTlv, &pNode->asofOpType);
break;
case PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->leftPrimExpr);
break;
case PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->rightPrimExpr);
break;
case PHY_SORT_MERGE_JOIN_CODE_LEFT_PRIM_SLOT_ID:
code = tlvDecodeI32(pTlv, &pNode->leftPrimSlotId);
break;
case PHY_SORT_MERGE_JOIN_CODE_RIGHT_PRIM_SLOT_ID:
code = tlvDecodeI32(pTlv, &pNode->rightPrimSlotId);
break;
case PHY_SORT_MERGE_JOIN_CODE_LEFT_EQ_COLS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pEqLeft);
break;
case PHY_SORT_MERGE_JOIN_CODE_RIGHT_EQ_COLS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pEqRight);
break;
case PHY_SORT_MERGE_JOIN_CODE_COL_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColOnCond);
break;
case PHY_SORT_MERGE_JOIN_CODE_FULL_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFullOnCond);
break;
case PHY_SORT_MERGE_JOIN_CODE_TARGETS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
break;
case PHY_SORT_MERGE_JOIN_CODE_TAG_EQUAL_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pColEqCond);
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM0:
code = tlvDecodeI64(pTlv, &pNode->inputStat[0].inputRowNum);
break;
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_NUM1:
code = tlvDecodeI64(pTlv, &pNode->inputStat[1].inputRowNum);
break;
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE0:
code = tlvDecodeI32(pTlv, &pNode->inputStat[0].inputRowSize);
break;
case PHY_SORT_MERGE_JOIN_CODE_INPUT_ROW_SIZE1:
code = tlvDecodeI32(pTlv, &pNode->inputStat[1].inputRowSize);
break;
case PHY_SORT_MERGE_JOIN_CODE_SEQ_WIN_GROUP:
code = tlvDecodeBool(pTlv, &pNode->seqWinGroup);
break;
case PHY_SORT_MERGE_JOIN_CODE_GROUP_JOIN:
code = tlvDecodeBool(pTlv, &pNode->grpJoin);
break;
default:
break;
@ -2530,14 +2685,25 @@ static int32_t msgToPhysiMergeJoinNode(STlvDecoder* pDecoder, void* pObj) {
enum {
PHY_HASH_JOIN_CODE_BASE_NODE = 1,
PHY_HASH_JOIN_CODE_JOIN_TYPE,
PHY_HASH_JOIN_CODE_JOIN_STYPE,
PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN,
PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN,
PHY_HASH_JOIN_CODE_LEFT_PRIM_EXPR,
PHY_HASH_JOIN_CODE_RIGHT_PRIM_EXPR,
PHY_HASH_JOIN_CODE_LEFT_PRIM_SLOTID,
PHY_HASH_JOIN_CODE_RIGHT_PRIM_SLOTID,
PHY_HASH_JOIN_CODE_TIME_RANGE_TARGET,
PHY_HASH_JOIN_CODE_ON_CONDITIONS,
PHY_HASH_JOIN_CODE_TARGETS,
PHY_HASH_JOIN_CODE_INPUT_ROW_NUM0,
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE0,
PHY_HASH_JOIN_CODE_INPUT_ROW_NUM1,
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1
PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1,
PHY_HASH_JOIN_CODE_LEFT_ON_COND,
PHY_HASH_JOIN_CODE_RIGHT_ON_COND,
PHY_HASH_JOIN_CODE_TIME_RANGE_SKEY,
PHY_HASH_JOIN_CODE_TIME_RANGE_EKEY,
};
static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2547,6 +2713,9 @@ static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_HASH_JOIN_CODE_JOIN_TYPE, pNode->joinType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeEnum(pEncoder, PHY_HASH_JOIN_CODE_JOIN_STYPE, pNode->subType);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN, nodeListToMsg, pNode->pOnLeft);
}
@ -2554,7 +2723,22 @@ static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN, nodeListToMsg, pNode->pOnRight);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pFilterConditions);
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_LEFT_PRIM_EXPR, nodeToMsg, pNode->leftPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_RIGHT_PRIM_EXPR, nodeToMsg, pNode->rightPrimExpr);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_LEFT_PRIM_SLOTID, pNode->leftPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_RIGHT_PRIM_SLOTID, pNode->rightPrimSlotId);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_TIME_RANGE_TARGET, pNode->timeRangeTarget);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pFullOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets);
@ -2571,6 +2755,19 @@ static int32_t physiHashJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI32(pEncoder, PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1, pNode->inputStat[1].inputRowSize);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_LEFT_ON_COND, nodeToMsg, pNode->pLeftOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeObj(pEncoder, PHY_HASH_JOIN_CODE_RIGHT_ON_COND, nodeToMsg, pNode->pRightOnCond);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_HASH_JOIN_CODE_TIME_RANGE_SKEY, pNode->timeRange.skey);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeI64(pEncoder, PHY_HASH_JOIN_CODE_TIME_RANGE_EKEY, pNode->timeRange.ekey);
}
return code;
}
@ -2588,14 +2785,32 @@ static int32_t msgToPhysiHashJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_HASH_JOIN_CODE_JOIN_TYPE:
code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType));
break;
case PHY_HASH_JOIN_CODE_JOIN_STYPE:
code = tlvDecodeEnum(pTlv, &pNode->subType, sizeof(pNode->subType));
break;
case PHY_HASH_JOIN_CODE_ON_LEFT_COLUMN:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pOnLeft);
break;
case PHY_HASH_JOIN_CODE_ON_RIGHT_COLUMN:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pOnRight);
break;
case PHY_HASH_JOIN_CODE_LEFT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->leftPrimExpr);
break;
case PHY_HASH_JOIN_CODE_RIGHT_PRIM_EXPR:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->rightPrimExpr);
break;
case PHY_HASH_JOIN_CODE_LEFT_PRIM_SLOTID:
code = tlvDecodeI32(pTlv, &pNode->leftPrimSlotId);
break;
case PHY_HASH_JOIN_CODE_RIGHT_PRIM_SLOTID:
code = tlvDecodeI32(pTlv, &pNode->rightPrimSlotId);
break;
case PHY_HASH_JOIN_CODE_TIME_RANGE_TARGET:
code = tlvDecodeI32(pTlv, &pNode->timeRangeTarget);
break;
case PHY_HASH_JOIN_CODE_ON_CONDITIONS:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFilterConditions);
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFullOnCond);
break;
case PHY_HASH_JOIN_CODE_TARGETS:
code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets);
@ -2612,6 +2827,18 @@ static int32_t msgToPhysiHashJoinNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_HASH_JOIN_CODE_INPUT_ROW_SIZE1:
code = tlvDecodeI32(pTlv, &pNode->inputStat[1].inputRowSize);
break;
case PHY_HASH_JOIN_CODE_LEFT_ON_COND:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pLeftOnCond);
break;
case PHY_HASH_JOIN_CODE_RIGHT_ON_COND:
code = msgToNodeFromTlv(pTlv, (void**)&pNode->pRightOnCond);
break;
case PHY_HASH_JOIN_CODE_TIME_RANGE_SKEY:
code = tlvDecodeI64(pTlv, &pNode->timeRange.skey);
break;
case PHY_HASH_JOIN_CODE_TIME_RANGE_EKEY:
code = tlvDecodeI64(pTlv, &pNode->timeRange.ekey);
break;
default:
break;
}
@ -2627,7 +2854,8 @@ enum {
PHY_AGG_CODE_GROUP_KEYS,
PHY_AGG_CODE_AGG_FUNCS,
PHY_AGG_CODE_MERGE_DATA_BLOCK,
PHY_AGG_CODE_GROUP_KEY_OPTIMIZE
PHY_AGG_CODE_GROUP_KEY_OPTIMIZE,
PHY_AGG_CODE_HAS_COUNT_LIKE_FUNCS,
};
static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
@ -2649,6 +2877,9 @@ static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_GROUP_KEY_OPTIMIZE, pNode->groupKeyOptimized);
}
if (TSDB_CODE_SUCCESS == code) {
code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_HAS_COUNT_LIKE_FUNCS, pNode->hasCountLikeFunc);
}
return code;
}
@ -2678,6 +2909,9 @@ static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) {
case PHY_AGG_CODE_GROUP_KEY_OPTIMIZE:
code = tlvDecodeBool(pTlv, &pNode->groupKeyOptimized);
break;
case PHY_AGG_CODE_HAS_COUNT_LIKE_FUNCS:
code = tlvDecodeBool(pTlv, &pNode->hasCountLikeFunc);
break;
default:
break;
}
@ -4222,6 +4456,9 @@ static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) {
case QUERY_NODE_CASE_WHEN:
code = caseWhenNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_WINDOW_OFFSET:
code = windowOffsetNodeToMsg(pObj, pEncoder);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = physiTagScanNodeToMsg(pObj, pEncoder);
break;
@ -4381,6 +4618,9 @@ static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) {
case QUERY_NODE_CASE_WHEN:
code = msgToCaseWhenNode(pDecoder, pObj);
break;
case QUERY_NODE_WINDOW_OFFSET:
code = msgToWindowOffsetNode(pDecoder, pObj);
break;
case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN:
code = msgToPhysiTagScanNode(pDecoder, pObj);
break;

14
source/libs/nodes/src/nodesTraverseFuncs.c
View File

@ -372,6 +372,14 @@ static EDealRes rewriteExpr(SNode** pRawNode, ETraversalOrder order, FNodeRewrit
}
break;
}
case QUERY_NODE_WINDOW_OFFSET: {
SWindowOffsetNode* pWin = (SWindowOffsetNode*)pNode;
res = rewriteExpr(&pWin->pStartOffset, order, rewriter, pContext);
if (DEAL_RES_ERROR != res && DEAL_RES_END != res) {
res = rewriteExpr(&pWin->pEndOffset, order, rewriter, pContext);
}
break;
}
case QUERY_NODE_COUNT_WINDOW: {
SCountWindowNode* pEvent = (SCountWindowNode*)pNode;
res = rewriteExpr(&pEvent->pCol, order, rewriter, pContext);
@ -415,7 +423,7 @@ void nodesRewriteExprsPostOrder(SNodeList* pList, FNodeRewriter rewriter, void*
(void)rewriteExprs(pList, TRAVERSAL_POSTORDER, rewriter, pContext);
}
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext) {
void nodesWalkSelectStmtImpl(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext) {
if (NULL == pSelect) {
return;
}
@ -451,6 +459,10 @@ void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker wa
return;
}
void nodesWalkSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeWalker walker, void* pContext) {
nodesWalkSelectStmtImpl(pSelect, clause, walker, pContext);
}
void nodesRewriteSelectStmt(SSelectStmt* pSelect, ESqlClause clause, FNodeRewriter rewriter, void* pContext) {
if (NULL == pSelect) {
return;

262
source/libs/nodes/src/nodesUtilFuncs.c
View File

@ -44,6 +44,74 @@ struct SNodeAllocator {
static threadlocal SNodeAllocator* g_pNodeAllocator;
static int32_t g_allocatorReqRefPool = -1;
char* getJoinTypeString(EJoinType type) {
static char* joinType[] = {"", "INNER", "LEFT", "RIGHT", "FULL"};
return joinType[type];
}
char* getJoinSTypeString(EJoinSubType type) {
static char* joinSType[] = {"", "", "OUTER", "SEMI", "ANTI", "ANY", "ASOF", "WINDOW"};
return joinSType[type];
}
char* getFullJoinTypeString(EJoinType type, EJoinSubType stype) {
static char* joinFullType[][8] = {
{"INNER", "INNER", "INNER", "INNER", "INNER", "INNER ANY", "INNER", "INNER"},
{"LEFT", "LEFT", "LEFT OUTER", "LEFT SEMI", "LEFT ANTI", "LEFT ANY", "LEFT ASOF", "LEFT WINDOW"},
{"RIGHT", "RIGHT", "RIGHT OUTER", "RIGHT SEMI", "RIGHT ANTI", "RIGHT ANY", "RIGHT ASOF", "RIGHT WINDOW"},
{"FULL", "FULL", "FULL OUTER", "FULL", "FULL", "FULL ANY", "FULL", "FULL"}
};
return joinFullType[type][stype];
}
int32_t mergeJoinConds(SNode** ppDst, SNode** ppSrc) {
if (NULL == *ppSrc) {
return TSDB_CODE_SUCCESS;
}
if (NULL == *ppDst) {
*ppDst = *ppSrc;
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc) && ((SLogicConditionNode*)(*ppSrc))->condType == LOGIC_COND_TYPE_AND) {
TSWAP(*ppDst, *ppSrc);
}
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppDst)) {
SLogicConditionNode* pDst = (SLogicConditionNode*)*ppDst;
if (pDst->condType == LOGIC_COND_TYPE_AND) {
if (QUERY_NODE_LOGIC_CONDITION == nodeType(*ppSrc) && ((SLogicConditionNode*)(*ppSrc))->condType == LOGIC_COND_TYPE_AND) {
nodesListStrictAppendList(pDst->pParameterList, ((SLogicConditionNode*)(*ppSrc))->pParameterList);
((SLogicConditionNode*)(*ppSrc))->pParameterList = NULL;
} else {
nodesListStrictAppend(pDst->pParameterList, *ppSrc);
*ppSrc = NULL;
}
nodesDestroyNode(*ppSrc);
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
}
SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION);
if (NULL == pLogicCond) {
return TSDB_CODE_OUT_OF_MEMORY;
}
pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL;
pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
pLogicCond->condType = LOGIC_COND_TYPE_AND;
pLogicCond->pParameterList = nodesMakeList();
nodesListStrictAppend(pLogicCond->pParameterList, *ppSrc);
nodesListStrictAppend(pLogicCond->pParameterList, *ppDst);
*ppDst = (SNode*)pLogicCond;
*ppSrc = NULL;
return TSDB_CODE_SUCCESS;
}
static SNodeMemChunk* callocNodeChunk(SNodeAllocator* pAllocator) {
SNodeMemChunk* pNewChunk = taosMemoryCalloc(1, sizeof(SNodeMemChunk) + pAllocator->chunkSize);
if (NULL == pNewChunk) {
@ -308,6 +376,8 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SHintNode));
case QUERY_NODE_VIEW:
return makeNode(type, sizeof(SViewNode));
case QUERY_NODE_WINDOW_OFFSET:
return makeNode(type, sizeof(SWindowOffsetNode));
case QUERY_NODE_SET_OPERATOR:
return makeNode(type, sizeof(SSetOperator));
case QUERY_NODE_SELECT_STMT:
@ -449,6 +519,7 @@ SNode* nodesMakeNode(ENodeType type) {
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT:
return makeNode(type, sizeof(SShowStmt));
case QUERY_NODE_SHOW_TABLE_TAGS_STMT:
return makeNode(type, sizeof(SShowTableTagsStmt));
@ -491,6 +562,12 @@ SNode* nodesMakeNode(ENodeType type) {
return makeNode(type, sizeof(SCreateViewStmt));
case QUERY_NODE_DROP_VIEW_STMT:
return makeNode(type, sizeof(SDropViewStmt));
case QUERY_NODE_CREATE_TSMA_STMT:
return makeNode(type, sizeof(SCreateTSMAStmt));
case QUERY_NODE_DROP_TSMA_STMT:
return makeNode(type, sizeof(SDropTSMAStmt));
case QUERY_NODE_TSMA_OPTIONS:
return makeNode(type, sizeof(STSMAOptions));
case QUERY_NODE_LOGIC_PLAN_SCAN:
return makeNode(type, sizeof(SScanLogicNode));
case QUERY_NODE_LOGIC_PLAN_JOIN:
@ -739,6 +816,8 @@ void nodesDestroyNode(SNode* pNode) {
taosMemoryFreeClear(pReal->pMeta);
taosMemoryFreeClear(pReal->pVgroupList);
taosArrayDestroyEx(pReal->pSmaIndexes, destroySmaIndex);
taosArrayDestroyP(pReal->tsmaTargetTbVgInfo, taosMemoryFree);
taosArrayDestroy(pReal->tsmaTargetTbInfo);
break;
}
case QUERY_NODE_TEMP_TABLE:
@ -746,6 +825,9 @@ void nodesDestroyNode(SNode* pNode) {
break;
case QUERY_NODE_JOIN_TABLE: {
SJoinTableNode* pJoin = (SJoinTableNode*)pNode;
nodesDestroyNode(pJoin->pWindowOffset);
nodesDestroyNode(pJoin->pJLimit);
nodesDestroyNode(pJoin->addPrimCond);
nodesDestroyNode(pJoin->pLeft);
nodesDestroyNode(pJoin->pRight);
nodesDestroyNode(pJoin->pOnCond);
@ -837,6 +919,12 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyNode(pOptions->pDeleteMark);
break;
}
case QUERY_NODE_TSMA_OPTIONS: {
STSMAOptions* pOptions = (STSMAOptions*)pNode;
nodesDestroyList(pOptions->pFuncs);
nodesDestroyNode(pOptions->pInterval);
break;
}
case QUERY_NODE_LEFT_VALUE: // no pointer field
case QUERY_NODE_COLUMN_REF: // no pointer field
break;
@ -877,6 +965,12 @@ void nodesDestroyNode(SNode* pNode) {
taosArrayDestroyEx(pView->pSmaIndexes, destroySmaIndex);
break;
}
case QUERY_NODE_WINDOW_OFFSET: {
SWindowOffsetNode* pWin = (SWindowOffsetNode*)pNode;
nodesDestroyNode(pWin->pStartOffset);
nodesDestroyNode(pWin->pEndOffset);
break;
}
case QUERY_NODE_SET_OPERATOR: {
SSetOperator* pStmt = (SSetOperator*)pNode;
nodesDestroyList(pStmt->pProjectionList);
@ -1105,7 +1199,8 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_SHOW_VIEWS_STMT:
case QUERY_NODE_SHOW_GRANTS_FULL_STMT:
case QUERY_NODE_SHOW_GRANTS_LOGS_STMT:
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT: {
case QUERY_NODE_SHOW_CLUSTER_MACHINES_STMT:
case QUERY_NODE_SHOW_TSMAS_STMT: {
SShowStmt* pStmt = (SShowStmt*)pNode;
nodesDestroyNode(pStmt->pDbName);
nodesDestroyNode(pStmt->pTbName);
@ -1192,6 +1287,15 @@ void nodesDestroyNode(SNode* pNode) {
}
case QUERY_NODE_DROP_VIEW_STMT:
break;
case QUERY_NODE_CREATE_TSMA_STMT: {
SCreateTSMAStmt* pStmt = (SCreateTSMAStmt*)pNode;
nodesDestroyNode((SNode*)pStmt->pOptions);
if (pStmt->pReq) {
tFreeSMCreateSmaReq(pStmt->pReq);
taosMemoryFreeClear(pStmt->pReq);
}
break;
}
case QUERY_NODE_LOGIC_PLAN_SCAN: {
SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
@ -1206,14 +1310,26 @@ void nodesDestroyNode(SNode* pNode) {
nodesDestroyList(pLogicNode->pTags);
nodesDestroyNode(pLogicNode->pSubtable);
taosArrayDestroyEx(pLogicNode->pFuncTypes, destroyFuncParam);
taosArrayDestroyP(pLogicNode->pTsmaTargetTbVgInfo, taosMemoryFree);
taosArrayDestroy(pLogicNode->pTsmaTargetTbInfo);
break;
}
case QUERY_NODE_LOGIC_PLAN_JOIN: {
SJoinLogicNode* pLogicNode = (SJoinLogicNode*)pNode;
destroyLogicNode((SLogicNode*)pLogicNode);
nodesDestroyNode(pLogicNode->pWindowOffset);
nodesDestroyNode(pLogicNode->pJLimit);
nodesDestroyNode(pLogicNode->addPrimEqCond);
nodesDestroyNode(pLogicNode->pPrimKeyEqCond);
nodesDestroyNode(pLogicNode->pOtherOnCond);
nodesDestroyNode(pLogicNode->pColEqCond);
nodesDestroyNode(pLogicNode->pColOnCond);
nodesDestroyNode(pLogicNode->pTagEqCond);
nodesDestroyNode(pLogicNode->pTagOnCond);
nodesDestroyNode(pLogicNode->pFullOnCond);
nodesDestroyList(pLogicNode->pLeftEqNodes);
nodesDestroyList(pLogicNode->pRightEqNodes);
nodesDestroyNode(pLogicNode->pLeftOnCond);
nodesDestroyNode(pLogicNode->pRightOnCond);
break;
}
case QUERY_NODE_LOGIC_PLAN_AGG: {
@ -1357,10 +1473,17 @@ void nodesDestroyNode(SNode* pNode) {
case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: {
SSortMergeJoinPhysiNode* pPhyNode = (SSortMergeJoinPhysiNode*)pNode;
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyNode(pPhyNode->pWindowOffset);
nodesDestroyNode(pPhyNode->pJLimit);
nodesDestroyNode(pPhyNode->leftPrimExpr);
nodesDestroyNode(pPhyNode->rightPrimExpr);
nodesDestroyList(pPhyNode->pEqLeft);
nodesDestroyList(pPhyNode->pEqRight);
nodesDestroyNode(pPhyNode->pPrimKeyCond);
nodesDestroyNode(pPhyNode->pOtherOnCond);
nodesDestroyNode(pPhyNode->pFullOnCond);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pColEqCond);
nodesDestroyNode(pPhyNode->pColOnCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_JOIN: {
@ -1368,12 +1491,17 @@ void nodesDestroyNode(SNode* pNode) {
destroyPhysiNode((SPhysiNode*)pPhyNode);
nodesDestroyList(pPhyNode->pOnLeft);
nodesDestroyList(pPhyNode->pOnRight);
nodesDestroyNode(pPhyNode->pFilterConditions);
nodesDestroyNode(pPhyNode->leftPrimExpr);
nodesDestroyNode(pPhyNode->rightPrimExpr);
nodesDestroyNode(pPhyNode->pFullOnCond);
nodesDestroyList(pPhyNode->pTargets);
nodesDestroyNode(pPhyNode->pPrimKeyCond);
nodesDestroyNode(pPhyNode->pColEqCond);
nodesDestroyNode(pPhyNode->pTagEqCond);
nodesDestroyNode(pPhyNode->pLeftOnCond);
nodesDestroyNode(pPhyNode->pRightOnCond);
break;
}
case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: {
@ -1639,6 +1767,16 @@ int32_t nodesListMakeStrictAppendList(SNodeList** pTarget, SNodeList* pSrc) {
return nodesListStrictAppendList(*pTarget, pSrc);
}
int32_t nodesListMakePushFront(SNodeList** pList, SNode* pNode) {
if (*pList == NULL) {
*pList = nodesMakeList();
if (*pList == NULL) {
terrno = TSDB_CODE_OUT_OF_MEMORY;
return TSDB_CODE_OUT_OF_MEMORY;
}
}
return nodesListPushFront(*pList, pNode);
}
int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) {
if (NULL == pList || NULL == pNode) {
@ -1655,6 +1793,7 @@ int32_t nodesListPushFront(SNodeList* pList, SNode* pNode) {
p->pNext = pList->pHead;
}
pList->pHead = p;
pList->pTail = pList->pTail ? pList->pTail : p;
++(pList->length);
return TSDB_CODE_SUCCESS;
}
@ -2025,6 +2164,7 @@ bool nodesIsBitwiseOp(const SOperatorNode* pOp) {
typedef struct SCollectColumnsCxt {
int32_t errCode;
const char* pTableAlias;
SSHashObj* pMultiTableAlias;
ECollectColType collectType;
SNodeList* pCols;
SHashObj* pColHash;
@ -2066,6 +2206,19 @@ static EDealRes collectColumns(SNode* pNode, void* pContext) {
return DEAL_RES_CONTINUE;
}
static EDealRes collectColumnsExt(SNode* pNode, void* pContext) {
SCollectColumnsCxt* pCxt = (SCollectColumnsCxt*)pContext;
if (QUERY_NODE_COLUMN == nodeType(pNode)) {
SColumnNode* pCol = (SColumnNode*)pNode;
if (isCollectType(pCxt->collectType, pCol->colType) && 0 != strcmp(pCol->colName, "*") &&
(NULL == pCxt->pMultiTableAlias || NULL != (pCxt->pTableAlias = tSimpleHashGet(pCxt->pMultiTableAlias, pCol->tableAlias, strlen(pCol->tableAlias))))) {
return doCollect(pCxt, pCol, pNode);
}
}
return DEAL_RES_CONTINUE;
}
int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char* pTableAlias, ECollectColType type,
SNodeList** pCols) {
if (NULL == pSelect || NULL == pCols) {
@ -2097,6 +2250,38 @@ int32_t nodesCollectColumns(SSelectStmt* pSelect, ESqlClause clause, const char*
return TSDB_CODE_SUCCESS;
}
int32_t nodesCollectColumnsExt(SSelectStmt* pSelect, ESqlClause clause, SSHashObj* pMultiTableAlias, ECollectColType type,
SNodeList** pCols) {
if (NULL == pSelect || NULL == pCols) {
return TSDB_CODE_FAILED;
}
SCollectColumnsCxt cxt = {
.errCode = TSDB_CODE_SUCCESS,
.pTableAlias = NULL,
.pMultiTableAlias = pMultiTableAlias,
.collectType = type,
.pCols = (NULL == *pCols ? nodesMakeList() : *pCols),
.pColHash = taosHashInit(128, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK)};
if (NULL == cxt.pCols || NULL == cxt.pColHash) {
return TSDB_CODE_OUT_OF_MEMORY;
}
*pCols = NULL;
nodesWalkSelectStmtImpl(pSelect, clause, collectColumnsExt, &cxt);
taosHashCleanup(cxt.pColHash);
if (TSDB_CODE_SUCCESS != cxt.errCode) {
nodesDestroyList(cxt.pCols);
return cxt.errCode;
}
if (LIST_LENGTH(cxt.pCols) > 0) {
*pCols = cxt.pCols;
} else {
nodesDestroyList(cxt.pCols);
}
return TSDB_CODE_SUCCESS;
}
int32_t nodesCollectColumnsFromNode(SNode* node, const char* pTableAlias, ECollectColType type, SNodeList** pCols) {
if (NULL == pCols) {
return TSDB_CODE_FAILED;
@ -2437,7 +2622,6 @@ SValueNode* nodesMakeValueNodeFromString(char* literal) {
pValNode->datum.p = p;
pValNode->literal = tstrdup(literal);
pValNode->translate = true;
pValNode->isDuration = false;
pValNode->isNull = false;
}
return pValNode;
@ -2450,7 +2634,6 @@ SValueNode* nodesMakeValueNodeFromBool(bool b) {
pValNode->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes;
nodesSetValueNodeValue(pValNode, &b);
pValNode->translate = true;
pValNode->isDuration = false;
pValNode->isNull = false;
}
return pValNode;
@ -2465,3 +2648,70 @@ bool nodesIsTableStar(SNode* pNode) {
return (QUERY_NODE_COLUMN == nodeType(pNode)) && ('\0' != ((SColumnNode*)pNode)->tableAlias[0]) &&
(0 == strcmp(((SColumnNode*)pNode)->colName, "*"));
}
void nodesSortList(SNodeList** pList, int32_t (*comp)(SNode* pNode1, SNode* pNode2)) {
if ((*pList)->length == 1) return;
uint32_t inSize = 1;
SListCell* pHead = (*pList)->pHead;
while (1) {
SListCell* p = pHead;
pHead = NULL;
SListCell* pTail = NULL;
uint32_t nMerges = 0;
while (p) {
++nMerges;
SListCell* q = p;
uint32_t pSize = 0;
for (uint32_t i = 0; i < inSize; ++i) {
++pSize;
q = q->pNext;
if (!q) {
break;
}
}
uint32_t qSize = inSize;
while (pSize > 0 || (qSize > 0 && q)) {
SListCell* pCell;
if (pSize == 0) {
pCell = q;
q = q->pNext;
--qSize;
} else if (qSize == 0 || !q) {
pCell = p;
p = p->pNext;
--pSize;
} else if (comp(q->pNode, p->pNode) >= 0) {
pCell = p;
p = p->pNext;
--pSize;
} else {
pCell = q;
q = q->pNext;
--qSize;
}
if (pTail) {
pTail->pNext = pCell;
pCell->pPrev = pTail;
} else {
pHead = pCell;
pHead->pPrev = NULL;
}
pTail = pCell;
}
p = q;
}
pTail->pNext = NULL;
if (nMerges <= 1) {
(*pList)->pHead = pHead;
(*pList)->pTail = pTail;
return;
}
inSize *= 2;
}
}

63
source/libs/nodes/test/nodesTestMain.cpp
View File

@ -56,6 +56,69 @@ TEST(NodesTest, traverseTest) {
nodesDestroyNode(pRoot);
}
int32_t compareValueNode(SNode* pNode1, SNode* pNode2) {
SValueNode* p1 = (SValueNode*)pNode1;
SValueNode* p2 = (SValueNode*)pNode2;
if (p1->datum.i < p2->datum.i)
return -1;
else if (p1->datum.i > p2->datum.i)
return 1;
else
return 0;
}
void assert_sort_result(SNodeList* pList) {
SNode* pNode;
int32_t i = 0;
FOREACH(pNode, pList) {
SValueNode* p = (SValueNode*)pNode;
ASSERT_EQ(p->datum.i, i++);
}
SListCell* pCell = pList->pHead;
ASSERT_TRUE(pCell->pPrev == NULL);
ASSERT_TRUE(pList->pTail->pNext == NULL);
int32_t len = 1;
while (pCell) {
if (pCell->pNext) {
ASSERT_TRUE(pCell->pNext->pPrev == pCell);
}
pCell = pCell->pNext;
if (pCell) len++;
}
ASSERT_EQ(len, pList->length);
}
TEST(NodesTest, sort) {
SValueNode *vn1 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn1->datum.i = 4;
SValueNode *vn2 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn2->datum.i = 3;
SValueNode *vn3 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn3->datum.i = 2;
SValueNode *vn4 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn4->datum.i = 1;
SValueNode *vn5 = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE);
vn5->datum.i = 0;
SNodeList* l = NULL;
nodesListMakeAppend(&l, (SNode*)vn1);
nodesListMakeAppend(&l, (SNode*)vn2);
nodesListMakeAppend(&l, (SNode*)vn3);
nodesListMakeAppend(&l, (SNode*)vn4);
nodesListMakeAppend(&l, (SNode*)vn5);
nodesSortList(&l, compareValueNode);
assert_sort_result(l);
nodesDestroyList(l);
}
int main(int argc, char* argv[]) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();

18
source/libs/parser/inc/parAst.h
View File

@ -116,6 +116,7 @@ SNode* createRawValueNodeExt(SAstCreateContext* pCxt, int32_t dataType, const ST
SNodeList* createHintNodeList(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createIdentifierValueNode(SAstCreateContext* pCxt, SToken* pLiteral);
SNode* createDurationValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createTimeOffsetValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* createDefaultDatabaseCondValue(SAstCreateContext* pCxt);
SNode* createPlaceholderValueNode(SAstCreateContext* pCxt, const SToken* pLiteral);
SNode* setProjectionAlias(SAstCreateContext* pCxt, SNode* pNode, SToken* pAlias);
@ -129,7 +130,7 @@ SNode* createNodeListNode(SAstCreateContext* pCxt, SNodeList* pList);
SNode* createNodeListNodeEx(SAstCreateContext* pCxt, SNode* p1, SNode* p2);
SNode* createRealTableNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pTableName, SToken* pTableAlias);
SNode* createTempTableNode(SAstCreateContext* pCxt, SNode* pSubquery, const SToken* pTableAlias);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createJoinTableNode(SAstCreateContext* pCxt, EJoinType type, EJoinSubType stype, SNode* pLeft, SNode* pRight, SNode* pJoinCond);
SNode* createViewNode(SAstCreateContext* pCxt, SToken* pDbName, SToken* pViewName);
SNode* createLimitNode(SAstCreateContext* pCxt, const SToken* pLimit, const SToken* pOffset);
SNode* createOrderByExprNode(SAstCreateContext* pCxt, SNode* pExpr, EOrder order, ENullOrder nullOrder);
@ -139,6 +140,7 @@ SNode* createEventWindowNode(SAstCreateContext* pCxt, SNode* pStartCond, SNo
SNode* createCountWindowNode(SAstCreateContext* pCxt, const SToken* pCountToken, const SToken* pSlidingToken);
SNode* createIntervalWindowNode(SAstCreateContext* pCxt, SNode* pInterval, SNode* pOffset, SNode* pSliding,
SNode* pFill);
SNode* createWindowOffsetNode(SAstCreateContext* pCxt, SNode* pStartOffset, SNode* pEndOffset);
SNode* createFillNode(SAstCreateContext* pCxt, EFillMode mode, SNode* pValues);
SNode* createGroupingSetNode(SAstCreateContext* pCxt, SNode* pNode);
SNode* createInterpTimeRange(SAstCreateContext* pCxt, SNode* pStart, SNode* pEnd);
@ -157,8 +159,9 @@ SNode* addLimitClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pLimit);
SNode* addRangeClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pRange);
SNode* addEveryClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pEvery);
SNode* addFillClause(SAstCreateContext* pCxt, SNode* pStmt, SNode* pFill);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable,
SNodeList* pHint);
SNode* addJLimitClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pJLimit);
SNode* addWindowOffsetClause(SAstCreateContext* pCxt, SNode* pJoin, SNode* pWinOffset);
SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
SNode* setSelectStmtTagMode(SAstCreateContext* pCxt, SNode* pStmt, bool bSelectTags);
SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* pLeft, SNode* pRight);
@ -271,6 +274,15 @@ SNode* createCreateViewStmt(SAstCreateContext* pCxt, bool orReplace, SNode* pVie
SNode* createDropViewStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pView);
SNode* createShowCompactDetailsStmt(SAstCreateContext* pCxt, SNode* pCompactIdNode);
SNode* createShowCompactsStmt(SAstCreateContext* pCxt, ENodeType type);
SNode* createCreateTSMAStmt(SAstCreateContext* pCxt, bool ignoreExists, SToken* tsmaName, SNode* pOptions,
SNode* pRealTable, SNode* pInterval);
SNode* createTSMAOptions(SAstCreateContext* pCxt, SNodeList* pFuncs);
SNode* createDefaultTSMAOptions(SAstCreateContext* pCxt);
SNode* createDropTSMAStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SNode* pRealTable);
SNode* createShowCreateTSMAStmt(SAstCreateContext* pCxt, SNode* pRealTable);
SNode* createShowTSMASStmt(SAstCreateContext* pCxt, SNode* dbName);
#ifdef __cplusplus
}
#endif

3
source/libs/parser/inc/parInt.h
View File

@ -37,8 +37,9 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS
int32_t calculateConstant(SParseContext* pParseCxt, SQuery* pQuery);
int32_t translatePostCreateStream(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t translatePostCreateSmaIndex(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t translatePostCreateTSMA(SParseContext* pParseCxt, SQuery* pQuery, SSDataBlock* pBlock);
int32_t buildQueryAfterParse(SQuery** pQuery, SNode* pRootNode, int16_t placeholderNo, SArray** pPlaceholderValues);
int32_t translateTable(STranslateContext* pCxt, SNode** pTable);
int32_t translateTable(STranslateContext* pCxt, SNode** pTable, SNode* pJoinParent);
int32_t getMetaDataFromHash(const char* pKey, int32_t len, SHashObj* pHash, void** pOutput);
void tfreeSParseQueryRes(void* p);

7
source/libs/parser/inc/parUtil.h
View File

@ -110,6 +110,8 @@ typedef struct SParseMetaCache {
SHashObj* pTableIndex; // key is tbFName, element is SArray<STableIndexInfo>*
SHashObj* pTableCfg; // key is tbFName, element is STableCfg*
SHashObj* pViews; // key is viewFName, element is SViewMeta*
SHashObj* pTableTSMAs; // key is tbFName, elements are SArray<STableTSMAInfo*>
SHashObj* pTSMAs; // key is tsmaFName, elemetns are STableTSMAInfo*
SArray* pDnodes; // element is SEpSet
bool dnodeRequired;
} SParseMetaCache;
@ -117,6 +119,7 @@ typedef struct SParseMetaCache {
int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...);
int32_t generateSyntaxErrMsgExt(SMsgBuf* pBuf, int32_t errCode, const char* pFormat, ...);
int32_t buildInvalidOperationMsg(SMsgBuf* pMsgBuf, const char* msg);
int32_t buildInvalidOperationMsgExt(SMsgBuf* pBuf, const char* pFormat, ...);
int32_t buildSyntaxErrMsg(SMsgBuf* pBuf, const char* additionalInfo, const char* sourceStr);
SSchema* getTableColumnSchema(const STableMeta* pTableMeta);
@ -152,6 +155,8 @@ int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache);
int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveDnodeRequiredInCache(SParseMetaCache* pMetaCache);
int32_t reserveTableTSMAInfoInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache);
int32_t reserveTSMAInfoInCache(int32_t acctId, const char* pDb, const char* pTsmaName, SParseMetaCache* pMetaCache);
int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
int32_t getViewMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta);
int32_t buildTableMetaFromViewMeta(STableMeta** pMeta, SViewMeta* pViewMeta);
@ -167,6 +172,8 @@ int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, ST
int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes);
void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request);
SNode* createSelectStmtImpl(bool isDistinct, SNodeList* pProjectionList, SNode* pTable, SNodeList* pHint);
int32_t getTableTsmasFromCache(SParseMetaCache* pMetaCache, const SName* pTbName, SArray** pTsmas);
int32_t getTsmaFromCache(SParseMetaCache* pMetaCache, const SName* pTsmaName, STableTSMAInfo** pTsma);
/**
* @brief return a - b with overflow check

Some files were not shown because too many files have changed in this diff Show More